diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml index 958b463..9e50bb0 100644 --- a/.github/workflows/test-jvm.yml +++ b/.github/workflows/test-jvm.yml @@ -750,7 +750,7 @@ jobs: echo "**Package**: arcadedb" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "ℹ️ **Note**: Some platform/Python combinations are excluded from testing:" >> $GITHUB_STEP_SUMMARY - echo "- Windows ARM64 (no GitHub-hosted runners available)" >> $GITHUB_STEP_SUMMARY + echo "- Windows ARM64 (GitHub-hosted windows-11-arm runners exist, but jpype1 1.7.1 ships no win_arm64 wheel and GraalVM CE 25.4.4.1.1 has no Windows ARM64 build, so neither package can be built there)" >> $GITHUB_STEP_SUMMARY echo "- macOS x86_64 (temporarily disabled)" >> $GITHUB_STEP_SUMMARY else echo "❌ **Some platforms failed testing**" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index e44865f..295ccdf 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -18,7 +18,7 @@ # # One wheel per platform, py3-none-: # linux-x86_64 manylinux_2_28 container manylinux_2_28_x86_64 libarcadedb.so + libjvector-x86_64.so -# linux-aarch64 manylinux_2_28 aarch64 container manylinux_2_28_aarch64 libarcadedb.so +# linux-aarch64 manylinux_2_28 aarch64 container manylinux_2_28_aarch64 libarcadedb.so + libjvector-aarch64.so # macos-arm64 native, heap and threads capped macosx__arm64 libarcadedb.dylib # windows-x64 native, MSVC environment loaded win_amd64 libarcadedb.dll # Only free GitHub-hosted standard runners. GraalVM Community Edition only (pinned by sha256 per platform), never Oracle GraalVM. @@ -426,6 +426,18 @@ jobs: dll="$(ls "$T"/lib/libarcadedb.dll)" dumpbin //dependents "$dll" | tee "$T/dumpbin.txt" + - name: "Gate: a Linux wheel carries jvector's library for its architecture" + if: ${{ matrix.kind == 'linux' }} + env: + PLAT: ${{ matrix.plat_re }} + run: | + set -euo pipefail + w="$(ls "$T"/wheel/dist/*.whl)" + arch="${KEY#linux-}" # linux-x86_64 -> x86_64, linux-aarch64 -> aarch64 (${PLAT##*_} gave "64" for x86_64) + unzip -Z1 "$w" | tee "$T/wheel-files.txt" > /dev/null + grep -Fx "arcadedb/_native/libarcadedb.so" "$T/wheel-files.txt" + grep -Fx "arcadedb/_native/libjvector-${arch}.so" "$T/wheel-files.txt" + - name: "Gate: twine check" run: uvx twine check --strict "$T"/wheel/dist/*.whl diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh index 72ac2aa..afebba6 100755 --- a/native/scripts/build-image.sh +++ b/native/scripts/build-image.sh @@ -4,7 +4,7 @@ # Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference). # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR] A laptop with other jobs runs it through its job gate; the CI calls it directly. -# Platforms: Linux (libarcadedb.so; on x86-64 also libjvector-x86_64.so, jvector ships the Highway library for x86-64 only here), macOS +# Platforms: Linux (libarcadedb.so; also jvector's Highway library, libjvector-x86_64.so or libjvector-aarch64.so, both are in the jvector 4.0.2 jar), macOS # (libarcadedb.dylib) and Windows in Git Bash with the MSVC environment loaded (libarcadedb.dll). The CI (test-native.yml) caps the # builder heap and threads on the 7 GB macOS runner (XMX=5g, NI_EXTRA=-H:NumberOfThreads=2). # e.g. NEED_GB=13 JOB_MEM_GB=14 quiet_build.sh bash native/scripts/build-image.sh @@ -55,15 +55,24 @@ grep -a -E -q 'PriorityForceInline. \(origin\(s\): .META-INF.native-image.org\.h INLINE_LOG="PriorityForceInline picked up from native-image.properties ($(grep -a -c 'PriorityForceInline' "$OUT/build.log") log lines)" cp "target/$LIBFILE" "$OUT/" LIBS=("$OUT/$LIBFILE") -# Highway (jvector's native provider), found next to libarcadedb.so by the image (ARCADEDB_NATIVE_LIBDIR); Linux x86-64 only: the other -# platforms ship no jvector library and the image falls back to the Java providers -if [ "$EXT" = so ] && [ "$(uname -m)" = x86_64 ]; then - J=$(unzip -l "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" | awk '/libjvector-x86_64.so/{print $4}' | head -1) - if [ -n "$J" ]; then unzip -p "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" "$J" > "$OUT/libjvector-x86_64.so"; else - echo "libjvector-x86_64.so is not in the jvector 4.0.2 jar" >&2 - exit 1 +# Highway (jvector's native library), found next to libarcadedb.so by the image (ARCADEDB_NATIVE_LIBDIR); Linux only: the jvector 4.0.2 jar +# carries libjvector-x86_64.so and libjvector-aarch64.so and nothing for macOS or Windows. The image uses it only when +# ARCADEDB_VECTOR_PROVIDER=native is set (the engine forces jvector's scalar provider in every image otherwise). +if [ "$EXT" = so ]; then + case "$(uname -m)" in + x86_64) JARCH=x86_64 ;; + aarch64 | arm64) JARCH=aarch64 ;; + *) JARCH= ;; + esac + if [ -n "$JARCH" ]; then + JAR="$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" + J=$(unzip -l "$JAR" | awk -v n="libjvector-$JARCH.so" 'index($4, n) {print $4}' | head -1) + if [ -n "$J" ]; then unzip -p "$JAR" "$J" > "$OUT/libjvector-$JARCH.so"; else + echo "libjvector-$JARCH.so is not in the jvector 4.0.2 jar" >&2 + exit 1 + fi + LIBS+=("$OUT/libjvector-$JARCH.so") fi - LIBS+=("$OUT/libjvector-x86_64.so") fi sha256 "${LIBS[@]}" > "$OUT/sha256.txt" echo "native image done: engine $ENGINE; march $MARCH; $INLINE_LOG; $(grep -a 'Finished generating' "$OUT/build.log")" | tee "$OUT/build.done" diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 37eb836..4e16caf 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -56,8 +56,10 @@ public final class ArcadeNative { } static { - // ARCADEDB_VECTOR_PROVIDER = native | panama | default (jvector's VectorizationProvider, forced); unset: jvector chooses - // (Highway through the FFM downcalls when the bundled libjvector loads, else the Vector API classes). + // ARCADEDB_VECTOR_PROVIDER = native | panama | default (jvector's VectorizationProvider, forced). Unset, the engine's + // VectorApiSupport.configureJVectorProvider() (ArcadeData/arcadedb#10077) sets jvector.vectorization_provider=default in every + // native image, so the distance kernels are jvector's scalar DefaultVectorizationProvider. Highway (the bundled libjvector + // through the FFM downcalls, Linux x86-64 and aarch64) runs only with ARCADEDB_VECTOR_PROVIDER=native. final String forced = System.getenv("ARCADEDB_VECTOR_PROVIDER"); if (forced != null && !forced.isEmpty() && System.getProperty("jvector.vectorization_provider") == null) System.setProperty("jvector.vectorization_provider", forced); diff --git a/native/wheel/NOTICE b/native/wheel/NOTICE index 6d17094..3b6427b 100644 --- a/native/wheel/NOTICE +++ b/native/wheel/NOTICE @@ -17,13 +17,13 @@ GraalVM native-image runtime and JDK classes in libarcadedb.so Other GraalVM modules carry the licences in licenses/GraalVM-THIRD_PARTY_LICENSE.txt (graal-sdk and polyglot: UPL-1.0). Oracle GraalVM (GFTC) is not used; only Community Edition. -libjvector-x86_64.so +libjvector-x86_64.so, libjvector-aarch64.so The native vector kernels of JVector 4.0.2 (https://github.com/datastax/jvector), Apache-2.0, unmodified, taken from the jvector jar. Java libraries, the Highway/jvector kernels and JDK data inside libarcadedb.so: THIRD_PARTY_NOTICES.md (generated by gen_notices.py). System libraries linked dynamically and NOT shipped: Linux: libz.so.1, libm.so.6, libc.so.6, libpthread.so.0, libdl.so.2, librt.so.1 (libarcadedb.so, built in a manylinux_2_28 container); - libstdc++.so.6, libgcc_s.so.1 (libjvector-x86_64.so only) + libstdc++.so.6, libgcc_s.so.1 (the libjvector libraries only) macOS: the system libraries of macOS Windows: the Microsoft Visual C++ runtime (VCRUNTIME140.dll, VCRUNTIME140_1.dll) and the Windows C runtime (api-ms-win-crt-*.dll); CPython itself needs the same files diff --git a/native/wheel/THIRD_PARTY_NOTICES.md b/native/wheel/THIRD_PARTY_NOTICES.md index 49708d9..8bfe2dd 100644 --- a/native/wheel/THIRD_PARTY_NOTICES.md +++ b/native/wheel/THIRD_PARTY_NOTICES.md @@ -72,7 +72,7 @@ Where a pom lists several licences they are alternatives (JTS: EPL-2.0 or EDL-1. ## Native code shipped next to the library -`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway library, pinned there as a git submodule at commit `16f5f5c768064d0bb52952888a14236bec75c104`. Highway's `LICENSE` at that commit says: dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped). +`libjvector-x86_64.so` (Linux x86-64) and `libjvector-aarch64.so` (Linux aarch64) are taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Their sources are in the jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway library, pinned there as a git submodule at commit `16f5f5c768064d0bb52952888a14236bec75c104`. Highway's `LICENSE` at that commit says: dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. The libraries link `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped). ## GraalVM Community Edition runtime and JDK classes in libarcadedb.so @@ -1313,7 +1313,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #### highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt [b8b1fae5894c] -Applies to: libjvector-x86_64.so (google/highway) +Applies to: libjvector-x86_64.so and libjvector-aarch64.so (google/highway) ``` This project is primarily dual-licensed under your choice of either the Apache diff --git a/native/wheel/build_wheel.py b/native/wheel/build_wheel.py index faf6e55..c07c96d 100644 --- a/native/wheel/build_wheel.py +++ b/native/wheel/build_wheel.py @@ -5,7 +5,7 @@ Steps: copy src/arcadedb to /stage/src/arcadedb (the shared tree is not duplicated in git; only the install hints of the extras are renamed from arcadedb-jvm[...] to arcadedb[...]); rewrite `import jpype` to the bundled stub; flip the default backend to native; add the library of the build platform (libarcadedb.so on Linux, -libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux x86-64 also libjvector-x86_64.so, jvector's Highway library: no other platform ships it); +libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux also libjvector-.so, jvector's Highway library: macOS and Windows have none); add LICENSE/NOTICE/GraalVM licence texts; check THIRD_PARTY_NOTICES.md against gen_notices.py (fails on drift); compute the platform tag from the library (Linux: manylinux__ from the GLIBC symbol versions, never below --glibc-policy; macOS: macosx__arm64 from the load commands; Windows: win_amd64); build with `uv build --wheel`. Run it on the platform the library was built on. @@ -29,7 +29,19 @@ "darwin": "libarcadedb.dylib", "win32": "libarcadedb.dll", } -JVECTOR = "libjvector-x86_64.so" # Highway, taken from the jvector jar by preview-build.sh on Linux x86-64 only +# Highway, taken from the jvector 4.0.2 jar by build-image.sh on Linux (the jar has no macOS or Windows library) +JVECTOR_BY_ARCH = { + "x86_64": "libjvector-x86_64.so", + "aarch64": "libjvector-aarch64.so", +} + + +def libraries_to_ship(sys_platform: str, arch: str, libname: str) -> list[str]: + """The files of --lib-dir the wheel carries next to the engine library: libname, and on Linux the jvector library of the architecture.""" + ship = [libname] + if sys_platform.startswith("linux") and arch in JVECTOR_BY_ARCH: + ship.append(JVECTOR_BY_ARCH[arch]) + return ship def host_arch() -> str: @@ -94,7 +106,7 @@ def main(): ap.add_argument( "--lib-dir", required=True, - help="directory with the library of this platform (libarcadedb.so / .dylib / .dll) and, on Linux x86-64, libjvector-x86_64.so", + help="directory with the library of this platform (libarcadedb.so / .dylib / .dll) and, on Linux, libjvector-.so", ) ap.add_argument( "--version", @@ -245,11 +257,7 @@ def sub(m): ) ip.write_text(t, encoding="utf-8") - ship = [libname] - if sys.platform.startswith("linux") and host_arch() == "x86_64": - ship.append( - JVECTOR - ) # the only platform that ships Highway; lib.py does not need it elsewhere + ship = libraries_to_ship(sys.platform, host_arch(), libname) for n in ship: shutil.copy( lib_dir / n, dst / "_native" / n diff --git a/native/wheel/gen_notices.py b/native/wheel/gen_notices.py index a3f8a18..64dc8e5 100644 --- a/native/wheel/gen_notices.py +++ b/native/wheel/gen_notices.py @@ -287,17 +287,17 @@ def main(): # ---- native code ---- w("## Native code shipped next to the library\n") w( - "`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the " + "`libjvector-x86_64.so` (Linux x86-64) and `libjvector-aarch64.so` (Linux aarch64) are taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Their sources are in the " "jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway " f"library, pinned there as a git submodule at commit `{HIGHWAY_COMMIT}`. Highway's `LICENSE` at that commit says: " "dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). " "Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. " - "The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).\n" + "The libraries link `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).\n" ) for f, who in ( ( f"highway-{HIGHWAY_COMMIT}.LICENSE.txt", - "libjvector-x86_64.so (google/highway)", + "libjvector-x86_64.so and libjvector-aarch64.so (google/highway)", ), ("jvector-4.0.2.NOTICE.txt", "io.github.jbellis:jvector:4.0.2"), ): diff --git a/scripts/select_matrix.py b/scripts/select_matrix.py index 05e7d9d..c6e340c 100644 --- a/scripts/select_matrix.py +++ b/scripts/select_matrix.py @@ -36,8 +36,10 @@ import os import sys -# Temporarily limited to four platforms: macOS x86_64 (runner macos-15-intel) and Windows ARM64 -# (windows-11-arm) are off. This is the platform column both workflows had before this script. +# Temporarily limited to four platforms: macOS x86_64 (runner macos-15-intel) is off. Windows ARM64 (runner +# windows-11-arm exists) is off because neither package can be built there: jpype1 1.7.1 ships no win_arm64 wheel +# (win_amd64 only) and GraalVM CE 25.4.4.1.1 has no Windows ARM64 build (windows-x64 only). This is the platform +# column both workflows had before this script. RUNNERS = { "linux/amd64": "ubuntu-24.04", "linux/arm64": "ubuntu-24.04-arm", @@ -85,7 +87,9 @@ def _legs(per_platform: dict[str, tuple[str, ...]]) -> list[dict[str, str]]: def select(workflow: str, event: str) -> dict[str, list[dict[str, str]]]: """The `strategy.matrix` of `workflow` ("jvm" or "examples") for `event`.""" if workflow not in WORKFLOWS: - raise ValueError(f"unknown workflow {workflow!r}; use one of {sorted(WORKFLOWS)}") + raise ValueError( + f"unknown workflow {workflow!r}; use one of {sorted(WORKFLOWS)}" + ) if workflow == "jvm": return {"include": _legs(JVM_LEGS)} if event == "pull_request": diff --git a/tests/test_native_jvector_library.py b/tests/test_native_jvector_library.py new file mode 100644 index 0000000..a927b2b --- /dev/null +++ b/tests/test_native_jvector_library.py @@ -0,0 +1,93 @@ +"""The native `arcadedb` wheel carries jvector's Highway library for the architecture it is built for. + +The jvector 4.0.2 jar has `libjvector-x86_64.so` and `libjvector-aarch64.so` and nothing for macOS or Windows. The Linux x86-64 +wheel shipped the first, the Linux aarch64 wheel shipped none (#144). These tests check the builder's ship list and the build +script's copy step, and ArcadeNative's comment on the provider default. + +Repository tooling: no engine, no JVM and no native library needed. +""" + +import importlib.util +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +BUILD_WHEEL = ROOT / "native" / "wheel" / "build_wheel.py" +BUILD_IMAGE = ROOT / "native" / "scripts" / "build-image.sh" +ARCADE_NATIVE = ( + ROOT + / "native" + / "src" + / "main" + / "java" + / "org" + / "humemai" + / "nativelib" + / "ArcadeNative.java" +) + + +def _build_wheel(): + spec = importlib.util.spec_from_file_location("native_build_wheel", BUILD_WHEEL) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "sys_platform, arch, libname, expected", + [ + ( + "linux", + "x86_64", + "libarcadedb.so", + ["libarcadedb.so", "libjvector-x86_64.so"], + ), + ( + "linux", + "aarch64", + "libarcadedb.so", + ["libarcadedb.so", "libjvector-aarch64.so"], + ), + ("darwin", "aarch64", "libarcadedb.dylib", ["libarcadedb.dylib"]), + ("win32", "x86_64", "libarcadedb.dll", ["libarcadedb.dll"]), + ("linux", "riscv64", "libarcadedb.so", ["libarcadedb.so"]), + ], +) +def test_the_wheel_ships_the_jvector_library_of_its_linux_architecture( + sys_platform, arch, libname, expected +): + assert _build_wheel().libraries_to_ship(sys_platform, arch, libname) == expected + + +def test_the_host_architecture_names_map_to_a_library_the_loader_looks_for(): + names = _build_wheel().JVECTOR_BY_ARCH + assert names == { + "x86_64": "libjvector-x86_64.so", + "aarch64": "libjvector-aarch64.so", + } + loader = ( + ROOT + / "native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java" + ).read_text() + for name in names.values(): + assert f'"/{name}"' in loader + + +def test_the_image_script_copies_the_library_of_the_linux_architecture(): + text = BUILD_IMAGE.read_text() + assert "x86_64) JARCH=x86_64" in text + assert re.search(r"aarch64 \| arm64\) JARCH=aarch64", text) + assert 'unzip -p "$JAR" "$J" > "$OUT/libjvector-$JARCH.so"' in text + assert 'LIBS+=("$OUT/libjvector-$JARCH.so")' in text + # the copy no longer waits for uname -m to say x86_64 + assert '[ "$(uname -m)" = x86_64 ]' not in text + + +def test_the_provider_comment_states_the_real_default(): + text = ARCADE_NATIVE.read_text() + assert "unset: jvector chooses" not in text + assert "DefaultVectorizationProvider" in text + assert "ARCADEDB_VECTOR_PROVIDER=native" in text