From 2a1ebad7fd61d0e3d20fbaea79f0a9c4000aab98 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 00:19:49 +0900 Subject: [PATCH] Native: start the isolate with -XX:MaximumHeapSizePercent=50 by default GraalVM's default heap limit is 80% of the memory the isolate sees (the container limit when there is one). With the engine's off-heap memory on top, the kernel killed the process in memory-limited containers. The glue now passes -XX:MaximumHeapSizePercent=50 as an isolate argument unless ARCADEDB_NATIVE_ISOLATE_ARGS already has -Xmx, -XX:MaximumHeapSizePercent or -XX:MaxRAM (theirs wins). The isolate arguments are now always non-empty (the log directory is always set), so the dead "if iso_args:" branch is gone. Tests: the default and the override rules, and on the real image the default max_heap is about half of the memory and a user percent, -Xmx or -XX:MaxRAM replaces it. Docs: a heap-size section in the installation page. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/installation.adoc | 20 ++++- src/arcadedb/_native/lib.py | 70 +++++++++------ tests/conftest.py | 2 + tests/test_native_heap.py | 103 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 27 deletions(-) create mode 100644 tests/test_native_heap.py diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index 4f9456f..be8f519 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -1,5 +1,5 @@ = Installation -:description: Install the arcadedb (native) or arcadedb-jvm (JVM) wheel, check that it works, and set JVM options. +:description: Install the arcadedb (native) or arcadedb-jvm (JVM) wheel, check that it works, and set JVM options and the native heap size. // Sources: bindings docs getting-started/installation.md, the PyPI metadata of // arcadedb and arcadedb-jvm. @@ -149,6 +149,24 @@ Each Python process starts its own JVM, and each JVM sees the whole container li The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. The calls that hand out a Java object (`get_java_database`, `get_java_document`) raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use. +[[native-heap]] +=== Heap size + +The engine runs in a GraalVM isolate. By default the native package limits its heap to 50% of the memory the process can see. It does this with the isolate argument `-XX:MaximumHeapSizePercent=50`. The GraalVM default is 80%. The engine also uses memory outside the heap. With 80%, the kernel killed the process in containers that have a memory limit. With 50%, the same cases ran clean, and the timings of the benchmark figures did not change by more than the noise. + +The percentage follows the container memory limit (the cgroup limit) when there is one. Without a limit it follows the memory of the host. In a container with a 4 GB limit, the heap limit is 2 GB. + +To set the heap yourself, put one of these arguments in the environment variable `ARCADEDB_NATIVE_ISOLATE_ARGS` before the first database is opened. The variable takes space-separated arguments: + +[source,bash] +---- +export ARCADEDB_NATIVE_ISOLATE_ARGS="-Xmx2g" # a fixed maximum heap +export ARCADEDB_NATIVE_ISOLATE_ARGS="-XX:MaximumHeapSizePercent=70" # another percentage +export ARCADEDB_NATIVE_ISOLATE_ARGS="-XX:MaxRAM=4g" # a memory size for the percentage; GraalVM's 80% applies to it +---- + +When one of these three is present, the package adds no default. Do not use `-XX:MaxRAMPercentage`. The native image does not accept it, and the isolate then fails to start with `graal_create_isolate failed (rc=22)`. The same variable also takes the engine's `-D` settings: see xref:how-to/logs.adoc[Where the logs go]. For what a native image build supports, see xref:arcadedb::how-to/operations/native-image.adoc[Native image] in the main ArcadeDB docs. + == Next steps * xref:quickstart.adoc[Quickstart] diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index d9d2a82..85fea6e 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -82,6 +82,23 @@ def _log_isolate_args(user_args, log_dir) -> list: return args +# GraalVM's default heap limit is 80% of the memory it can see (the container limit when there is one). With the engine's off-heap +# memory on top, that let the kernel kill the process in a memory-limited container; 50% ran clean in the same cases, at the same speed. +HEAP_PERCENT = 50 +_HEAP_ARGS = ("-Xmx", "-XX:MaximumHeapSizePercent=", "-XX:MaxRAM=") + + +def _heap_isolate_args(user_args) -> list: + """The isolate argument that limits the heap to ``HEAP_PERCENT`` of the memory; none when the user already sets the heap (theirs wins). + + ``-XX:MaxRAMPercentage`` is not accepted by the image (the isolate is not created, rc=22). ``-Xmx``, ``-XX:MaximumHeapSizePercent`` + and ``-XX:MaxRAM`` are, so any of them in ``user_args`` replaces the default. + """ + if any(a.startswith(_HEAP_ARGS) for a in user_args): + return [] + return [f"-XX:MaximumHeapSizePercent={HEAP_PERCENT}"] + + def _prepare_logging(user_args) -> list: """Create the log directory, write its logging.properties and return the isolate arguments for it (see :func:`_log_isolate_args`).""" global LOG_DIR_USED @@ -141,34 +158,35 @@ def _load(): ctypes.POINTER(ctypes.c_void_p), ] lib.graal_create_isolate.restype = ctypes.c_int - params = None # run-time image options for the isolate (e.g. "-XX:InitialCollectionPolicy=Adaptive -XX:MaximumYoungGenerationSizePercent=20") - iso_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() - # the log directory: always set, so that nothing is written to the working directory (the same rule as the JVM build) - iso_args += _prepare_logging(iso_args) - if iso_args: - - class _Params(ctypes.Structure): - _fields_ = [ - ("version", ctypes.c_int), - ("reserved_address_space_size", ctypes.c_size_t), - ("auxiliary_image_path", ctypes.c_char_p), - ("auxiliary_image_reserved_space_size", ctypes.c_size_t), - ("argc", ctypes.c_int), - ("argv", ctypes.POINTER(ctypes.c_char_p)), - ("pkey", ctypes.c_int), - ("ignore_unrecognized_args", ctypes.c_char), - ("_reserved_4", ctypes.c_char), - ("_reserved_5", ctypes.c_char), - ] - - argv = (ctypes.c_char_p * (len(iso_args) + 1))( - b"arcadedb", *[a.encode() for a in iso_args] - ) - params = _Params(version=5, argc=len(argv), argv=argv) - lib.graal_create_isolate.argtypes[0] = ctypes.POINTER(_Params) + user_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() + # the heap limit and the log directory are always set (the log rule is the same as the JVM build's: nothing is written to the + # working directory); an argument the user gave for either is left out + iso_args = ( + user_args + _heap_isolate_args(user_args) + _prepare_logging(user_args) + ) + + class _Params(ctypes.Structure): + _fields_ = [ + ("version", ctypes.c_int), + ("reserved_address_space_size", ctypes.c_size_t), + ("auxiliary_image_path", ctypes.c_char_p), + ("auxiliary_image_reserved_space_size", ctypes.c_size_t), + ("argc", ctypes.c_int), + ("argv", ctypes.POINTER(ctypes.c_char_p)), + ("pkey", ctypes.c_int), + ("ignore_unrecognized_args", ctypes.c_char), + ("_reserved_4", ctypes.c_char), + ("_reserved_5", ctypes.c_char), + ] + + argv = (ctypes.c_char_p * (len(iso_args) + 1))( + b"arcadedb", *[a.encode() for a in iso_args] + ) + params = _Params(version=5, argc=len(argv), argv=argv) + lib.graal_create_isolate.argtypes[0] = ctypes.POINTER(_Params) rc = lib.graal_create_isolate( - ctypes.byref(params) if params is not None else None, + ctypes.byref(params), ctypes.byref(isolate), ctypes.byref(thread), ) diff --git a/tests/conftest.py b/tests/conftest.py index b721b07..a6b069d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,8 @@ collect_ignore.append("test_native_xml_limits.py") # where the native library writes its log (isolate arguments); the JVM build is tested in test_log_dir.py collect_ignore.append("test_native_log_dir.py") + # the heap limit that the native glue puts on the isolate (a GraalVM isolate argument; the JVM build has heap_size) + collect_ignore.append("test_native_heap.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_native_heap.py b/tests/test_native_heap.py new file mode 100644 index 0000000..b0bfcd9 --- /dev/null +++ b/tests/test_native_heap.py @@ -0,0 +1,103 @@ +"""Native build: the isolate gets a heap limit of 50% of the memory it can see, unless the user sets the heap. + +GraalVM's own default is 80% of the memory (the container limit when there is one). Together with the engine's off-heap memory that +let the kernel kill the process in a memory-limited container, so the glue passes ``-XX:MaximumHeapSizePercent=50`` as an isolate +argument. An ``-Xmx``, ``-XX:MaximumHeapSizePercent`` or ``-XX:MaxRAM`` in ARCADEDB_NATIVE_ISOLATE_ARGS replaces it. +""" + +import json +import os +import subprocess # nosec B404 - test-controlled child process +import sys +from pathlib import Path + +import pytest + +from arcadedb._native import lib as native_lib + +DEFAULT = "-XX:MaximumHeapSizePercent=50" + +CHILD = """ +import json +from arcadedb._native import lib +print(json.dumps(lib.diag())) +""" + + +def test_default_isolate_arguments_cap_the_heap_at_half(): + assert native_lib._heap_isolate_args([]) == [DEFAULT] + + +@pytest.mark.parametrize( + "user", + [ + ["-Xmx2g"], + ["-XX:MaximumHeapSizePercent=25"], + ["-XX:MaxRAM=4g"], + ["-Dfoo=bar", "-Xmx512m", "-XX:InitialCollectionPolicy=Adaptive"], + ], +) +def test_user_heap_arguments_win_over_the_default(user): + assert native_lib._heap_isolate_args(user) == [] + + +def test_other_user_arguments_keep_the_default(): + user = ["-XX:InitialCollectionPolicy=Adaptive", "-Dfoo=bar"] + assert native_lib._heap_isolate_args(user) == [DEFAULT] + + +def _diag(tmp_path, isolate_args=None) -> dict: + """``lib.diag()`` of an isolate that a fresh process starts with ``isolate_args`` in ARCADEDB_NATIVE_ISOLATE_ARGS.""" + env = {k: v for k, v in os.environ.items() if k != "ARCADEDB_NATIVE_ISOLATE_ARGS"} + env["ARCADEDB_LOG_DIR"] = str(tmp_path / "log") + if isolate_args is not None: + env["ARCADEDB_NATIVE_ISOLATE_ARGS"] = isolate_args + result = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", CHILD], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def _memory_limit() -> int: + """Bytes the isolate sees on Linux: the smaller of the host memory and a cgroup limit.""" + limits = [os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")] + for name in ( + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + ): + try: + limits.append(int(Path(name).read_text())) + except (OSError, ValueError): # absent, or "max" (no limit) + pass + return min(limits) + + +def test_default_max_heap_is_half_of_the_memory(tmp_path): + default = _diag(tmp_path)["max_heap"] + full = _diag(tmp_path, "-XX:MaximumHeapSizePercent=100")["max_heap"] + assert 0.45 <= default / full <= 0.55, (default, full) + if sys.platform == "linux": # follows the container limit, not the host RAM + assert 0.40 <= default / _memory_limit() <= 0.60, default + + +def test_user_percent_replaces_the_default_in_the_isolate(tmp_path): + default = _diag(tmp_path)["max_heap"] + quarter = _diag(tmp_path, "-XX:MaximumHeapSizePercent=25")["max_heap"] + assert 0.45 <= quarter / default <= 0.55, (quarter, default) + + +def test_user_xmx_replaces_the_default_in_the_isolate(tmp_path): + max_heap = _diag(tmp_path, "-Xmx512m")["max_heap"] + assert 0.8 * 512 * 2**20 <= max_heap <= 512 * 2**20, max_heap + + +def test_user_max_ram_replaces_the_default_in_the_isolate(tmp_path): + # with MaxRAM the percentage is taken of that size; the default 50% is not added, so GraalVM's own percentage (80%) applies + max_heap = _diag(tmp_path, "-XX:MaxRAM=4g")["max_heap"] + assert 0.7 * 4 * 2**30 <= max_heap <= 0.85 * 4 * 2**30, max_heap