diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 722b4cb..a3c2408 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,8 @@ version: 2 # the cooldown and still open the moment an advisory lands. updates: # pyproject.toml is the package definition (`[tool.uv] managed = false`, no - # uv.lock), so it is tracked as a pip project rather than a uv one. + # uv.lock), so it is tracked as a pip project rather than a uv one: Dependabot's `uv` ecosystem reads a uv.lock + # that this repo does not have. `pip` here is only the Dependabot ecosystem name; nothing in CI runs pip. # # The lower bounds in pyproject.toml are the oldest versions this package # supports, not the newest it was tested with. The "Dependency floor audit" diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml index 4b90426..958b463 100644 --- a/.github/workflows/test-jvm.yml +++ b/.github/workflows/test-jvm.yml @@ -105,14 +105,24 @@ jobs: with: python-version: "3.12" - - name: Install Bandit - run: python -m pip install "bandit==1.9.4" + # Bandit and pip-audit run through uvx (a throwaway environment each), so the jobs install nothing into the runner's Python. + - name: Setup UV package manager + shell: bash + run: | + # Retry transient network failures (a connection reset failed a whole examples job on #8897); the + # script is downloaded first so a retried transfer is never piped half-written into sh. + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + uv --version - name: Run Bandit on src and tests (must be clean) - run: python -m bandit -c pyproject.toml -r src tests --severity-level low --confidence-level low + run: uvx bandit==1.9.4 -c pyproject.toml -r src tests --severity-level low --confidence-level low - name: Run Bandit on examples (must be clean at medium+/high-confidence) - run: python -m bandit -c pyproject.toml -r examples --severity-level medium --confidence-level high + run: uvx bandit==1.9.4 -c pyproject.toml -r examples --severity-level medium --confidence-level high # Meterian cannot scan this folder: `uv lock` refuses on a project carrying # `[tool.uv] managed = false`, so it produces no lockfile to read (see #5616). @@ -129,9 +139,9 @@ jobs: with: python-version: "3.12" - # Pinned rather than tracking latest, unlike the other uv installs in this - # repo: this job's own output depends on how uv resolves, so a silent - # resolver change would silently change what gets audited. + # The uv version is pinned in every install URL of this repo. It matters most here: + # this job's own output depends on how uv resolves, so a silent resolver change + # would silently change what gets audited. - name: Setup UV package manager shell: bash run: | @@ -144,9 +154,6 @@ jobs: export PATH="$HOME/.local/bin:$PATH" uv --version - - name: Install pip-audit - run: python -m pip install "pip-audit==2.10.1" - # --resolution lowest-direct pins every declared dependency to the oldest # release its own constraint admits, which is the version this manifest # actually promises to work with. The default resolution would pick the @@ -166,30 +173,50 @@ jobs: # The list is read from the classifiers rather than repeated here, so adding # an interpreter to the package cannot leave it unaudited. # - # --disable-pip reads the pinned file directly rather than asking pip to - # resolve it. Without it, a floor whose version has no wheel for the - # running interpreter fails as an sdist build error instead of reporting - # the advisory that caused the regression. + # `uv pip compile` is not used: the floors are resolved with `uv lock` on a throwaway project, one per + # interpreter, that carries the package's dependencies and the audited extras and nothing else (pyproject.toml + # itself stays a non-uv project, and `uv lock` would refuse it with `managed = false` and would also resolve + # the dev extra). `requires-python = "==3.X.*"` makes the lock resolve for that interpreter only, which is what + # `--python-version` did. `uv export` writes the pinned file; --disable-pip makes pip-audit read it directly + # rather than asking pip to resolve it. Without it, a floor whose version has no wheel for the running + # interpreter fails as an sdist build error instead of reporting the advisory that caused the regression. - name: Audit declared dependency floors on every supported interpreter run: | # The trailing dot in the prefix is load-bearing: the classifiers also # carry a bare "Programming Language :: Python :: 3", which without it - # yields --python-version 3 and fails. - versions=$(python -c 'import tomllib;p="Programming Language :: Python :: 3.";print(" ".join(c.rsplit(" ",1)[1] for c in tomllib.load(open("pyproject.toml","rb"))["project"]["classifiers"] if c.startswith(p)))') + # yields version 3 and fails. + versions=$(uv run --no-project python -c 'import tomllib;p="Programming Language :: Python :: 3.";print(" ".join(c.rsplit(" ",1)[1] for c in tomllib.load(open("pyproject.toml","rb"))["project"]["classifiers"] if c.startswith(p)))') # No classifiers would audit nothing and still pass, which is the kind # of silent gap this job exists to close. [ -n "$versions" ] || { echo "no Python version classifiers found in pyproject.toml"; exit 1; } echo "auditing declared floors for: $versions" for v in $versions; do echo "::group::Resolve floors for Python $v" - uv pip compile pyproject.toml \ - --extra test --extra vector --extra examples --extra arrow --extra pandas \ - --resolution lowest-direct \ - --python-version "$v" \ - -o "$RUNNER_TEMP/floors-$v.txt" + proj="$RUNNER_TEMP/floors-$v" + mkdir -p "$proj" + uv run --no-project python - "$v" "$proj/pyproject.toml" << 'PY' + import sys + import tomllib + + version, out = sys.argv[1], sys.argv[2] + extras = ["test", "vector", "examples", "arrow", "pandas"] + with open("pyproject.toml", "rb") as f: + project = tomllib.load(f)["project"] + deps = list(project["dependencies"]) + for extra in extras: + deps += project["optional-dependencies"][extra] + with open(out, "w") as f: + f.write('[project]\nname = "floors"\nversion = "0"\n') + f.write('requires-python = "==%s.*"\n' % version) + f.write("dependencies = [\n") + for d in deps: + f.write(" %r,\n" % d) + f.write("]\n") + PY + (cd "$proj" && uv lock --resolution lowest-direct && uv export --frozen --no-hashes --no-annotate --no-emit-project -o "$RUNNER_TEMP/floors-$v.txt") cat "$RUNNER_TEMP/floors-$v.txt" echo "::endgroup::" - pip-audit --no-deps --disable-pip -r "$RUNNER_TEMP/floors-$v.txt" + uvx pip-audit==2.10.1 --no-deps --disable-pip -r "$RUNNER_TEMP/floors-$v.txt" done # First job: Download ArcadeDB JARs (platform-agnostic) @@ -304,7 +331,7 @@ jobs: # Retry transient network failures (a connection reset failed a whole examples job on #8897); the # script is downloaded first so a retried transfer is never piped half-written into sh. installer="$(mktemp)" - curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" export PATH="$HOME/.local/bin:$PATH" @@ -328,11 +355,6 @@ jobs: if: matrix.platform == 'linux/amd64' || matrix.platform == 'linux/arm64' uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - name: Install Python build dependencies - shell: bash - run: | - uv pip install --system build wheel setuptools - - name: Set UTF-8 encoding (Windows only) if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' shell: bash @@ -417,7 +439,7 @@ jobs: # Retry transient network failures (a connection reset failed a whole examples job on #8897); the # script is downloaded first so a retried transfer is never piped half-written into sh. installer="$(mktemp)" - curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" export PATH="$HOME/.local/bin:$PATH" @@ -439,7 +461,7 @@ jobs: # The tests below run on the runner (glibc 2.39). This step runs the wheel where its tag says it must # work: a manylinux_2_28 container (glibc 2.28, AlmaLinux 8, the oldest system the Linux wheels are tagged for). - # pip refuses a wheel tagged above 2.28 there, so a JRE that raises the floor again fails here as well as in + # uv (like pip) refuses a wheel tagged above 2.28 there, so a JRE that raises the floor again fails here as well as in # scripts/verify_wheel_platform_tag.py. The wheel's own JRE does the work: no Java in the container. - name: Install and run the wheel in manylinux_2_28 (Linux only) if: startsWith(matrix.platform, 'linux/') @@ -455,15 +477,17 @@ jobs: esac PYTAG="cp${PYVER//./}-cp${PYVER//./}" ls -l test-install/*.whl - docker run --rm -i -e PYTAG="$PYTAG" -v "$PWD/test-install:/wheels:ro" \ + # The runner's uv is mounted into the container (a glibc 2.17 build); the image's own Python is used, never downloaded. + docker run --rm -i -e PYTAG="$PYTAG" -e UV_PYTHON_DOWNLOADS=never -v "$PWD/test-install:/wheels:ro" \ + -v "$(command -v uv):/usr/local/bin/uv:ro" \ "quay.io/pypa/manylinux_2_28_${ARCH}:latest" bash -es <<'CONTAINER' ldd --version | head -n 1 test -x "/opt/python/$PYTAG/bin/python" || { echo "no $PYTAG in this image" >&2; ls /opt/python >&2; exit 1; } - "/opt/python/$PYTAG/bin/python" -m venv /tmp/smoke - # --only-binary: a source build of JPype must not hide a missing wheel - /tmp/smoke/bin/python -m pip install --quiet --only-binary=:all: /wheels/*.whl + with=() + for w in /wheels/*.whl; do with+=(--with "$w"); done cd /tmp - /tmp/smoke/bin/python - <<'PYSMOKE' + # --no-build: a source build of JPype must not hide a missing wheel + uv run --no-project --no-build --quiet --python "/opt/python/$PYTAG/bin/python" "${with[@]}" python - <<'PYSMOKE' import tempfile import arcadedb with tempfile.TemporaryDirectory() as tmp: @@ -477,18 +501,6 @@ jobs: # Note: Java is NOT required for arcadedb (JRE is bundled) # This package works without any external Java installation! - - name: Install wheel and test dependencies - shell: bash - run: | - # This list is hand-maintained and mirrors the `test` extra. It had - # already drifted: pyarrow and pandas were missing, so to_arrow() and - # to_dataframe() tests skipped themselves in this job while it - # reported green. setuptools is there for tests/test_wheel_packages.py and - # tests/test_wheel_tags.py, which run setup.py on a copy of the sources (this job - # no longer installs build tools: it builds nothing). The guard step below now fails the build if any - # test skips for a missing import, so the next drift is loud. - uv pip install --system test-install/*.whl "setuptools>=84.1.0" pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12" "duckdb>=1.4.2" "ladybug>=0.21.2" "networkx>=3.4" - - name: Run pytest on host id: pytest # A HUNG TEST ENDS THE STEP, NOT THE DAY. On 2026-09-25 the windows/3.13 @@ -497,9 +509,20 @@ jobs: # GitHub's 6-hour default. The step takes 108-319 s on every platform. timeout-minutes: 30 shell: bash + env: + PYVER: ${{ matrix.python-version }} run: | echo "๐Ÿงช Testing arcadedb (with bundled JRE)..." + # The wheel and its `test` extra go into a throwaway `uv run` environment, so nothing is installed into the + # runner's Python. The extra mirrors what this job once listed by hand (it had drifted: pyarrow and pandas were + # missing, so to_arrow() and to_dataframe() tests skipped themselves while the job reported green; setuptools is + # in it for tests/test_wheel_packages.py and tests/test_wheel_tags.py, which run setup.py on a copy of the + # sources). The guard below fails the build if any test skips for a missing import, so a drift is loud. + wheels=(test-install/*.whl) + [ "${#wheels[@]}" -eq 1 ] || { echo "expected one wheel in test-install, found ${#wheels[@]}" >&2; exit 1; } + UVRUN=(uv run --no-project --python "$PYVER" --with "${wheels[0]}[test]") + # Run pytest with JUnit XML output for structured results # pytest runs with its default fd capture on every platform. Windows used to run with # --capture=sys (humemai/arcadedb-embedded-python#10): the default capture swaps fds 1 @@ -509,7 +532,7 @@ jobs: # to the console on Windows by default (ARCADEDB_LOG_CONSOLE, src/arcadedb/_logdir.py), # which removes the cause; tests/test_log_redirect.py redirects fd 1 and 2 on purpose. set +e # Don't exit on test failures - pytest tests/ -v --tb=short --color=yes --junitxml=test-results.xml + "${UVRUN[@]}" python -m pytest tests/ -v --tb=short --color=yes --junitxml=test-results.xml PYTEST_EXIT_CODE=$? set -e diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 1838802..e44865f 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -394,12 +394,12 @@ jobs: MANYLINUX_IMAGE: ${{ matrix.container }} run: | set -euo pipefail - docker run --rm -v "$T/wheel/dist:/wheel:ro" -v "$GITHUB_WORKSPACE/scripts/smoke_wheel.py:/smoke_wheel.py:ro" "$MANYLINUX_IMAGE" bash -euo pipefail -c ' + # The runner's uv is mounted into the container (a glibc 2.17 build); the image's own Python is used, never downloaded. + # --no-index: nothing but the wheel is installed. + docker run --rm -e UV_PYTHON_DOWNLOADS=never -v "$(command -v uv):/usr/local/bin/uv:ro" -v "$T/wheel/dist:/wheel:ro" -v "$GITHUB_WORKSPACE/scripts/smoke_wheel.py:/smoke_wheel.py:ro" "$MANYLINUX_IMAGE" bash -euo pipefail -c ' ldd --version | sed -n 1p - /opt/python/cp312-cp312/bin/python -m venv /tmp/venv - /tmp/venv/bin/python -m pip install --no-index /wheel/*.whl cd /tmp - /tmp/venv/bin/python /smoke_wheel.py + uv run --no-project --no-index --python /opt/python/cp312-cp312/bin/python --with /wheel/*.whl python /smoke_wheel.py ' - name: "Gate: the macOS library links system libraries only" diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index 27da294..b0df568 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -233,17 +233,12 @@ jobs: # Retry transient network failures (a connection reset failed a whole examples job on #8897); the # script is downloaded first so a retried transfer is never piped half-written into sh. installer="$(mktemp)" - curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" export PATH="$HOME/.local/bin:$PATH" uv --version - - name: Install Python build dependencies - shell: bash - run: | - uv pip install --system build wheel setuptools - - name: Set UTF-8 encoding (Windows only) if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' shell: bash @@ -334,7 +329,7 @@ jobs: # Retry transient network failures (a connection reset failed a whole examples job on #8897); the # script is downloaded first so a retried transfer is never piped half-written into sh. installer="$(mktemp)" - curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" export PATH="$HOME/.local/bin:$PATH" @@ -349,29 +344,54 @@ jobs: # Note: Java is NOT required - arcadedb has bundled JRE! - - name: Install ArcadeDB Python bindings - shell: bash - run: | - uv pip install --system test-install/arcadedb_jvm-*.whl - - - name: Install example dependencies + # The wheel and the example dependencies live in a uv project of their own (examples-env/, not the repo's pyproject.toml, + # which is a package definition and not a uv project), synced with `uv sync`. Nothing is installed into the runner's Python. + # torch comes from the CPU-only PyTorch index (explicit: only torch is looked up there), which saves the CUDA download. + # PyTorch is required by sentence-transformers, even on macOS. + # numpy is not capped: the old macOS `numpy<2.0` was for Intel Macs, where PyTorch stops at 2.2 (built against + # NumPy 1.x). That leg (darwin/amd64) is disabled; re-enabling it needs the cap back for that platform only. + # On Apple Silicon the cap breaks pyarrow, which requires NumPy 2 (example 27). + - name: Install ArcadeDB Python bindings and example dependencies shell: bash + env: + PYVER: ${{ matrix.python-version }} run: | - # Install dependencies needed by examples - # PyTorch is required by sentence-transformers, even on macOS - # For Python 3.13+, wheels might only be on PyTorch official index - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS: Install CPU-only PyTorch (supports MPS acceleration) - uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu - # numpy is not capped: the old macOS `numpy<2.0` was for Intel Macs, where PyTorch stops at 2.2 (built against - # NumPy 1.x). That leg (darwin/amd64) is disabled; re-enabling it needs the cap back for that platform only. - # On Apple Silicon the cap breaks pyarrow, which requires NumPy 2 (example 27). - uv pip install --system numpy requests sentence-transformers pandas polars pyarrow - else - # Linux & Windows: Install CPU-only PyTorch to save space (avoid CUDA) - uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu - uv pip install --system numpy requests sentence-transformers pandas polars pyarrow - fi + wheels=(test-install/arcadedb_jvm-*.whl) + [ "${#wheels[@]}" -eq 1 ] || { echo "expected one wheel in test-install, found ${#wheels[@]}" >&2; exit 1; } + mkdir -p examples-env + cat > examples-env/pyproject.toml <> "$GITHUB_ENV" + "$ex_py" -c "import arcadedb; print('arcadedb', arcadedb.__version__)" # Restored before the dataset step, which is what triggers the model # download. Keyed on the model name so bumping the model invalidates it. @@ -394,9 +414,8 @@ jobs: shell: bash continue-on-error: true run: | - uv pip install --system sentence-transformers >/dev/null 2>&1 || true for attempt in 1 2 3; do - if python3 -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2'); print('model ready')"; then + if "$EX_PY" -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2'); print('model ready')"; then echo "โœ… model cached on attempt $attempt" exit 0 fi @@ -408,7 +427,6 @@ jobs: - name: Download datasets shell: bash run: | - uv pip install --system tqdm py7zr lxml cd examples # Determine which examples are selected @@ -438,7 +456,7 @@ jobs: # every failure into "exited 0" and would have made this skip # everything, including real ones. local rc=0 - if python3 download_data.py "$@"; then + if "$EX_PY" download_data.py "$@"; then return 0 else rc=$? @@ -693,13 +711,13 @@ jobs: # Run the example with appropriate parameters if [ -n "$example_jvm_args" ]; then - if ARCADEDB_JVM_ARGS="$example_jvm_args" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration python "$example" $example_args > "$log_file" 2>&1; then + if ARCADEDB_JVM_ARGS="$example_jvm_args" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then exit_code=0 else exit_code=$? fi else - if ARCADEDB_JVM_ARGS="" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration python "$example" $example_args > "$log_file" 2>&1; then + if ARCADEDB_JVM_ARGS="" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then exit_code=0 else exit_code=$? @@ -713,11 +731,11 @@ jobs: ref_args="${example_args/--db arcadedb_cypher/--db python_memory}" ref_args="${ref_args/--run-label ci10/--run-label ci10ref}" ref_log="${example%.py}-reference.log" - if ! $TIMEOUT_CMD $timeout_duration python "$example" $ref_args > "$ref_log" 2>&1; then + if ! $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $ref_args > "$ref_log" 2>&1; then echo "โŒ reference run (python_memory) of $example failed; last 20 lines:" tail -n 20 "$ref_log" exit_code=1 - elif ! python scripts/compare_query_hashes.py "$log_file" "$ref_log"; then + elif ! "$EX_PY" scripts/compare_query_hashes.py "$log_file" "$ref_log"; then echo "โŒ $example: answers differ from the python_memory reference (above)" exit_code=1 fi diff --git a/CLAUDE.md b/CLAUDE.md index b45f712..f1cfade 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,16 +35,16 @@ ARCADEDB_TAG=26.11.1-SNAPSHOT ./scripts/build.sh # pin the arcadedata/arcaded Tests run against the **built wheel**, not the source tree. `pyproject.toml` has `[tool.uv] managed = false`, so it is not a uv project. ```bash -uv venv # once -uv pip install --reinstall-package arcadedb "$(ls dist/*.whl | head -1)[test]" # after each build -uv run pytest # all -uv run pytest tests/test_core.py -v # one file -uv run pytest tests/test_core.py::TestX::test_y # one test -uv run pytest -k transaction -uv run pytest -m server / -m "not server" # markers: server, server_wire, integration +W="$(ls dist/*.whl | head -1)" +uv run --no-project --with "$W[test]" python -m pytest # all +uv run --no-project --with "$W[test]" python -m pytest tests/test_core.py -v # one file +uv run --no-project --with "$W[test]" python -m pytest tests/test_core.py::TestX::test_y # one test +uv run --no-project --with "$W[test]" python -m pytest -k transaction +uv run --no-project --with "$W[test]" python -m pytest -m server # markers: server, server_wire, integration ``` -- `--reinstall-package` is needed because rebuilds keep the same version. +- `--with` builds a throwaway environment from the wheel file, so there is no venv to keep and nothing to reinstall after a rebuild. If uv's cache still serves an older wheel of the same version (rebuilds keep the version), add `--reinstall-package arcadedb-jvm`. +- Use `python -m pytest`, not the `pytest` entry point: with `--with`, the entry point does not see the packages of the environment. - **No skips policy**: CI fails on any skipped test not listed in `scripts/check_test_skips.py` (the list is empty). Don't add `pytest.skip`/`importorskip` for missing features or deps; tests that can't run on a platform go in `collect_ignore` in `tests/conftest.py`; known engine bugs use strict `xfail`. - Never bind or address a hand-picked port (e.g. 2480) in a test: use `server.get_http_port()` or the `_free_ports()` pattern in `test_server_wire_protocols.py`. Parallel runs share the machine and collisions surface as unrelated failures. - Never assert on raw wall-clock elapsed time; a JVM pause makes it flaky. Timeouts are hang detectors, not latency bounds. @@ -55,8 +55,8 @@ uv run pytest -m server / -m "not server" # markers: server, server_wire, Pre-commit: black, isort (`--profile black`), shfmt (`-i 4 -ci -sr`), pretty-format-yaml (not on `.github/`), actionlint. CI also runs Bandit, which must be clean: ```bash -python -m bandit -c pyproject.toml -r src tests --severity-level low --confidence-level low -python -m bandit -c pyproject.toml -r examples --severity-level medium --confidence-level high +uvx bandit==1.9.4 -c pyproject.toml -r src tests --severity-level low --confidence-level low +uvx bandit==1.9.4 -c pyproject.toml -r examples --severity-level medium --confidence-level high ``` ## Architecture @@ -71,7 +71,7 @@ python -m bandit -c pyproject.toml -r examples --severity-level medium --confide Each package has one test and one release workflow: `test-jvm.yml` and `release-jvm.yml` for `arcadedb-jvm`, `test-native.yml` and `release-native.yml` for `arcadedb` (the release workflows call the test workflows with `workflow_call`; the PyPI trusted publishers are keyed on the release file names and the environments `pypi-jvm` and `pypi`, so never rename them without changing PyPI first). `test-python-examples.yml` runs `examples/`. `docs.yml` builds the Antora component in `docs/` as a site of its own (`docs/antora-playbook.yml`, Antora 3.2.1, a warning fails it) and attaches it as the `docs-site` artifact; it is the only check of a docs-only pull request. Link into the main manual with `{arcadedb-docs-url}/`, never an `arcadedb::` xref (the component on its own cannot resolve it). -`.github/workflows/test-jvm.yml` builds ONE wheel per platform (linux/amd64, linux/arm64, darwin/arm64, windows/amd64), tagged `py3-none-` by `setup.py` (`bdist_wheel.get_tag`; `scripts/verify_wheel_tag.py` checks it), and tests that wheel on Python 3.11โ€“3.14; also runs nightly against `arcadedata/arcadedb:latest`. `requires-python` is `>=3.11,<3.15` because the tag no longer keeps pip off 3.15 and JPype has no 3.15 wheel; drop the cap with it. The org shares a runner cap with upstream, so the unit tests run 10 legs on every event (linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14, the shape of test-native.yml), and the examples run all four platforms on 3.14 on a pull request (4 legs, 28 checks in all) and 16 legs on push to main, the nightly run, dispatch and the release (40 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. +`.github/workflows/test-jvm.yml` builds ONE wheel per platform (linux/amd64, linux/arm64, darwin/arm64, windows/amd64), tagged `py3-none-` by `setup.py` (`bdist_wheel.get_tag`; `scripts/verify_wheel_tag.py` checks it), and tests that wheel on Python 3.11โ€“3.14; also runs nightly against `arcadedata/arcadedb:latest`. `requires-python` is `>=3.11,<3.15` because the tag no longer keeps pip off 3.15 and JPype has no 3.15 wheel; drop the cap with it. The org shares a runner cap with upstream, so the unit tests run 10 legs on every event (linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14, the shape of test-native.yml), and the examples run all four platforms on 3.14 on a pull request (4 legs, 28 checks in all) and 16 legs on push to main, the nightly run, dispatch and the release (40 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit locks the dependencies and extras on a throwaway uv project with `--resolution lowest-direct` for each Python classifier and runs `uvx pip-audit` on the export, so raise lower bounds when a floor has an advisory. No tool is installed with pip anywhere in CI or the scripts: one-off tools run through `uvx tool==X`, the wheel is built with `uvx --from "build[uv]==1.6.1" pyproject-build --installer uv`, and the uv install URL is pinned (`0.12.0`). `.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/after_tag_verify.sh` checks the finished release. The rule is stated once, on the docs page `development/release.adoc`. diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index ba5c001..1002fab 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -27,9 +27,9 @@ The package supports four platforms and four Python versions (3.11 to 3.14). Tha The wheel of `arcadedb-jvm` is tagged `py3-none-` (`setup.py`), like the native wheel: it carries a JRE for its platform, but nothing in it is compiled against CPython. So a Python version is a test, not a build. `test-jvm.yml` has two jobs: * `Build arcadedb-jvm wheel ()` builds the wheel once for each platform and uploads it as `wheel-`. `scripts/verify_wheel_tag.py` fails the build when the wheel is not `py3-none-`. -* `Test arcadedb (, Python )` downloads that wheel and installs it into one Python. It builds nothing. The release publishes these same four files. +* `Test arcadedb (, Python )` downloads that wheel and runs the tests on one Python in a throwaway `uv run --no-project --with "[test]" python -m pytest` environment. It builds nothing. The release publishes these same four files. -The `requires-python` of the package is `>=3.11,<3.15`. The wheel tag no longer keeps pip off Python 3.15, so the cap does it. JPype has no 3.15 wheel yet; remove the cap, and add 3.15 to the matrix, when it has one. +The `requires-python` of the package is `>=3.11,<3.15`. The wheel tag no longer keeps an installer off Python 3.15, so the cap does it. JPype has no 3.15 wheel yet; remove the cap, and add 3.15 to the matrix, when it has one. == What runs when diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index d09b7dd..6ec7f6f 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -60,7 +60,7 @@ The second argument is the previous release. Use `-` for the first release of th * The workflow run, and the conclusion of the publish job and of its upload step. * The files on PyPI: one `py3-none` wheel for each platform (the same file for every Python version), each under 100 MB, none yanked. * The sha256 of every file on PyPI against the wheels the workflow checked. -* A `pip download` for each of the four platforms, and a fresh install on Python 3.12 and 3.14 that opens a database and runs one SQL and one openCypher query. +* A download of the wheel of each of the four platforms from PyPI (picked from the PyPI JSON, fetched with `curl`, sha256 compared with the one PyPI lists), and a fresh `uv run --with` install on Python 3.12 and 3.14 that opens a database and runs one SQL and one openCypher query. It prints `PASS` or `FAIL` for each step, and its exit code is the number of failures. diff --git a/pyproject.toml b/pyproject.toml index 50d5556..69caffe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,8 @@ description = "ArcadeDB embedded multi-model database, JVM build with a bundled readme = "README.md" # The upper bound is JPype: jpype1 1.7.1 has wheels for CPython 3.11 to 3.14 and none for 3.15 (JPype # merged 3.15 support in jpype-project/jpype#1501, not released yet). The wheels of this package are -# py3-none-, so the wheel tag no longer keeps pip off an interpreter: without this cap pip -# would install the wheel on 3.15 and then try to build JPype from source. +# py3-none-, so the wheel tag no longer keeps an installer off an interpreter: without this cap an installer +# would put the wheel on 3.15 and then try to build JPype from source. # REMOVE THE CAP when jpype1 ships cp315 wheels: then add the 3.15 classifier below, 3.15 to PYTHONS in # scripts/select_matrix.py and the cap in tests/test_select_matrix.py. requires-python = ">=3.11,<3.15" @@ -48,9 +48,9 @@ dependencies = [ [tool.uv] # This pyproject is the *package definition* consumed by the wheel build # (scripts/build.sh). It is not a uv project: tests run against the built wheel, -# installed with its `test` extra into a virtual environment (see README, -# Testing). `managed = false` makes uv skip it during project discovery, so -# `uv run` uses that environment instead of trying to sync this package. +# with its `test` extra, in a throwaway `uv run --no-project --with` environment +# (see README, Testing). `managed = false` makes uv skip it during project +# discovery, so `uv run` never tries to sync this package. managed = false [project.optional-dependencies] @@ -123,7 +123,7 @@ pandas = [ ] # Optional: ResultSet.to_polars() and insert_dataframe() with a Polars frame. Both go # through Arrow, so pyarrow comes with it. insert_dataframe() with a pandas frame needs -# `pip install 'arcadedb-jvm[arrow,pandas]'` (the pandas extra alone has no pyarrow, +# `uv add 'arcadedb-jvm[arrow,pandas]'` (the pandas extra alone has no pyarrow, # because to_dataframe() does not need it). polars = [ "polars>=1.0", diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index 3d21d20..c08e5e3 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -191,9 +191,10 @@ RUN apt-get update && apt-get install -y \ curl \ && rm -rf /var/lib/apt/lists/* -# Install uv for faster, deterministic installs +# Install uv (pinned, the same version as the CI workflows) +ARG UV_VERSION=0.12.0 ENV PATH="/root/.cargo/bin:/root/.local/bin:${PATH}" -RUN curl -LsSf https://astral.sh/uv/install.sh | sh && uv --version +RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh && uv --version WORKDIR /build @@ -215,9 +216,10 @@ COPY pyproject.toml ./ COPY README.md ./ -# Install Python build dependencies -# No jpype1: setup.py never imports it, and a Python without a JPype wheel (3.15, until JPype > 1.7.1) would compile it -RUN uv pip install --system build wheel setuptools +# The wheel is built with `build` run through uvx, in an isolated environment that uv creates from [build-system] +# (setuptools, wheel). No jpype1: setup.py never imports it, and a Python without a JPype wheel (3.15, until JPype > 1.7.1) +# would compile it. Nothing is installed into the image's Python, and uv never downloads another one. +ENV UV_PYTHON_DOWNLOADS=never # Re-declare build args for this stage (required after FROM) ARG PACKAGE_NAME @@ -268,12 +270,12 @@ RUN if [ -n "${BUILD_VERSION}" ]; then \ fi; \ if [ -n "${WHEEL_PLAT}" ]; then \ echo "๐Ÿท๏ธ Building wheel with platform tag: ${WHEEL_PLAT}"; \ - python3 -m build --wheel --config-setting=--build-option=--plat-name=${WHEEL_PLAT}; \ + uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel --config-setting=--build-option=--plat-name=${WHEEL_PLAT}; \ else \ - python3 -m build --wheel; \ + uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel; \ fi; \ else \ - python3 -m build --wheel; \ + uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel; \ fi && \ if echo "${TARGET_PLATFORM}" | grep -q '^linux-'; then \ if [ -n "${WHEEL_PLAT}" ] && ! ls /build/dist/*${WHEEL_PLAT}*.whl 1> /dev/null 2>&1; then \ @@ -304,17 +306,19 @@ RUN apt-get update && apt-get install -y \ curl \ && rm -rf /var/lib/apt/lists/* -# Install uv for faster, deterministic installs +# Install uv (pinned, the same version as the CI workflows) +ARG UV_VERSION=0.12.0 ENV PATH="/root/.cargo/bin:/root/.local/bin:${PATH}" -RUN curl -LsSf https://astral.sh/uv/install.sh | sh && uv --version +RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh && uv --version WORKDIR /test # Copy the wheel from builder COPY --from=python-builder /build/dist/*.whl /tmp/ -# Install the wheel and test dependencies -RUN uv pip install --system /tmp/*.whl pytest pytest-cov +# The wheel and the test dependencies go into a throwaway `uv run` environment (nothing is installed into the image's +# Python). The RUN below and the CMD use the same --with set, so the environment is in uv's cache when the container starts. +ENV UV_PYTHON_DOWNLOADS=never # Copy tests COPY --from=python-builder /build/tests ./tests/ @@ -352,6 +356,6 @@ finally:\n\ ' > /test/test_install.py && chmod +x /test/test_install.py # Run the installation test -RUN python3 /test/test_install.py +RUN uv run --no-project --with "$(ls /tmp/*.whl)" --with pytest --with pytest-cov python /test/test_install.py -CMD ["python3", "/test/test_install.py"] +CMD ["sh", "-c", "exec uv run --no-project --with \"$(ls /tmp/*.whl)\" --with pytest --with pytest-cov python /test/test_install.py"] diff --git a/scripts/after_tag_verify.sh b/scripts/after_tag_verify.sh index 730333f..c211a43 100755 --- a/scripts/after_tag_verify.sh +++ b/scripts/after_tag_verify.sh @@ -10,7 +10,7 @@ # the four platforms, one wheel (py3-none-) for each platform. # # Run it under bash (not zsh). Every step prints PASS or FAIL; the exit code is the number of FAILs. -# Set PIN=4-5 to pin the smoke runs to those cores. It downloads about 0.6 GB (the 4 checked wheels of the run and four from PyPI): TMPDIR or WORK should point at a disk, not a tmpfs. Needs: gh, curl, python3, uv. +# Set PIN=4-5 to pin the smoke runs to those cores. It downloads about 0.6 GB (the 4 checked wheels of the run and four from PyPI): TMPDIR or WORK should point at a disk, not a tmpfs. Needs: gh, curl, python3, sha256sum, uv. set -uo pipefail V="${1:?version}" PREV="${2:?previous version}" @@ -98,9 +98,24 @@ sys.exit(1 if bad else 0) EOF fi -# (d) every platform resolves, and a fresh environment installs and works (3.12 and 3.14) -for plat in manylinux_2_34_x86_64 manylinux_2_34_aarch64 macosx_11_0_arm64 win_amd64; do - if uvx pip download --no-deps --only-binary=:all: --platform "$plat" --python-version 3.12 "$DIST==$V" -d "$W/dl-$plat" > "$W/dl-$plat.log" 2>&1; then pass "pip download $plat (Python 3.12)"; else fail "pip download $plat: $(tail -1 "$W/dl-$plat.log")"; fi +# (d) every platform has a wheel on PyPI that downloads and matches the sha256 PyPI lists for it, and a fresh environment installs and works (3.12 and 3.14) +# uv has no `download` command, and `uv pip` is not used here: the wheel of each platform is picked from the PyPI JSON read in (b) and fetched with curl. +# The install of the release on the host platform is exercised by the uv run smoke below. +for plat in 'manylinux.*x86_64' 'manylinux.*aarch64' 'macosx.*arm64' 'win_amd64'; do + pick=$( + python3 - "$W/pypi-new.json" "$plat" << 'EOF' +import json,re,sys +hits=[u for u in json.load(open(sys.argv[1]))["urls"] if re.search(r"-py3-none-",u["filename"]) and re.search(sys.argv[2],u["filename"])] +if len(hits)==1: print(hits[0]["url"], hits[0]["digests"]["sha256"], hits[0]["filename"]) +EOF + ) + read -r url sha name <<< "$pick" + if [ -z "${url:-}" ]; then + fail "no single wheel for $plat on PyPI" + continue + fi + mkdir -p "$W/dl" + if curl -fsSL --retry 3 -o "$W/dl/$name" "$url" > "$W/dl-$name.log" 2>&1 && [ "$(sha256sum "$W/dl/$name" | cut -d' ' -f1)" = "$sha" ]; then pass "download $name ($plat), sha256 matches PyPI"; else fail "download $name ($plat): $(tail -1 "$W/dl-$name.log")"; fi done for py in 3.12 3.14; do [ -n "${SKIP_SMOKE:-}" ] && { diff --git a/scripts/build-native.sh b/scripts/build-native.sh index 9d9b775..ee6f72d 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -140,32 +140,17 @@ if ! command -v jlink &> /dev/null; then exit 1 fi -# Check for build module (required for wheel building) -PYTHON_WITH_BUILD="" - -# Look for Python with working build module -# Test that -m build actually works, not just that import build succeeds -for py_cmd in python3.13 python3.12 python3.11 python3 python; do - if command -v "$py_cmd" &> /dev/null; then - PY_PATH="$(command -v "$py_cmd")" - # Test both import AND that -m build works - if "$PY_PATH" -c "import build" 2> /dev/null && "$PY_PATH" -m build --version &> /dev/null 2>&1; then - PYTHON_WITH_BUILD="$PY_PATH" - echo -e "${CYAN}โœ… Found working build module in: ${YELLOW}${PYTHON_WITH_BUILD}${NC}" - break - fi - fi -done - -if [[ -z "$PYTHON_WITH_BUILD" ]]; then - echo -e "${RED}โŒ build module not found or broken in any Python installation${NC}" - echo -e "${YELLOW}๐Ÿ’ก Please ensure build module is properly installed:${NC}" - echo -e "${YELLOW} 1. If you have a venv, activate it and run: uv pip install build${NC}" - echo -e "${YELLOW} 2. Otherwise, create a venv:${NC}" - echo -e "${YELLOW} python3 -m venv .build-env${NC}" - echo -e "${YELLOW} source .build-env/bin/activate${NC}" - echo -e "${YELLOW} uv pip install build${NC}" - echo -e "${YELLOW} ./scripts/build.sh${NC}" +# Wheel building runs `build` through uvx (an isolated environment, nothing installed into any Python), so uv is the only requirement +if ! command -v uv &> /dev/null; then + echo -e "${RED}โŒ uv not found${NC}" + echo -e "${YELLOW}๐Ÿ’ก Install uv: https://docs.astral.sh/uv/getting-started/installation/${NC}" + exit 1 +fi +BUILD_CMD=(uvx --from "build[uv]==1.6.1" pyproject-build --installer uv) +# Any Python 3 on PATH runs the small helper scripts (stdlib only) +PYTHON_BIN="$(command -v python3 || command -v python || true)" +if [[ -z "$PYTHON_BIN" ]]; then + echo -e "${RED}โŒ no python found${NC}" exit 1 fi @@ -294,7 +279,7 @@ fi echo -e "${CYAN}๐Ÿ“ฆ Preparing package...${NC}" # Build and copy JRE -copy_tree_with_symlinks "$PY_BINDINGS_DIR/temp_jre" "$PY_BINDINGS_DIR/src/arcadedb/jre" "$PYTHON_WITH_BUILD" +copy_tree_with_symlinks "$PY_BINDINGS_DIR/temp_jre" "$PY_BINDINGS_DIR/src/arcadedb/jre" "$PYTHON_BIN" JAR_COUNT=$(ls -1 "$JARS_DIR"/*.jar | wc -l) echo -e "${GREEN}โœ… Package prepared (${JAR_COUNT} JARs + JRE)${NC}" @@ -370,10 +355,10 @@ fi # Build wheel mkdir -p "$DIST_DIR" rm -f "$DIST_DIR"/*.whl -"$PYTHON_WITH_BUILD" -m build --wheel --outdir "$DIST_DIR" +"${BUILD_CMD[@]}" --wheel --outdir "$DIST_DIR" # Rename wheel to have correct platform tag if needed -# (python -m build may not set it correctly for cross-platform builds) +# (the build may not set it correctly for cross-platform builds) WHEEL_FILE=$(ls "$DIST_DIR"/*.whl | head -n1) if [[ -n "$WHEEL_FILE" ]]; then # Extract components from wheel filename diff --git a/scripts/build.sh b/scripts/build.sh index 7ba4e54..8aa15f3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -548,7 +548,6 @@ fi echo "" echo -e "${BLUE}๐Ÿ’ก Next steps:${NC}" -echo -e " ๐Ÿงช Run tests against this wheel (from the repository root; uv venv once first):" -echo -e " ${YELLOW}uv pip install --reinstall-package arcadedb \"[test]\"${NC}" -echo -e " ${YELLOW}uv run pytest${NC}" +echo -e " ๐Ÿงช Run tests against this wheel (from the repository root):" +echo -e " ${YELLOW}uv run --no-project --with \"[test]\" python -m pytest${NC}" echo "" diff --git a/scripts/ensure-build-tools.sh b/scripts/ensure-build-tools.sh deleted file mode 100755 index 09c57cb..0000000 --- a/scripts/ensure-build-tools.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Ensure build tools are installed for all Python versions available - -set -e - -echo "Checking Python installations and installing build tools..." - -if ! command -v uv &> /dev/null; then - echo "โŒ uv not found" - echo "๐Ÿ’ก Install uv: https://astral.sh/uv" - exit 1 -fi - -# Check system python3 -if command -v python3 &> /dev/null; then - PYTHON_PATH=$(command -v python3) - echo "Found python3 at: $PYTHON_PATH" - - # Try to install build in the current environment - # This will fail silently if the environment is managed (like Homebrew) - uv pip install --python "$PYTHON_PATH" build 2> /dev/null || echo " โš ๏ธ Could not install in $PYTHON_PATH (managed environment)" -fi - -# Also check common Homebrew Python 3.14 location -if [[ -f "/opt/homebrew/bin/python3.14" ]]; then - echo "Found /opt/homebrew/bin/python3.14" - uv pip install --python /opt/homebrew/bin/python3.14 build 2> /dev/null || echo " โš ๏ธ Could not install in /opt/homebrew/bin/python3.14 (managed environment)" -fi - -echo "โœ… Done"