Merge upstream develop into QC develop (2026-11-05)

.github/vercel-cli/pages-html-routes.py

@@ -12,8 +12,9 @@
 Usage: pages-html-routes.py OUTPUT_DIR
 
 OUTPUT_DIR is a `.vercel/output` directory holding `static/` and a
-`config.json` whose routes only set headers. The routes this script adds
-follow those header routes.
+`config.json` whose routes set headers, or redirect or rewrite a request
+that the HTML routes must not see, such as the Explorer's host. The routes
+this script adds follow them.
 """
 
 import json
@@ -44,6 +45,12 @@ def html_routes(static: Path) -> list[dict]:
         # Only reached when no file or directory matched.
         redirect(r"^/(.+)/$", "/$1", 307),
         {"src": r"^/(.+)$", "dest": "/$1.html", "check": True},
+        # An address that matches no file gets the build's 404 page, with a 404 status.
+        *(
+            [{"handle": "error"}, {"src": "^/.*$", "dest": "/404.html", "status": 404}]
+            if (static / "404.html").is_file()
+            else []
+        ),
     ]
 
 
@@ -52,7 +59,7 @@ def main(output: Path) -> None:
     config = json.loads(config_path.read_text())
     routes = config.get("routes", [])
     if any("handle" in route for route in routes):
-        sys.exit(f"{config_path}: routes must only set headers when HTML handling is added")
+        sys.exit(f"{config_path}: routes must not start a phase ('handle') when HTML handling is added")
     config["routes"] = routes + html_routes(output / "static")
     config_path.write_text(json.dumps(config, indent=2) + "\n")
 

.github/workflows/ci.yml

@@ -105,6 +105,7 @@ jobs:
             --with tabulate --with xxhash \
             pytest scripts/tests/test_measurement_id.py \
               scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py \
+              scripts/tests/test_refresh_bench_site.py \
               scripts/tests/test_codspeed_shards.py
 
   python-test:

.github/workflows/codspeed.yml

@@ -48,8 +48,8 @@ jobs:
               - ".github/workflows/codspeed.yml"
 
   # CodSpeed rejects an upload with too many benchmarks, so the simulation benchmarks run in
-  # shards of at most 800. The planner reads only Cargo.toml files and the committed benchmark
-  # counts, so this job needs no Rust toolchain and takes seconds.
+  # shards of at most 800. The planner balances recorded target runtimes across the available
+  # slots, keeping allocation modes separate. It reads only manifests and committed estimates.
   plan-codspeed:
     name: "Plan Codspeed shards"
     if: github.event.pull_request.draft != true

.github/workflows/commit-metadata.yml

@@ -47,8 +47,6 @@ jobs:
           RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
           RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
           AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
-          BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
-          BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
         run: |
           set -Eeuo pipefail
           echo -n > empty.jsonl
@@ -61,3 +59,13 @@ jobs:
             --commit-sha "${{ github.sha }}" \
             --branch "${{ github.ref_name }}" \
             --region "${AWS_REGION}"
+
+  # Tell the benchmarks website that the new commit exists. develop-bench.yml refreshes again
+  # after its benchmark results are ingested, but it does not wait for this workflow, so a push
+  # whose benchmark ingests all fail could otherwise leave this commit hidden by the cache.
+  refresh-site:
+    needs: commit-metadata
+    if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+    uses: ./.github/workflows/refresh-bench-site.yml
+    secrets:
+      BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}

.github/workflows/develop-bench.yml

@@ -14,11 +14,9 @@ permissions:
 jobs:
   build:
     timeout-minutes: 60
-    # GitHub drops runner labels longer than 255 characters, and the job then waits for a runner
-    # forever. With the long `family` list and the longest benchmark id, this label is close.
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
+          && format('runs-on={0}/region=us-east-1/runner=arm64-bench/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
           || 'ubuntu-latest' }}
     strategy:
       fail-fast: false
@@ -155,6 +153,39 @@ jobs:
             litdata==0.2.76 torch pyarrow numpy
           echo "LITDATA_PYTHON=$PWD/.venv-litdata/bin/python" >> "$GITHUB_ENV"
 
+      # Parquet and LitData do not depend on Vortex code, so they are cached; LitData alone is
+      # about 80% of data generation time. Vortex, Arrow IPC and Lance are written every run.
+      # The key covers the dataset generators, the crates that write and seed the Parquet files,
+      # and the LitData conversion. Keep the litdata version in sync with the Install LitData step.
+      - name: Compute benchmark data cache key
+        id: data-cache-key
+        if: matrix.benchmark.id == 'random-access-bench'
+        shell: bash
+        run: |
+          set -Eeu -o pipefail
+          crates=$(awk '/^\[\[package\]\]/ { name = "" }
+            /^name = / { name = $3 }
+            /^version = / && name ~ /^"(parquet|arrow-.*|rand.*)"$/ { print name, $3 }' Cargo.lock \
+            | sort | sha256sum | cut -c1-16)
+          echo "crates=$crates" >> "$GITHUB_OUTPUT"
+
+      - name: Restore benchmark data
+        id: data-cache
+        if: matrix.benchmark.id == 'random-access-bench'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            vortex-bench/data/taxi/taxi.parquet
+            vortex-bench/data/random_access/*/*.parquet
+            vortex-bench/data/random_access/*/*.litdata
+          key: >-
+            random-access-data-v1-${{ matrix.benchmark.remote_data_dir && 's3' || 'local' }}-litdata0.2.76-${{ steps.data-cache-key.outputs.crates }}-${{
+              hashFiles(
+                'vortex-bench/src/datasets/**',
+                'vortex-bench/src/random_access/**',
+                'scripts/litdata-random-access.py',
+                'benchmarks/random-access-bench/src/litdata.rs') }}
+
       # Download and convert all benchmark data up front, so the benchmark step below only
       # contains timed runs. The data is written with the same settings the benchmark uses.
       - name: Generate ${{ matrix.benchmark.name }} data
@@ -167,6 +198,16 @@ jobs:
           python3 "scripts/${bench%-bench}-split.py" --prepare-data \
             ${{ matrix.benchmark.remote_data_dir && format('--remote-data-dir {0}', matrix.benchmark.remote_data_dir) || '' }}
 
+      - name: Save benchmark data
+        if: matrix.benchmark.id == 'random-access-bench' && steps.data-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            vortex-bench/data/taxi/taxi.parquet
+            vortex-bench/data/random_access/*/*.parquet
+            vortex-bench/data/random_access/*/*.litdata
+          key: ${{ steps.data-cache.outputs.cache-primary-key }}
+
       - name: Upload benchmark data to S3
         if: matrix.benchmark.remote_data_dir != null
         shell: bash
@@ -238,8 +279,6 @@ jobs:
           RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
           RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
           AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
-          BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
-          BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
         run: |
           set -Eeuo pipefail
           curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
@@ -266,3 +305,13 @@ jobs:
     with:
       mode: "develop"
       matrix_preset: "develop"
+
+  # Tell the benchmarks website that new results exist, once per push, after every benchmark job
+  # of the push has ingested its results. `always()` runs it even when a benchmark job failed, so
+  # the results that other jobs ingested still appear.
+  refresh-site:
+    needs: [bench, sql]
+    if: ${{ always() }}
+    uses: ./.github/workflows/refresh-bench-site.yml
+    secrets:
+      BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}

.github/workflows/docs-preview.yml

@@ -52,9 +52,10 @@ jobs:
                 '.github/workflows/docs-build.yml',
                 '.github/workflows/docs-preview.yml',
                 '.github/workflows/deploy-vercel.yml',
+                '.github/workflows/web-build.yml',
               ]);
               eligible = files.some(file => [file.filename, file.previous_filename].some(
-                path => path && (path.startsWith('docs/') ||
+                path => path && (path.startsWith('docs/') || path.startsWith('vortex-web/') ||
                   path.startsWith('.github/vercel-cli/') || workflows.has(path)),
               ));
             }
@@ -71,11 +72,22 @@ jobs:
       cancel-in-progress: true
     uses: ./.github/workflows/docs-build.yml
 
+  web:
+    name: Build Explorer preview
+    needs: changes
+    if: needs.changes.outputs.eligible == 'true'
+    permissions:
+      contents: read
+    concurrency:
+      group: docs-preview-web-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    uses: ./.github/workflows/web-build.yml
+
   # Each Vercel deployment has its own URL, so a stale build cannot replace
   # a newer preview, and the PR needs no recheck before publishing.
   publish:
     name: Publish preview
-    needs: build
+    needs: [build, web]
     permissions:
       contents: read
       deployments: write
@@ -85,6 +97,6 @@ jobs:
     secrets: inherit
     with:
       environment: vercel-docs
-      artifact-ids: ${{ needs.build.outputs.artifact-id }}
+      artifact-ids: ${{ needs.build.outputs.artifact-id }},${{ needs.web.outputs.artifact-id }}
       config: docs/vercel-output-config.json
       pages-html-handling: true

.github/workflows/docs.yml

@@ -21,9 +21,16 @@ jobs:
   build:
     uses: ./.github/workflows/docs-build.yml
 
+  web:
+    name: Build Explorer
+    permissions:
+      contents: read
+    uses: ./.github/workflows/web-build.yml
+
+  # One Vercel project serves the docs and the Explorer.
   deploy:
     name: Deploy to Vercel
-    needs: build
+    needs: [build, web]
     permissions:
       contents: read
       deployments: write
@@ -33,7 +40,7 @@ jobs:
     secrets: inherit
     with:
       environment: vercel-docs
-      artifact-ids: ${{ needs.build.outputs.artifact-id }}
+      artifact-ids: ${{ needs.build.outputs.artifact-id }},${{ needs.web.outputs.artifact-id }}
       config: docs/vercel-output-config.json
       pages-html-handling: true
       production: ${{ github.ref == 'refs/heads/develop' }}

.github/workflows/nightly-bench.yml

@@ -31,3 +31,12 @@ jobs:
         machine_type:
           - id: arm64
             instance_name: c8gd.metal-24xl
+
+  # Tell the benchmarks website that new results exist, once per run, after every machine type
+  # has ingested its results. `always()` runs it even when one machine type failed.
+  refresh-site:
+    needs: sql
+    if: ${{ always() }}
+    uses: ./.github/workflows/refresh-bench-site.yml
+    secrets:
+      BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}

.github/workflows/package.yml

@@ -202,6 +202,8 @@ jobs:
         with:
           enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
       - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29  # v2.2.1
+        with:
+          version: 0.17.0
       - name: Install cargo-zigbuild
         uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
         with:

.github/workflows/pr-bench-runner.yml

@@ -44,11 +44,9 @@ permissions:
 jobs:
   build:
     timeout-minutes: 60
-    # GitHub drops runner labels longer than 255 characters, and the job then waits for a runner
-    # forever. With the long `family` list and the longest benchmark id, this label is close.
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
+          && format('runs-on={0}/region=us-east-1/runner=arm64-bench/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
     steps:
       - uses: runs-on/action@v2
@@ -167,6 +165,39 @@ jobs:
             litdata==0.2.76 torch pyarrow numpy
           echo "LITDATA_PYTHON=$PWD/.venv-litdata/bin/python" >> "$GITHUB_ENV"
 
+      # Parquet and LitData do not depend on Vortex code, so they are cached; LitData alone is
+      # about 80% of data generation time. Vortex, Arrow IPC and Lance are written every run.
+      # The key covers the dataset generators, the crates that write and seed the Parquet files,
+      # and the LitData conversion. Keep the litdata version in sync with the Install LitData step.
+      - name: Compute benchmark data cache key
+        id: data-cache-key
+        if: inputs.benchmark_id == 'random-access-bench'
+        shell: bash
+        run: |
+          set -Eeu -o pipefail
+          crates=$(awk '/^\[\[package\]\]/ { name = "" }
+            /^name = / { name = $3 }
+            /^version = / && name ~ /^"(parquet|arrow-.*|rand.*)"$/ { print name, $3 }' Cargo.lock \
+            | sort | sha256sum | cut -c1-16)
+          echo "crates=$crates" >> "$GITHUB_OUTPUT"
+
+      - name: Restore benchmark data
+        id: data-cache
+        if: inputs.benchmark_id == 'random-access-bench'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            vortex-bench/data/taxi/taxi.parquet
+            vortex-bench/data/random_access/*/*.parquet
+            vortex-bench/data/random_access/*/*.litdata
+          key: >-
+            random-access-data-v1-${{ inputs.remote_data_dir != '' && 's3' || 'local' }}-litdata0.2.76-${{ steps.data-cache-key.outputs.crates }}-${{
+              hashFiles(
+                'vortex-bench/src/datasets/**',
+                'vortex-bench/src/random_access/**',
+                'scripts/litdata-random-access.py',
+                'benchmarks/random-access-bench/src/litdata.rs') }}
+
       # Download and convert all benchmark data up front, so the benchmark step below only
       # contains timed runs. The data is written with the same settings the benchmark uses.
       - name: Generate ${{ inputs.benchmark_name }} data
@@ -179,6 +210,16 @@ jobs:
           python3 "scripts/${bench%-bench}-split.py" --prepare-data \
             ${{ inputs.remote_data_dir != '' && format('--remote-data-dir {0}', inputs.remote_data_dir) || '' }}
 
+      - name: Save benchmark data
+        if: inputs.benchmark_id == 'random-access-bench' && steps.data-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            vortex-bench/data/taxi/taxi.parquet
+            vortex-bench/data/random_access/*/*.parquet
+            vortex-bench/data/random_access/*/*.litdata
+          key: ${{ steps.data-cache.outputs.cache-primary-key }}
+
       - name: Upload benchmark data to S3
         if: inputs.remote_data_dir != '' && github.event.pull_request.head.repo.fork == false
         shell: bash

.github/workflows/publish-dry-runs.yml

@@ -46,6 +46,8 @@ jobs:
         with:
           save-if: ${{ github.ref_name == 'develop' }}
       - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29  # v2.2.1
+        with:
+          version: 0.17.0
       - name: Install uv
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
@@ -104,6 +106,8 @@ jobs:
             with:
               enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
           - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29  # v2.2.1
+            with:
+              version: 0.17.0
       - name: Install cargo-zigbuild
         uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
         with:

.github/workflows/refresh-bench-site.yml

@@ -0,0 +1,29 @@
+# Tells the benchmarks website that new results exist, then warms its default pages. Callers run
+# it once per run, after their benchmark ingests finish, instead of once per benchmark job. This
+# keeps the website from flushing and refilling its cache dozens of times an hour while
+# benchmarks run. The script does nothing when the URL or the token is not set.
+
+name: Refresh benchmarks website
+
+on:
+  workflow_call:
+    secrets:
+      BENCH_REVALIDATE_TOKEN:
+        required: false
+
+jobs:
+  refresh:
+    if: vars.BENCH_SITE_BASE_URL != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+        with:
+          sparse-checkout: scripts
+      - name: Revalidate and warm the benchmarks website
+        env:
+          BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
+          BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
+        run: python3 scripts/refresh-bench-site.py

.github/workflows/release-binaries.yml

@@ -57,6 +57,8 @@ jobs:
       - name: Setup Zig
         if: contains(matrix.target, 'linux')
         uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29  # v2.2.1
+        with:
+          version: 0.17.0
 
       - name: Install cargo-zigbuild
         if: contains(matrix.target, 'linux')