develop-bench.yml

# Runs after every commit to `develop` (or in other words, _after_ every pull request merges).

name: Develop Benchmarks

on:
  push:
    branches: [develop]

permissions:
  id-token: write  # enables AWS-GitHub OIDC
  actions: read
  contents: write

jobs:
  build:
    timeout-minutes: 60
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/region=us-east-1/runner=arm64-bench/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
          || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        benchmark:
          - id: random-access-bench
            build_args: "--features lance"
          - id: compress-bench
            build_args: "--features lance"
          - id: string-bench
            build_args: ""
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/assert-cpu
        if: github.repository == 'vortex-data/vortex'
      - uses: ./.github/actions/setup-prebuild
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
      - name: Build binary
        shell: bash
        env:
          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
        run: |
          cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }}
      - name: Pre-upload benchmark debuginfo to Polar Signals
        uses: ./.github/actions/upload-parca-debuginfo
        continue-on-error: true
        with:
          paths: |
            target/release_debug/${{ matrix.benchmark.id }}
          polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
          project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
      - name: Upload benchmark binary
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7
        with:
          name: ${{ matrix.benchmark.id }}-binary
          path: target/release_debug/${{ matrix.benchmark.id }}

  bench:
    needs: build
    if: ${{ !cancelled() && needs.build.result == 'success' }}
    timeout-minutes: 120
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag={1}{2}', github.run_id, matrix.benchmark.id, matrix.benchmark.variant_id)
          || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        benchmark:
          - id: random-access-bench
            name: Random Access
            build_args: "--features lance"
            v4_ingest: true
          # Same benchmark, reading the data from S3 instead of local NVMe.
          - id: random-access-bench
            variant_id: "-s3"
            name: Random Access (S3)
            build_args: "--features lance"
            v4_ingest: true
            remote_data_dir: >-
              s3://vortex-ci-benchmark-datasets/develop/random-access/
          - id: compress-bench
            name: Compression
            build_args: "--features lance"
            v4_ingest: true
          - id: string-bench
            name: String Encoding
            v4_ingest: true
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7

      - name: Install DuckDB
        uses: ./.github/actions/setup-duckdb

      - uses: ./.github/actions/system-info

      - name: Download benchmark binary
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8
        with:
          name: ${{ matrix.benchmark.id }}-binary
          path: target/release_debug/

      - name: Make benchmark binary executable
        shell: bash
        run: chmod +x target/release_debug/${{ matrix.benchmark.id }}

      - name: Setup Polar Signals
        uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
        # Profiling is observability only, so a flaky agent download must not fail the benchmark.
        continue-on-error: true
        with:
          polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.benchmark.id }}"
          project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
          job_name: "${{ matrix.benchmark.id }}"
          profiling_frequency: 199
          extra_args: "--debuginfo-strip=false"
          parca_agent_version: "0.49.0"

      - name: Setup AWS CLI
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6
        with:
          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
          aws-region: us-east-1

      - name: Install uv for LitData
        if: matrix.benchmark.id == 'random-access-bench' && matrix.benchmark.remote_data_dir == null
        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
        with:
          sync: false

      - name: Install LitData
        # LitData is Python-only, so random-access-bench times it through
        # scripts/litdata-random-access.py, run by the interpreter named in LITDATA_PYTHON.
        if: matrix.benchmark.id == 'random-access-bench' && matrix.benchmark.remote_data_dir == null
        shell: bash
        run: |
          set -Eeu -o pipefail -x

          uv venv --python 3.12 .venv-litdata
          uv pip install --python .venv-litdata \
            --index-strategy unsafe-best-match \
            --extra-index-url https://download.pytorch.org/whl/cpu \
            litdata==0.2.76 torch pyarrow numpy
          echo "LITDATA_PYTHON=$PWD/.venv-litdata/bin/python" >> "$GITHUB_ENV"

      # Parquet and LitData do not depend on Vortex code, so they are cached; LitData alone is
      # about 80% of data generation time. Vortex, Arrow IPC and Lance are written every run.
      # The key covers the dataset generators, the crates that write and seed the Parquet files,
      # and the LitData conversion. Keep the litdata version in sync with the Install LitData step.
      - name: Compute benchmark data cache key
        id: data-cache-key
        if: matrix.benchmark.id == 'random-access-bench'
        shell: bash
        run: |
          set -Eeu -o pipefail
          crates=$(awk '/^\[\[package\]\]/ { name = "" }
            /^name = / { name = $3 }
            /^version = / && name ~ /^"(parquet|arrow-.*|rand.*)"$/ { print name, $3 }' Cargo.lock \
            | sort | sha256sum | cut -c1-16)
          echo "crates=$crates" >> "$GITHUB_OUTPUT"

      - name: Restore benchmark data
        id: data-cache
        if: matrix.benchmark.id == 'random-access-bench'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            vortex-bench/data/taxi/taxi.parquet
            vortex-bench/data/random_access/*/*.parquet
            vortex-bench/data/random_access/*/*.litdata
          key: >-
            random-access-data-v1-${{ matrix.benchmark.remote_data_dir && 's3' || 'local' }}-litdata0.2.76-${{ steps.data-cache-key.outputs.crates }}-${{
              hashFiles(
                'vortex-bench/src/datasets/**',
                'vortex-bench/src/random_access/**',
                'scripts/litdata-random-access.py',
                'benchmarks/random-access-bench/src/litdata.rs') }}

      # Download and convert all benchmark data up front, so the benchmark step below only
      # contains timed runs. The data is written with the same settings the benchmark uses.
      - name: Generate ${{ matrix.benchmark.name }} data
        shell: bash
        env:
          RUST_BACKTRACE: full
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
        run: |
          bench="${{ matrix.benchmark.id }}"
          python3 "scripts/${bench%-bench}-split.py" --prepare-data \
            ${{ matrix.benchmark.remote_data_dir && format('--remote-data-dir {0}', matrix.benchmark.remote_data_dir) || '' }}

      - name: Save benchmark data
        if: matrix.benchmark.id == 'random-access-bench' && steps.data-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            vortex-bench/data/taxi/taxi.parquet
            vortex-bench/data/random_access/*/*.parquet
            vortex-bench/data/random_access/*/*.litdata
          key: ${{ steps.data-cache.outputs.cache-primary-key }}

      - name: Upload benchmark data to S3
        if: matrix.benchmark.remote_data_dir != null
        shell: bash
        env:
          AWS_REGION: "us-east-1"
        run: |
          set -Eeu -o pipefail -x

          aws s3 rm --recursive "${{ matrix.benchmark.remote_data_dir }}"
          aws s3 cp --recursive vortex-bench/data "${{ matrix.benchmark.remote_data_dir }}"

      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh

      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
        if: matrix.benchmark.id == 'random-access-bench'
        shell: bash
        env:
          RUST_BACKTRACE: full
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
          AWS_REGION: "us-east-1"
        run: |
          python3 scripts/random-access-split.py --emit-ingest-records \
            ${{ matrix.benchmark.remote_data_dir && format('--remote-data-dir {0}', matrix.benchmark.remote_data_dir) || '' }}

      - name: Run ${{ matrix.benchmark.name }} benchmark (per-dataset)
        if: matrix.benchmark.id == 'compress-bench'
        shell: bash
        env:
          RUST_BACKTRACE: full
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
        run: |
          python3 scripts/compress-split.py --formats arrow-ipc,parquet,lance,vortex --emit-ingest-records

      - name: Run ${{ matrix.benchmark.name }} benchmark (per-column-encoder)
        if: matrix.benchmark.id == 'string-bench'
        shell: bash
        env:
          RUST_BACKTRACE: full
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
        run: |
          python3 scripts/string-split.py

      # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
      # benchmarks website for suites that emit v4 records; a failure here fails the job.
      # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for
      # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from
      # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
      #
      # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
      # the uv binary, never the synced workspace. A full sync would rebuild
      # vortex-python via sccache->S3, which fails under the ingest-role creds
      # (rds-db:connect only) and is pure waste here.
      - name: Install uv for v4 ingest
        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
        with:
          sync: false
      - name: Configure AWS credentials for v4 ingest (OIDC)
        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6
        with:
          role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
          aws-region: ${{ vars.RDS_BENCH_REGION }}
      - name: Ingest results to v4 Postgres
        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
        shell: bash
        env:
          RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
          RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
          AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
        run: |
          set -Eeuo pipefail
          curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
            -o "${RUNNER_TEMP}/rds-global-bundle.pem"
          DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
          uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \
            scripts/post-ingest.py results.ingest.jsonl \
            --postgres "${DSN}" \
            --commit-sha "${{ github.sha }}" \
            --branch "${{ github.ref_name }}" \
            --region "${AWS_REGION}"

      - name: Alert incident.io
        if: failure()
        uses: ./.github/actions/alert-incident-io
        with:
          api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
          alert-title: "${{ matrix.benchmark.name }} benchmark failed on develop"
          deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure

  sql:
    uses: ./.github/workflows/sql-bench-matrix.yml
    secrets: inherit
    with:
      mode: "develop"
      matrix_preset: "develop"

  # Tell the benchmarks website that new results exist, once per push, after every benchmark job
  # of the push has ingested its results. `always()` runs it even when a benchmark job failed, so
  # the results that other jobs ingested still appear.
  refresh-site:
    needs: [bench, sql]
    if: ${{ always() }}
    uses: ./.github/workflows/refresh-bench-site.yml
    secrets:
      BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}