Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/workflows/benchmark-gate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,12 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e . polars pandas pyarrow pytest
pip install -e . polars pandas pyarrow pytest pytest-cov

- name: Run gate & engine unit tests
run: pytest tests/test_polars_engine.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q
# pyproject addopts enforce 90% repo-wide coverage; this 3-file subset
# keeps coverage on with the lower floor that other subset lanes use.
run: pytest tests/test_backend_honesty.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q --cov-fail-under=40

- name: Record Candidate (PR Head) Benchmark
run: |
Expand All @@ -61,7 +63,9 @@ jobs:

- name: Evaluate Regression Gate
run: |
git checkout "${{ github.event.pull_request.head.sha }}"
# The base step copied the PR's benchmarks/ over the base tree; --force
# discards that overlay so PRs that change benchmarks/ can switch back.
git checkout --force "${{ github.event.pull_request.head.sha }}"
python -m benchmarks.perf_gate compare \
--baseline /tmp/baseline_metrics.json \
--candidate /tmp/candidate_metrics.json \
Expand Down
87 changes: 12 additions & 75 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ concurrency:
permissions:
contents: read

# Dependency pinning: the gating jobs install with `-c constraints/ci.txt`
# (exported from uv.lock), so an upstream release cannot turn CI red overnight.
# Dependency pinning: the gating jobs install with `uv sync --frozen` from
# uv.lock, so an upstream release cannot turn CI red overnight.
# test-matrix deliberately floats to the latest compatible releases: it is the
# early warning for upstream breakage. Regenerate with `make constraints`.
# early warning for upstream breakage. Regenerate with `uv lock`.

jobs:
lockfile:
Expand All @@ -34,15 +34,10 @@ jobs:
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: uv.lock and constraints/ci.txt are up to date
- name: uv.lock is up to date
run: |
python -m pip install "uv==0.12.10"
uv lock --check
uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt
if ! git diff --exit-code -- constraints/ci.txt; then
echo "::error::constraints/ci.txt is stale; run 'make constraints' and commit the result"
exit 1
fi

quality-fast:
runs-on: ubuntu-latest
Expand All @@ -56,7 +51,8 @@ jobs:
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,ml]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev
- name: Lint
run: ruff check .
- name: Test (required fast lane)
Expand All @@ -68,33 +64,6 @@ jobs:
pip install "numpy<2.5"
mypy src/freshdata

truthbench:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
cache: pip
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,ml]"
- name: TruthBench PR ratchet
# PRs must not regress any gate that passes in the committed
# baseline.json; the known-red gates are release blockers and are
# enforced absolutely by the release workflow (make truthbench-release),
# so they are reported here as KNOWN-RED without failing every
# unrelated PR. Infrastructure errors and partial runs still fail.
run: make truthbench-pr
- name: Upload run artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: truthbench-results
path: benchmarks/truthbench/results/

test-matrix:
runs-on: ubuntu-latest
timeout-minutes: 30
Expand Down Expand Up @@ -144,7 +113,8 @@ jobs:
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,ml]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev
- name: Nightly online/large drift checks
# --no-cov: this lane deliberately runs ~20 tests, so the repo-wide
# --cov-fail-under=93 inherited from addopts can never be met here.
Expand Down Expand Up @@ -190,7 +160,8 @@ jobs:
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,ml]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev
- name: Assert the large/online lane still collects tests
run: |
N=$(pytest -m "online or large or tier1" --collect-only -q -o addopts="" \
Expand Down Expand Up @@ -223,41 +194,6 @@ jobs:
python -m build
twine check dist/*

freshcore-native:
# Builds the optional FreshCore Rust extension (crates/freshcore) and runs
# the FreshCore tests against it. No other job installs the extension, so
# the native parity tests skip everywhere else.
runs-on: ubuntu-latest
timeout-minutes: 10
env:
RUSTUP_TOOLCHAIN: "1.98.1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
cache: pip
- name: Install Rust toolchain
run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal
- name: Install
# maturin develop installs into a virtualenv, so create one in the
# workspace and put it first on PATH for the remaining steps.
run: |
python -m venv .venv
echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
.venv/bin/python -m pip install --upgrade pip
.venv/bin/pip install -c constraints/ci.txt -e ".[dev,freshcore]"
- name: Rust unit tests
run: cargo test --locked --manifest-path crates/freshcore/Cargo.toml
- name: Build the extension
run: maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module
- name: FreshCore tests
# The coverage gate in addopts applies to the full suite, not this subset.
run: |
python -c "import freshdata_freshcore"
pytest tests/test_execution -k freshcore -o addopts="--strict-markers" -q

coverage-badge:
# Publish a self-hosted shields endpoint badge to the `badges` branch
# (no third-party account needed). Runs only on main.
Expand All @@ -276,7 +212,8 @@ jobs:
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,ml]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev
- name: Compute coverage and write badge JSON
run: |
pytest -m "not online and not large and not tier1" \
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/perf-regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ jobs:
- name: Install
run: |
python -m pip install -U pip
pip install -c constraints/ci.txt -e ".[bench]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact
- name: Resolve base commit
id: base
env:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/performance-large.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ jobs:
- name: Install benchmark dependencies
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[dev,bench,ml]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev

- name: Run large performance matrix
id: benchmark
Expand Down
49 changes: 0 additions & 49 deletions .github/workflows/plugin-contract.yml

This file was deleted.

3 changes: 2 additions & 1 deletion .github/workflows/public-benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ jobs:
- name: Install
run: |
python -m pip install --upgrade pip
pip install -c constraints/ci.txt -e ".[bench]"
python -m pip install "uv==0.12.10"
UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact
- name: Run public benchmark
env:
SCALE: ${{ inputs.scale }}
Expand Down
6 changes: 0 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,6 @@ jobs:
run: mkdocs build --strict
- name: Validation Gauntlet
run: python -m benchmarks.gauntlet run --rows 300 --check
- name: CleanBench release gates
run: python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --check-gates
- name: TruthBench release verification
# Fail-closed: a partial run, missing backend, or any absolute gate
# failure blocks the release before an artifact is built.
run: make truthbench-release

# ---------------------------------------------------------------------------
# 3. Build sdist + wheel exactly once; checksum them; attest provenance.
Expand Down
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,9 @@ benchmarks/results/performance/*
benchmarks/generated_fixtures/

# Rust native backend build output
crates/freshcore/target/

# Phase 5 training pipeline: teacher prompt/response audit cache is a local
# dev artifact (regenerated by running teacher tasks), not committed content.
training/cache/*
!training/cache/.gitkeep
benchmarks/gauntlet/results/
# TruthBench: the full run output is large and regenerated by the release
# command; only the compact README and the regression baseline are committed.
Expand Down
1 change: 0 additions & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ include README.md
include LICENSE
include pyproject.toml
recursive-include src/freshdata *.py py.typed
recursive-include crates/freshcore Cargo.toml Cargo.lock *.rs
prune tests
prune benchmarks
prune scripts
17 changes: 1 addition & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,9 @@

PY ?= python

# training-* targets are matched by the pattern rule below (pattern rules
# cannot be .PHONY; the delegated targets are .PHONY inside training/Makefile).
.PHONY: help benchmark benchmark-ci benchmark-report benchmark-fixtures benchmark-test \
cleanbench-full truthbench-release truthbench-pr automation-pr performance-ci \
performance-baseline performance-profile performance-report coverage-report \
constraints
performance-baseline performance-profile performance-report coverage-report

help:
@echo "Targets:"
Expand All @@ -25,14 +22,6 @@ help:
@echo " performance-profile Profile one 100k-row performance case"
@echo " performance-report Analyze and render compact performance evidence"
@echo " coverage-report Show per-module coverage for the fast test lane"
@echo " training-* Phase-5 training pipeline (see training/Makefile)"
@echo " constraints Refresh uv.lock and the pinned constraints/ci.txt CI installs with"

# Refresh the lockfile and the pip constraints the gating CI jobs install with.
# The CI `lockfile` job fails when either is stale; commit both after running.
constraints:
uv lock
uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt

# Run the same test scope as the required PR lane and print uncovered lines for
# every measured module so contributors can identify useful testing targets.
Expand Down Expand Up @@ -69,10 +58,6 @@ truthbench-pr:
automation-pr:
$(PY) scripts/automate_freshdata.py --output-dir scripts/.automation_out

# Phase-5 training pipeline targets delegate to training/Makefile.
training-%:
$(MAKE) -C training PY=$(PY) $@

# Full-scale local run. Override sizes per fixture by editing DEFAULT_SIZES in
# benchmarks/bench.py, or call bench.py single --size <n> for the 5M+ variants.
benchmark:
Expand Down
Loading
Loading