diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfbe6a2..141dc77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,19 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' run: git diff --exit-code -- docs/reference/validation.md + - name: Truth report (ubuntu / 3.13 only; a report, not a gate) + # How closely the methods recover answers known by construction. Kept as an + # artifact on every run so the figures behind validation.md's last section are at hand. + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + run: python validation/truth_report.py --out truth_report.md + + - name: Upload the truth report + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: truth-report + path: truth_report.md + - name: Upload coverage to Codecov if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index f819ab0..c1774c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,10 +73,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`examples/` and `validation/effects_visualization.ipynb`.** These demos predated the 0.2.0 API (`ProcessDataFrame`, `study.sds`, moved data files) and no longer ran; nothing linked to them. The maintained, CI-executed tutorials are in `docs/tutorials/`. +- **`tests/test_bishop_reference.py`.** Its one test compared residual values with a workbook + of Dr. Bishop's that is not in the repository, so it skipped on every run. Residual charts are + still compared with VAS through their limits. ### CI / Infrastructure - **Python 3.14 is tested** on Ubuntu and Windows, in CI and in the release gate, and listed in the package classifiers. +- **The truth report runs in CI.** `validation/truth_report.py` runs on every push (Ubuntu, + Python 3.13) and its output is kept as the `truth-report` artifact. ## [0.5.0] - 2026-10-29 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5815bd..7a9fee6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,13 +74,7 @@ The full test suite is the gate: if it fails, the change is not ready. The file `validation/PBTESTDATABASE_T100.csv` is ground-truth Minitab output for DS 1–6 from Bishop's reference materials. Numerical changes that touch residuals (R1–R5), DS detection, limit calculations, or variance -decomposition must keep the relevant Bishop-reference tests green: - -```bash -pytest tests/test_bishop_reference.py -v # skips unless Bishop's residual workbooks are present -``` - -There is also a full end-to-end check that compares every chart center, control +decomposition must keep the end-to-end check green. It compares every chart center, control limit, capability index and loss component against Bishop's VAS runs for the 10-12-2026 manual (PM SDS 1–6, PM INERT, his known-effects data and the Medicare data): diff --git a/docs/reference/validation.md b/docs/reference/validation.md index 50e6031..5d15859 100644 --- a/docs/reference/validation.md +++ b/docs/reference/validation.md @@ -52,8 +52,7 @@ These outputs have no VAS reference in this run. They are covered by unit tests, other references, but no VAS value is checked: - **Residual values one by one.** Charts of R1-R6 are compared through their limits above; - `tests/test_bishop_reference.py` compares the values themselves when the reference - spreadsheets are present, which they are not in this repository, so it skips. + the values themselves are not compared. - **Signals** (run rules and points beyond the limits). - **Staged (phased) Xbar and S charts.** These are checked against Wheeler's cavity data and Minitab's per-stage limits in `tests/test_phased_xbar_s.py`. @@ -64,7 +63,8 @@ other references, but no VAS value is checked: `validation/truth_report.py` asks a different question: how closely the methods, as implemented, recover answers known by construction (pure noise, and data built from known -effects). It is a report, not a gate. +effects). It is a report, not a gate. CI runs it on every push and keeps its output as the +`truth-report` artifact. ## Values with no reference yet diff --git a/tests/test_bishop_reference.py b/tests/test_bishop_reference.py deleted file mode 100644 index 38bac16..0000000 --- a/tests/test_bishop_reference.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Validate VAS residuals (R1–R5) against Tom Bishop's Minitab reference output. - -Each test loads a cn-tab comparison Excel file containing both input data and -Tom's known-good residuals, runs our pipeline fresh, and asserts our residuals -match within floating-point tolerance. - -This is the only way to catch formula-level bugs — external ground truth, -not self-referential identity checks. -""" - -import os - -import numpy as np -import pandas as pd -import pytest - -from processbehavior import ProcessBehavior - -pytestmark = pytest.mark.integration - -VALIDATION_DIR = 'validation' -SHEET_NAME = 'VAS-ANALYSIS-DATABASE.mwx' - -RESIDUAL_COLS = ['R1', 'R2', 'R3', 'R4', 'R5'] -REF_RESIDUAL_COLS = [ - 'R1 RESIDUALS', - 'R2 RESIDUALS', - 'R3 RESIDUALS', - 'R4 RESIDUALS', - 'R5 RESIDUALS', -] - - -def _ref_path(sds_num: int) -> str: - return f'{VALIDATION_DIR}/cn-tab comparison sds {sds_num} (3-5-26).xlsx' - - -def _load_reference(sds_num: int) -> pd.DataFrame: - """Load the cn-tab comparison file for a given SDS type.""" - return pd.read_excel(_ref_path(sds_num), sheet_name=SHEET_NAME) - - -# SDS 1 only: its R2 is the within-cell deviation, unchanged by the 10-1 manual. The 3-5-26 -# workbooks for SDS 2, 3 and 6 predate the 10-1 R2 (the condition-and-period-adjusted, rescaled -# difference) and no longer describe the reference; Tom's 10/3/2026 run replaces them. -@pytest.mark.parametrize('sds_num', [1]) -def test_residuals_match_bishop_reference(sds_num: int) -> None: - if not os.path.exists(_ref_path(sds_num)): - pytest.skip(f'Reference file not found: {_ref_path(sds_num)}') - ref = _load_reference(sds_num) - - # --- Run our pipeline fresh --- - input_df = ref[['time', 'factor 1', 'factor 2', 'y']].copy() - study = ProcessBehavior(input_df).formulate(response='y', factors=['factor 1', 'factor 2'], time='time') - ds = study.dataset - - # --- Merge on y to align rows regardless of sort order --- - merged = pd.merge( - ds[['y'] + RESIDUAL_COLS], - ref[['y'] + REF_RESIDUAL_COLS], - on='y', - how='inner', - ) - assert len(merged) == len(ref), ( - f'SDS {sds_num}: row count mismatch after merge (got {len(merged)}, expected {len(ref)})' - ) - - # --- Assert each residual matches --- - for our_col, ref_col in zip(RESIDUAL_COLS, REF_RESIDUAL_COLS, strict=False): - ours = merged[our_col] - toms = merged[ref_col] - - # Drop rows where the reference leaves a residual blank. - mask = toms.notna() - - np.testing.assert_allclose( - ours[mask].to_numpy(), - toms[mask].to_numpy(), - atol=1e-8, - err_msg=f'SDS {sds_num} {our_col} mismatch vs Bishop reference', - ) diff --git a/validation/e2e_bishop_report.py b/validation/e2e_bishop_report.py index c19e5d4..eb547f0 100644 --- a/validation/e2e_bishop_report.py +++ b/validation/e2e_bishop_report.py @@ -870,8 +870,7 @@ def generate_myst_summary(all_results, aux_by_run, runs): 'other references, but no VAS value is checked:', '', '- **Residual values one by one.** Charts of R1-R6 are compared through their limits above;', - ' `tests/test_bishop_reference.py` compares the values themselves when the reference', - ' spreadsheets are present, which they are not in this repository, so it skips.', + ' the values themselves are not compared.', '- **Signals** (run rules and points beyond the limits).', "- **Staged (phased) Xbar and S charts.** These are checked against Wheeler's cavity data and", " Minitab's per-stage limits in `tests/test_phased_xbar_s.py`.", @@ -882,7 +881,8 @@ def generate_myst_summary(all_results, aux_by_run, runs): '', '`validation/truth_report.py` asks a different question: how closely the methods, as', 'implemented, recover answers known by construction (pure noise, and data built from known', - 'effects). It is a report, not a gate.', + 'effects). It is a report, not a gate. CI runs it on every push and keeps its output as the', + '`truth-report` artifact.', '', '## Values with no reference yet', '',