diff --git a/docs/testing.md b/docs/testing.md index 430237c..2d472ff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -58,6 +58,8 @@ terminates the active pool before the browser closes. Set the complete exceptions corpus and requires identical ordered results, including resource/teardown observations. Use `--corpus PATH` for another corpus. It writes timings and host/corpus identities to `build/browser-worker-benchmark.json`. +Oracle validation/cache warming runs once before the timed samples and appears +separately as `reference_setup`, so a cold cache cannot bias the serial sample. The classes console compares its bulk successful language programs through two installed console host/worker sessions. Each run still creates its own module diff --git a/scripts/benchmark_browser_workers.py b/scripts/benchmark_browser_workers.py index d4d0679..2d37bc6 100644 --- a/scripts/benchmark_browser_workers.py +++ b/scripts/benchmark_browser_workers.py @@ -11,7 +11,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'runtime')) -from check_classes import check +from check_classes import cached_references, check def main(): @@ -23,6 +23,11 @@ def main(): args = parser.parse_args() if args.repeats < 1: parser.error('--repeats must be positive') + reference_start = time.monotonic() + reference = cached_references(args.corpus) + if reference['summary']['fail']: + raise AssertionError(reference['summary']) + reference_seconds = round(time.monotonic() - reference_start, 3) baseline, measurements = None, [] original = os.environ.get('POCKET_PYTHON_BROWSER_WORKERS') try: @@ -53,6 +58,8 @@ def main(): args.report.write_text(json.dumps({'schema_version': 1, 'python': platform.python_version(), 'host': platform.platform(), 'browser': report['host_version'], 'corpus_sha256': report['corpus_sha256'], 'repeats': args.repeats, + 'reference_setup': {'seconds': reference_seconds, 'summary': reference['summary'], + 'cache': reference['reference_cache']}, 'measurements': measurements}, indent=2) + '\n', encoding='utf-8')