From 997e9a88a4b5d64c5d988588bdb7dafd4c969604 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:36:46 -0800 Subject: [PATCH 1/6] perf(CI-04): reuse validated CPython corpus references --- .github/workflows/ci.yml | 11 ++++++ docs/testing.md | 9 +++++ runtime/check_classes.py | 49 ++++++++++++++++++++++++- tests/test_reference_diagnostics.py | 57 +++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2affde2..dabebae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,11 @@ jobs: version: "0.12.19" enable-cache: true - run: uv sync --locked + - uses: actions/cache@v5 + with: + path: build/reference-cache + key: oracle-v1-${{ runner.os }}-node-${{ matrix.shard }}-${{ hashFiles('tests/fixtures/interpreter/*.json', 'runtime/check_classes.py', 'runtime/check_functions.py') }} + restore-keys: oracle-v1-${{ runner.os }}-node-${{ matrix.shard }}- - run: uv run --no-sync python scripts/run_tests.py interpreter --shard ${{ matrix.shard }} --shards 4 - uses: actions/upload-artifact@v5 if: always() @@ -128,6 +133,12 @@ jobs: - run: uv run --no-sync python scripts/run_tests.py prepare-browser - if: matrix.suite == 'browser-integration' run: uv run --no-sync python scripts/prepare_runtime.py + - uses: actions/cache@v5 + if: matrix.suite == 'browser-conformance' + with: + path: build/reference-cache + key: oracle-v1-${{ runner.os }}-browser-${{ matrix.shard }}-${{ hashFiles('tests/fixtures/interpreter/*.json', 'runtime/check_classes.py', 'runtime/check_functions.py') }} + restore-keys: oracle-v1-${{ runner.os }}-browser-${{ matrix.shard }}- - run: uv run --no-sync python scripts/run_tests.py ${{ matrix.suite }} --shard ${{ matrix.shard }} --shards ${{ matrix.shards }} - uses: actions/upload-artifact@v5 if: always() diff --git a/docs/testing.md b/docs/testing.md index c6658c7..613a566 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -41,6 +41,15 @@ selected suites/features, command results and durations. Use `--report PATH` for a separate report location. Failures print the end of their log and return a nonzero exit status after the remaining selected checks finish. +Class-corpus differential checks reuse validated CPython reports under +`build/reference-cache` (override with `POCKET_PYTHON_REFERENCE_CACHE`). Cache +identity includes the complete corpus, Python build/host and both reference +drivers; failed or damaged entries are recomputed. Reports record cache hits. +CI restores each shard's cache across PR updates, including unchanged corpora +when another corpus changes. Standalone `--reference-only` checks always execute +CPython again, so nightly/manual/release validation independently verifies the +recorded expectations. Remove the directory to benchmark a cold run. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/runtime/check_classes.py b/runtime/check_classes.py index c423cde..b948e68 100644 --- a/runtime/check_classes.py +++ b/runtime/check_classes.py @@ -3,9 +3,11 @@ import argparse import hashlib import json +import os from pathlib import Path import platform import subprocess +import sys import tempfile from check_conformance import run_browser @@ -65,8 +67,52 @@ def references(corpus_path=CORPUS): for status in ['pass', 'fail']}} -def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, profile=None, parked=False): +def cached_references(corpus_path, cache_dir=None): + """Reuse only successful, intact reports for these exact oracle inputs.""" + cache_dir = Path(cache_dir or os.environ.get('POCKET_PYTHON_REFERENCE_CACHE', + ROOT.parent / 'build/reference-cache')) + identity = { + 'corpus': hashlib.sha256(corpus_path.read_bytes()).hexdigest(), + 'python': sys.version, 'implementation': platform.python_implementation(), + 'host': platform.platform(), + 'harness': {path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in (Path(__file__), ROOT / 'check_functions.py')}, + } + def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode('utf-8')).hexdigest() + + key = digest(identity) + path = cache_dir / f'{corpus_path.stem}-{key}.json' + try: + saved = json.loads(path.read_bytes()) + report = saved['report'] + if (saved['identity'] == identity and saved['report_sha256'] == digest(report) and + report['corpus_sha256'] == identity['corpus'] and + report['summary'] == {'pass': len(report['cases']), 'fail': 0} and + all(case['reference_status'] == 'pass' for case in report['cases'])): + return {**report, 'reference_cache': {'key': key, 'hit': True}} + except (FileNotFoundError, ValueError, KeyError, TypeError): + pass report = references(corpus_path) + if not report['summary']['fail']: + cache_dir.mkdir(parents=True, exist_ok=True) + # Atomic replacement also permits independent processes to share a cache. + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=cache_dir, + delete=False) as output: + json.dump({'identity': identity, 'report_sha256': digest(report), 'report': report}, output) + temporary = Path(output.name) + try: + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + for previous in cache_dir.glob(corpus_path.stem + '-*.json'): + if previous != path: + previous.unlink(missing_ok=True) + return {**report, 'reference_cache': {'key': key, 'hit': False}} + + +def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, profile=None, parked=False): + report = cached_references(corpus_path) if report['summary']['fail']: raise AssertionError('The class reference corpus differs from its recorded expectations.') corpus = json.loads(corpus_path.read_bytes()) @@ -130,6 +176,7 @@ def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, 'omitted_stages': sorted(available - {case['stage'] for case in cases}), 'omitted_profile_cases': [case['id'] for case in corpus['cases'] if selected_profile not in case.get('profiles', [corpus['profile']])], 'reference_summary': report['summary'], + 'reference_cache': report['reference_cache'], 'implementation_status': 'Implemented stages: ' + ', '.join(adopted) + '. Other recorded stages must refuse.', 'comparison': 'Adopted stdout and error types match CPython; messages/repr identity are not compared. Later stages must refuse explicitly.', 'inputs_sha256': {p.name: hashlib.sha256(p.read_bytes().replace(b'\r\n', b'\n')).hexdigest() diff --git a/tests/test_reference_diagnostics.py b/tests/test_reference_diagnostics.py index 1560208..62fe0ba 100644 --- a/tests/test_reference_diagnostics.py +++ b/tests/test_reference_diagnostics.py @@ -12,6 +12,63 @@ class ReferenceDiagnosticsTests(unittest.TestCase): + def test_cache_reuses_success_and_invalidates_corpus_python_and_harness(self): + corpus = {'profile': 'classes-source/1', 'cases': [ + {'id': 'one', 'stage': 'C1', 'source': 'print(7)', 'stdout': '7\n'}]} + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + path, cache = root / 'corpus.json', root / 'cache' + path.write_text(json.dumps(corpus), encoding='utf-8') + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': '7\n'}) as reference: + first = check_classes.cached_references(path, cache) + second = check_classes.cached_references(path, cache) + self.assertFalse(first['reference_cache']['hit']) + self.assertTrue(second['reference_cache']['hit']) + self.assertEqual(first['cases'], second['cases']) + self.assertEqual(reference.call_count, 1) + # Metadata and expectation changes also invalidate the whole corpus. + corpus['cases'][0]['stage'] = 'C2' + path.write_text(json.dumps(corpus), encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + with patch.object(check_classes.sys, 'version', sys.version.replace(sys.version.split()[0], '9.9.9', 1)): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + with patch.object(check_classes.platform, 'platform', return_value='another host'): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + original = Path.read_bytes + def changed_harness(file): + data = original(file) + return data + b'\n# changed reference' if file.name == 'check_functions.py' else data + with patch.object(Path, 'read_bytes', changed_harness): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + self.assertEqual(reference.call_count, 5) + self.assertEqual(len(list(cache.glob('*.json'))), 1) + + def test_damaged_cache_is_recomputed_and_failed_reports_are_not_saved(self): + corpus = {'profile': 'classes-source/1', 'cases': [ + {'id': 'one', 'stage': 'C1', 'source': 'print(7)', 'stdout': '7\n'}]} + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + path, cache = root / 'corpus.json', root / 'cache' + path.write_text(json.dumps(corpus), encoding='utf-8') + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': '7\n'}) as reference: + check_classes.cached_references(path, cache) + saved_path, = cache.glob('*.json') + for damaged in ('{truncated', 'null', '{}'): + saved_path.write_text(damaged, encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + saved = json.loads(saved_path.read_bytes()) + saved['report']['cases'][0]['reference']['stdout'] = 'tampered\n' + saved_path.write_text(json.dumps(saved), encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + self.assertEqual(reference.call_count, 5) + # Reference-only validation always executes the oracle afresh. + check_classes.references(path) + self.assertEqual(reference.call_count, 6) + saved_path.unlink() + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': 'wrong\n'}): + self.assertEqual(check_classes.cached_references(path, cache)['summary']['fail'], 1) + self.assertFalse(list(cache.iterdir())) + def test_failed_process_identifies_the_case_and_preserves_the_cause(self): corpus = {"profile": "classes-source/1", "cases": [{"id": "failing-input", "stage": "test", "source": "pass"}]} for failure, detail in [(subprocess.CalledProcessError(-11, ["reference"]), "exit -11"), From a53ed85c81bb14462bb8350694e135893b19e918 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:41:56 -0800 Subject: [PATCH 2/6] perf(CI-05): bound concurrent browser conformance workers --- docs/testing.md | 9 ++++ runtime/browser-pool.mjs | 50 ++++++++++++++++++++ runtime/browser-pool.test.mjs | 69 ++++++++++++++++++++++++++++ runtime/check_conformance.py | 31 ++++--------- scripts/benchmark_browser_workers.py | 60 ++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 21 deletions(-) create mode 100644 runtime/browser-pool.mjs create mode 100644 runtime/browser-pool.test.mjs create mode 100644 scripts/benchmark_browser_workers.py diff --git a/docs/testing.md b/docs/testing.md index 613a566..01006b2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -50,6 +50,15 @@ when another corpus changes. Standalone `--reference-only` checks always execute CPython again, so nightly/manual/release validation independently verifies the recorded expectations. Remove the directory to benchmark a cold run. +Browser differential requests use two concurrent fresh workers, with the same +10-second per-case deadline and results retained in input order. A failure +terminates the active pool before the browser closes. Set +`POCKET_PYTHON_BROWSER_WORKERS=1` for a serial comparison (accepted range: 1–8). +`scripts/benchmark_browser_workers.py --repeats 3` compares 1/2/4/8 workers on +the complete exceptions corpus and requires identical ordered results, including +resource/teardown observations. Use `--corpus PATH` for another corpus. It writes +timings and host/corpus identities to `build/browser-worker-benchmark.json`. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/runtime/browser-pool.mjs b/runtime/browser-pool.mjs new file mode 100644 index 0000000..8fdc11e --- /dev/null +++ b/runtime/browser-pool.mjs @@ -0,0 +1,50 @@ +// Test transport: each request owns a fresh worker, even when requests overlap. +export async function runWorkerRequests(workerPath, requests, { + concurrency = 2, timeout = 10000, WorkerClass = globalThis.Worker, +} = {}) { + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) { + throw new RangeError('Browser concurrency must be an integer from 1 to 8.'); + } + if (!Number.isFinite(timeout) || timeout <= 0) throw new RangeError('Invalid worker timeout.'); + const results = new Array(requests.length); + const active = new Set(); + let next = 0, failed = false; + const run = request => new Promise((resolve, reject) => { + let worker, timer; + const cleanup = () => { + clearTimeout(timer); + active.delete(cleanup); + if (worker) { + worker.onmessage = worker.onerror = worker.onmessageerror = null; + worker.terminate(); + } + }; + const fail = error => { + failed = true; + cleanup(); + for (const close of active) close(); + reject(new Error(`Conformance worker ${request.id}: ${error.message}`, {cause: error})); + }; + try { + worker = new WorkerClass(workerPath, {type: 'module'}); + active.add(cleanup); + timer = setTimeout(() => fail(new Error(`timed out after ${timeout}ms`)), timeout); + worker.onmessage = event => { + cleanup(); + resolve({...event.data, id: request.id}); + }; + worker.onerror = event => fail(new Error(event.message)); + worker.onmessageerror = () => fail(new Error('could not deserialize response')); + worker.postMessage(request); + } catch (error) { + fail(error); + } + }); + await Promise.all(Array.from({length: Math.min(concurrency, requests.length)}, async () => { + while (!failed && next < requests.length) { + const index = next++; + results[index] = await run(requests[index]); + } + })); + return results; +} diff --git a/runtime/browser-pool.test.mjs b/runtime/browser-pool.test.mjs new file mode 100644 index 0000000..4bdd286 --- /dev/null +++ b/runtime/browser-pool.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import {runWorkerRequests} from './browser-pool.mjs'; + +function workers() { + const instances = []; + let live = 0, peak = 0; + class FakeWorker { + constructor(path, options) { + assert.equal(path, '/fixture.mjs'); + assert.deepEqual(options, {type: 'module'}); + instances.push(this); + peak = Math.max(peak, ++live); + } + postMessage(request) { + this.request = request; + if (request.throw) throw new Error('post failed'); + if (request.hang) return; + this.timer = setTimeout(() => { + if (request.error) this.onerror?.({message: 'worker failed'}); + else if (request.messageerror) this.onmessageerror?.(); + else this.onmessage?.({data: {ok: true, value: request.id}}); + }, request.delay ?? 1); + } + terminate() { + assert.equal(this.closed, undefined, 'worker terminated once'); + this.closed = true; + clearTimeout(this.timer); + live--; + } + } + return {WorkerClass: FakeWorker, instances, live: () => live, peak: () => peak}; +} + +test('bounded fresh workers preserve input order despite out-of-order completion', async () => { + for (const concurrency of [1, 2, 4, 8]) { + const fixture = workers(); + const requests = Array.from({length: 12}, (_, id) => ({id, delay: id === 0 ? 20 : 1})); + const results = await runWorkerRequests('/fixture.mjs', requests, {...fixture, concurrency}); + assert.deepEqual(results, requests.map(({id}) => ({id, ok: true, value: id}))); + assert.equal(fixture.instances.length, requests.length); + assert.equal(fixture.peak(), concurrency); + assert.equal(fixture.live(), 0); + } +}); + +test('errors, serialization failures and per-case deadlines close all workers and stop scheduling', async () => { + for (const failure of [{error: true}, {messageerror: true}, {throw: true}, {hang: true}]) { + const fixture = workers(); + const requests = [{id: 'blocked', hang: true}, {id: 'fails', ...failure}, {id: 'never'}]; + await assert.rejects(runWorkerRequests('/fixture.mjs', requests, + {...fixture, concurrency: 2, timeout: 25}), /Conformance worker (fails|blocked):/); + assert.equal(fixture.live(), 0); + assert.equal(fixture.instances.length, 2); + } +}); + +test('worker construction failure is reported and invalid bounds are rejected', async () => { + await assert.rejects(runWorkerRequests('/fixture.mjs', [{id: 'one'}], { + WorkerClass: class {constructor() {throw new Error('construction failed');}}, + }), /one: construction failed/); + for (const concurrency of [0, 9, 1.5, NaN]) { + await assert.rejects(runWorkerRequests('', [], {concurrency}), RangeError); + } + for (const timeout of [0, -1, Infinity]) { + await assert.rejects(runWorkerRequests('', [], {timeout}), RangeError); + } + assert.deepEqual(await runWorkerRequests('', []), []); +}); diff --git a/runtime/check_conformance.py b/runtime/check_conformance.py index 3b26961..37ad37c 100644 --- a/runtime/check_conformance.py +++ b/runtime/check_conformance.py @@ -5,6 +5,7 @@ import hashlib from http.server import ThreadingHTTPServer import json +import os from pathlib import Path import platform import random @@ -54,10 +55,13 @@ def run_node(engine, requests): return response["host"], response["results"] -def run_browser(engine, requests, channel): +def run_browser(engine, requests, channel, concurrency=None): from browser_smoke import Handler from playwright.sync_api import sync_playwright + concurrency = int(os.environ.get('POCKET_PYTHON_BROWSER_WORKERS', '2')) if concurrency is None else concurrency + if not 1 <= concurrency <= 8: + raise ValueError('Browser concurrency must be between 1 and 8.') server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(ROOT))) thread = Thread(target=server.serve_forever, daemon=True) thread.start() @@ -68,26 +72,11 @@ def run_browser(engine, requests, channel): try: page = browser.new_page() page.goto(origin) - responses = page.evaluate("""async ({workerPath, requests}) => { - const results = []; - for (const request of requests) { - const result = await new Promise((resolve, reject) => { - const worker = new Worker(workerPath, {type: 'module'}); - const timer = setTimeout(() => { - worker.terminate(); reject(new Error('Conformance worker timed out.')); - }, 10000); - worker.onmessage = event => { - clearTimeout(timer); worker.terminate(); resolve(event.data); - }; - worker.onerror = event => { - clearTimeout(timer); worker.terminate(); reject(new Error(event.message)); - }; - worker.postMessage(request); - }); - results.push({id: request.id, ...result}); - } - return results; - }""", {"workerPath": "/" + engine["worker"], "requests": requests}) + responses = page.evaluate("""async ({workerPath, requests, concurrency}) => { + const {runWorkerRequests} = await import('/browser-pool.mjs'); + return runWorkerRequests(workerPath, requests, {concurrency}); + }""", {"workerPath": "/" + engine["worker"], "requests": requests, + "concurrency": concurrency}) return browser.version, responses finally: browser.close() diff --git a/scripts/benchmark_browser_workers.py b/scripts/benchmark_browser_workers.py new file mode 100644 index 0000000..d4d0679 --- /dev/null +++ b/scripts/benchmark_browser_workers.py @@ -0,0 +1,60 @@ +"""Compare isolated browser worker pool sizes on the same differential corpus.""" + +import argparse +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'runtime')) +from check_classes import check + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--corpus', type=Path, default=ROOT / 'tests/fixtures/interpreter/exceptions.json') + parser.add_argument('--repeats', type=int, default=3) + parser.add_argument('--channel') + parser.add_argument('--report', type=Path, default=ROOT / 'build/browser-worker-benchmark.json') + args = parser.parse_args() + if args.repeats < 1: + parser.error('--repeats must be positive') + baseline, measurements = None, [] + original = os.environ.get('POCKET_PYTHON_BROWSER_WORKERS') + try: + for concurrency in (1, 2, 4, 8): + durations = [] + os.environ['POCKET_PYTHON_BROWSER_WORKERS'] = str(concurrency) + for _ in range(args.repeats): + start = time.monotonic() + report = check('browser', args.channel, args.corpus) + durations.append(round(time.monotonic() - start, 3)) + if report['summary']['fail']: + raise AssertionError(report['summary']) + observed = [(case['id'], case['status'], case['actual']) for case in report['cases']] + if baseline is None: + baseline = observed + if observed != baseline: + raise AssertionError('Pool size changed the ordered differential results') + row = {'workers': concurrency, 'seconds': durations, + 'median_seconds': statistics.median(durations), 'cases': len(baseline)} + measurements.append(row) + print(json.dumps(row), flush=True) + finally: + if original is None: + os.environ.pop('POCKET_PYTHON_BROWSER_WORKERS', None) + else: + os.environ['POCKET_PYTHON_BROWSER_WORKERS'] = original + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps({'schema_version': 1, 'python': platform.python_version(), + 'host': platform.platform(), 'browser': report['host_version'], + 'corpus_sha256': report['corpus_sha256'], 'repeats': args.repeats, + 'measurements': measurements}, indent=2) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + main() From 44417a117246464c8b3f0f3024a0b41b2a3b8ee9 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:49:09 -0800 Subject: [PATCH 3/6] perf(CI-06): batch bulk console language acceptance --- docs/testing.md | 9 +++++ scripts/classes_console_browser_smoke.py | 51 +++++++++++++++++++++--- scripts/console_cases.py | 30 ++++++++++++++ tests/test_console_cases.py | 40 +++++++++++++++++++ 4 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 scripts/console_cases.py create mode 100644 tests/test_console_cases.py diff --git a/docs/testing.md b/docs/testing.md index 01006b2..31caeaa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -59,6 +59,15 @@ the complete exceptions corpus and requires identical ordered results, including resource/teardown observations. Use `--corpus PATH` for another corpus. It writes timings and host/corpus identities to `build/browser-worker-benchmark.json`. +The classes console compares its bulk successful language programs through two +installed console host/worker sessions. Each run still creates its own module +session with the shipped profile, libraries, output grant and resource limits. +Corpus stdout comes from the same committed expectations checked independently +by the differential matrix; standalone examples still run isolated CPython. +Representative form submissions, all source-error/diagnostic checks, offline +reloads, service-worker/profile updates and every Stop/Restart cycle retain +their UI paths. The console report records UI/worker counts and elapsed time. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/scripts/classes_console_browser_smoke.py b/scripts/classes_console_browser_smoke.py index 31149e2..92d50b2 100644 --- a/scripts/classes_console_browser_smoke.py +++ b/scripts/classes_console_browser_smoke.py @@ -15,6 +15,7 @@ from playwright.sync_api import sync_playwright from interpreter_browser_smoke import EXAMPLE, Handler, Server +from console_cases import program_expectations from pocket_python.interpreter import build_interpreter, read_interpreter ROOT = Path(__file__).resolve().parents[1] @@ -109,6 +110,8 @@ def __getattribute__(self,n): def smoke(channel=None): checks, cases = [], [] + started = time.perf_counter() + transport = {'ui': 0, 'worker': 0} with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) source = EXAMPLE.write_source(root / 'source', source=SOURCE) @@ -140,6 +143,7 @@ def ready(): page.wait_for_function('()=>navigator.serviceWorker.controller!==null') def run(source, expected): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() # Repeated corpus runs need the normal click handler, @@ -150,6 +154,7 @@ def run(source, expected): assert page.locator('#result').inner_text() == expected def refused(source, prefix): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() page.locator('#run').dispatch_event('click') @@ -365,10 +370,45 @@ def refused(source, prefix): programs.append(PROPERTY_NAMES_SOURCE) programs.append(ABSTRACT_FLAGS_SOURCE) programs.append(DOCUMENTATION_SOURCE) - for program in programs: - expected = subprocess.check_output([sys.executable, '-I', '-S', '-X', 'utf8', '-c', program], - text=True, encoding='utf-8', timeout=5) - run(program, expected) + expectations = program_expectations(programs) + # Keep a representative real form submission before the bulk + # comparisons; error, offline/update and Stop/Restart checks + # below continue through their existing UI paths. + run(programs[0], expectations[0]) + config = json.loads((root / 'site/nested/classes' / selected['configuration']).read_bytes()) + config['library_sources'] = [{**{key: module[key] for key in ('name', 'package', 'imports')}, + **{key: (root / 'site/nested/classes' / module[key]['path']).read_text(encoding='utf-8') + for key in ('source', 'notice')}} for module in config['libraries']] + results = page.evaluate('''async ({host,config,programs})=>{ + const {mountInterpreter}=await import('./'+host); + const results=new Array(programs.length),active=new Set();let next=0,failed=false; + try{ + await Promise.all(Array.from({length:Math.min(2,programs.length)},async()=>{ + let output=''; + const app=mountInterpreter({application:config.application,capabilities:config.capabilities, + limits:config.limits,libraryProfile:config.library_profile,libraries:config.library_sources, + hostProfile:config.host_profile,onOutput:text=>{output+=text;}}); + active.add(app); + try{ + await app.ready; + while(!failed&&next Date: Mon, 2 Nov 2026 01:52:51 -0800 Subject: [PATCH 4/6] perf(CI-07): prefer measured validation shard costs --- docs/testing.md | 21 ++++++++- scripts/run_tests.py | 4 ++ scripts/test_manifest.py | 20 +++++++-- scripts/update_test_costs.py | 68 +++++++++++++++++++++++++++++ tests/test_test_costs.py | 85 ++++++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 scripts/update_test_costs.py create mode 100644 tests/test_test_costs.py diff --git a/docs/testing.md b/docs/testing.md index 31caeaa..430237c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -13,7 +13,7 @@ uv run --no-sync python scripts/run_tests.py focused --feature native-descriptor # The same suites and shard assignments used in CI. uv run --no-sync python scripts/run_tests.py interpreter --shard 1 --shards 4 uv run --no-sync python scripts/run_tests.py prepare-browser -uv run --no-sync python scripts/run_tests.py browser-conformance --shard 1 --shards 4 +uv run --no-sync python scripts/run_tests.py browser-conformance --shard 1 --shards 6 uv run --no-sync python scripts/run_tests.py browser-integration --shard 1 --shards 3 uv run --no-sync python scripts/run_tests.py python ``` @@ -68,6 +68,25 @@ Representative form submissions, all source-error/diagnostic checks, offline reloads, service-worker/profile updates and every Stop/Restart cycle retain their UI paths. The console report records UI/worker counts and elapsed time. +Shard weights prefer the checked-in `scripts/test_costs.json` inventory of +successful command timings. New checks retain the manifest's fallback estimate; +costs affect assignments only, never selection or preparation/packaging order. +To refresh from a passing Linux CI run, download its `interpreter-*`, +`browser-conformance-*` and `browser-integration-*` artifacts under one directory, +then run: + +```sh +uv run --no-sync python scripts/update_test_costs.py build/ci-timings \ + --source https://github.com/outside-labs/pocket-python/actions/runs/RUN_ID +``` + +The importer records report hashes/revisions, ignores removed IDs, deduplicates +reports and takes the median for repeated samples. Failed, incomplete or invalid +timings are rejected. Review the generated inventory before committing it; do +not mix different platforms or worker configurations. Runner reports now record +the host and worker configuration. The inventory is a scheduling estimate, +never a performance assertion or a substitute for running the complete gate. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 7295bd0..12ec6f6 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path +import platform import shlex import subprocess import sys @@ -53,6 +54,9 @@ def run(selected, report_path, metadata): report = { 'schema_version': 1, 'revision': git_output('rev-parse', 'HEAD'), 'working_tree_modified': bool(git_output('status', '--porcelain')), + 'host': {'os': platform.system(), 'machine': platform.machine(), + 'python': platform.python_version(), + 'browser_workers': env.get('POCKET_PYTHON_BROWSER_WORKERS', '2')}, 'started_at': datetime.now(UTC).isoformat(), **metadata, 'checks': [], } try: diff --git a/scripts/test_manifest.py b/scripts/test_manifest.py index 06cb5de..9eccf39 100644 --- a/scripts/test_manifest.py +++ b/scripts/test_manifest.py @@ -1,7 +1,8 @@ """Shared validation inventory; commands run from the repository root.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace import json +import math from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -124,7 +125,19 @@ class Check: 'sequences': ('Q1', 'Q2', 'Q3', 'Hints')} -def checks(): +def measured_costs(path): + if not path.exists(): + return {} + inventory = json.loads(path.read_bytes()) + costs = inventory['seconds'] + if (inventory['schema_version'] != 1 or not isinstance(costs, dict) or + any(isinstance(value, bool) or not isinstance(value, (int, float)) or + not math.isfinite(value) or value <= 0 for value in costs.values())): + raise ValueError('Invalid measured validation costs') + return costs + + +def checks(costs_path=None): """Expand profile/driver matrices once for local and CI callers.""" result = [] @@ -216,4 +229,5 @@ def add(id, suite, *command, seconds=1): add('smoke-build-wheel', 'smoke', 'uv', 'build', '--wheel') add('smoke-installed-cli', 'smoke', 'python', 'scripts/smoke_install.py') add('smoke-interpreter-startup', 'smoke', 'node', 'scripts/startup_smoke.mjs') - return result + costs = measured_costs(costs_path or ROOT / 'scripts/test_costs.json') + return [replace(check, seconds=costs.get(check.id, check.seconds)) for check in result] diff --git a/scripts/update_test_costs.py b/scripts/update_test_costs.py new file mode 100644 index 0000000..24aead6 --- /dev/null +++ b/scripts/update_test_costs.py @@ -0,0 +1,68 @@ +"""Build deterministic shard costs from successful validation runner reports.""" + +import argparse +from collections import defaultdict +import hashlib +import json +import math +from pathlib import Path +import statistics +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.test_manifest import checks + + +def build_inventory(paths, source, known_ids=None): + known_ids = {check.id for check in checks()} if known_ids is None else set(known_ids) + files = sorted({file for path in paths for file in + (path.rglob('*-of-*.json') if path.is_dir() else [path])}) + samples, sources, seen = defaultdict(list), [], set() + for path in files: + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest in seen: + continue + seen.add(digest) + report = json.loads(raw) + rows, summary = report['checks'], report['summary'] + if (report['schema_version'] != 1 or summary['fail'] != 0 or + summary['pass'] != len(rows) or summary['selected'] != len(rows) or not rows): + raise ValueError(f'{path}: requires a complete successful runner report') + ids = set() + for row in rows: + seconds = row['seconds'] + if (row['returncode'] != 0 or row['id'] in ids or + isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or + not math.isfinite(seconds) or seconds < 0): + raise ValueError(f'{path}: invalid or failed timing sample') + ids.add(row['id']) + if row['id'] in known_ids: + samples[row['id']].append(max(0.001, seconds)) + sources.append({'sha256': digest, 'revision': report['revision'], + **{key: report[key] for key in ('suites', 'shard', 'shards', 'host') if key in report}}) + if not samples: + raise ValueError('No timings match the current validation inventory') + return {'schema_version': 1, 'source': source, 'reports': sources, + 'seconds': {id: round(statistics.median(values), 3) for id, values in sorted(samples.items())}} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('reports', nargs='+', type=Path, help='Runner JSON files or artifact directories') + parser.add_argument('--source', required=True, help='Workflow run URL or measurement provenance') + parser.add_argument('--output', type=Path, default=ROOT / 'scripts/test_costs.json') + args = parser.parse_args(argv) + try: + inventory = build_inventory(args.reports, args.source) + except (OSError, ValueError, KeyError, TypeError) as error: + parser.error(str(error)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(inventory, indent=2) + '\n', encoding='utf-8') + print(f'{len(inventory["seconds"])} measured costs from {len(inventory["reports"])} reports: {args.output}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_test_costs.py b/tests/test_test_costs.py new file mode 100644 index 0000000..e201d98 --- /dev/null +++ b/tests/test_test_costs.py @@ -0,0 +1,85 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.update_test_costs import build_inventory +from scripts.test_manifest import checks, measured_costs +from scripts.run_tests import shards + + +class TestCostInventoryTests(unittest.TestCase): + def test_measured_costs_control_balancing_without_changing_coverage_or_order(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'costs.json' + fallback = checks(path) + selected = [check for check in fallback if check.suite == 'browser-conformance'] + expensive = selected[0].id + path.write_text(json.dumps({'schema_version': 1, 'seconds': {expensive: 10000}}), encoding='utf-8') + measured = checks(path) + self.assertEqual([check.id for check in fallback], [check.id for check in measured]) + self.assertEqual([check.command for check in fallback], [check.command for check in measured]) + self.assertEqual(measured[0].seconds, fallback[0].seconds) + selected = [check for check in measured if check.suite == 'browser-conformance'] + self.assertEqual(selected[0].seconds, 10000) + groups = shards(selected, 6) + self.assertEqual([check.id for check in groups[0]], [expensive]) + self.assertCountEqual([check.id for group in groups for check in group], [check.id for check in selected]) + self.assertEqual(groups, shards(selected, 6)) + for invalid in (0, -1, float('nan'), float('inf'), True, 'slow'): + path.write_text(json.dumps({'schema_version': 1, 'seconds': {'known': invalid}}), encoding='utf-8') + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + measured_costs(path) + + def report(self, seconds=10): + return {'schema_version': 1, 'revision': 'a' * 40, 'suites': ['browser-conformance'], + 'shard': 1, 'shards': 6, + 'checks': [{'id': 'known', 'seconds': seconds, 'returncode': 0}], + 'summary': {'pass': 1, 'fail': 0, 'selected': 1}} + + def test_medians_provenance_deduplication_and_stale_ids(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for name, seconds in [('one', 10), ('two', 20), ('duplicate', 20), ('three', 30)]: + (root / f'{name}-1-of-1.json').write_text(json.dumps(self.report(seconds)), encoding='utf-8') + stale = self.report(1000) + stale['checks'][0]['id'] = 'removed' + (root / 'stale-1-of-1.json').write_text(json.dumps(stale), encoding='utf-8') + observed = build_inventory([root], 'workflow-url', ['known']) + self.assertEqual(observed['seconds'], {'known': 20}) + self.assertEqual(len(observed['reports']), 4) + self.assertEqual(observed['source'], 'workflow-url') + self.assertEqual(observed, build_inventory([root, root], 'workflow-url', ['known'])) + self.assertTrue(all(row['revision'] == 'a' * 40 for row in observed['reports'])) + + def test_incomplete_failed_duplicate_and_invalid_samples_are_rejected(self): + reports = [] + for seconds in (-1, float('nan'), float('inf'), True, '10'): + reports.append(self.report(seconds)) + failed = self.report() + failed['checks'][0]['returncode'] = 1 + reports.append(failed) + partial = self.report() + partial['summary']['selected'] = 2 + reports.append(partial) + failed_summary = self.report() + failed_summary['summary']['fail'] = 1 + reports.append(failed_summary) + duplicate = self.report() + duplicate['checks'] *= 2 + duplicate['summary']['pass'] = duplicate['summary']['selected'] = 2 + reports.append(duplicate) + for report in reports: + with self.subTest(report=report), tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'report.json' + path.write_text(json.dumps(report), encoding='utf-8') + with self.assertRaises(ValueError): + build_inventory([path], 'workflow-url', ['known']) + + def test_zero_duration_is_positive_and_unknown_inventory_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'report.json' + path.write_text(json.dumps(self.report(0)), encoding='utf-8') + self.assertEqual(build_inventory([path], 'local', ['known'])['seconds'], {'known': 0.001}) + with self.assertRaisesRegex(ValueError, 'No timings'): + build_inventory([path], 'local', ['new']) From e0e5a648ffc78c28facde7d4d25ba92d2dc1c67f Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:59:38 -0800 Subject: [PATCH 5/6] perf(CI-07): record optimized Linux command timings --- scripts/test_costs.json | 878 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 878 insertions(+) create mode 100644 scripts/test_costs.json diff --git a/scripts/test_costs.json b/scripts/test_costs.json new file mode 100644 index 0000000..0d29cc5 --- /dev/null +++ b/scripts/test_costs.json @@ -0,0 +1,878 @@ +{ + "schema_version": 1, + "source": "Optimized Linux CI: https://github.com/outside-labs/pocket-python/actions/runs/41423024431 (interpreter/conformance); https://github.com/outside-labs/pocket-python/actions/runs/41424015512 (integration)", + "reports": [ + { + "sha256": "cbbf03f727700cb44ecc1cfd9cd3c764b3a678fe1cff91b9afde6d4077e840df", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 1, + "shards": 6 + }, + { + "sha256": "0dc1c27d05fd0a5f0f5e5d86026b01838080a4033b3532d8fe096915690f1383", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "f1a07a947b79b9efac7c1f96f8450e2e46b2a375080c4329f81f47d0595d02a2", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 2, + "shards": 6 + }, + { + "sha256": "13e9013fdc3a70a47a4cf960e14a76ddf9b52f157da086417b2a6dedbbe6e716", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "5112a9b6b9730da9ea2efd0841b3f921376b798fb17cfe3a65741684732754c0", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 3, + "shards": 6 + }, + { + "sha256": "99dbc47880a224a07102d54f0e49cd3bd1261deac81f0ab125195c12961c753c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "1893135d935e8cc692e777126af6a2e0eb7f6d913de636a0658cf6b444981d5c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 4, + "shards": 6 + }, + { + "sha256": "fa3034a607d064679f837444b276b057c1a7fb88eeaa965655ae81ed49cb6a35", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "89980e5b934c7dc343c1a885b78a769bf35783c4111762cad99116dbc647dca0", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 5, + "shards": 6 + }, + { + "sha256": "e948d09ec207018dbac7d643dd4612db6501da90221f605853a3191ab3fae14c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "39d6aaa0369f3530535bc78f0228badec0434b2b829e65dacf1e102f63a68de1", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 6, + "shards": 6 + }, + { + "sha256": "3d0a112daa3d36784739bb95fe0385b524b2d05562cf05b6c19b87de9f03037e", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "dabed0579d1e72d2a181fe7eb7103428c3043023c8e7284761ec8023ec9ab5a2", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 1, + "shards": 4 + }, + { + "sha256": "c5fbce181ca3e6181050fe3ca302b1be5c7f7126f169de6031dabb1d99116c76", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 2, + "shards": 4 + }, + { + "sha256": "4f5609d3425b49693f498a196952072a1ff76a16b477a25eddad23500db3aadf", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 3, + "shards": 4 + }, + { + "sha256": "38f43ec0aae2c780d08c8c025c78567e95f8772164f23cf3ac2df3a5b11124ca", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 4, + "shards": 4 + }, + { + "sha256": "e8a60a0321d67436cc308eef8de144e48da20b81119af1fe8f84e6d54f215d7c", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 1, + "shards": 3 + }, + { + "sha256": "04bda8dbc8a9f45a57fcda2f206d4611409e907e0041df02faa947a955195e6b", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "75c74ac8510ba8b8b43b23654b6bc0617dcc6d175e34db844b3b8fb66ab27b79", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 2, + "shards": 3 + }, + { + "sha256": "9d4ec95ba7cd3f817a24d6fb95586e0dc09b2c628c017e211b96cc5b28d79d37", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "7276463aacf44d0ef8f7a958010d7b244aec39bfe39b758c70b316e0e88f88d8", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 3, + "shards": 3 + }, + { + "sha256": "c7b9ff619a502cb94825e38bfdd78c2228c9156437b7f2bc23952180068a99a9", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + } + ], + "seconds": { + "abstract-flags-classes-drain-browser": 11.777, + "abstract-flags-classes-drain-node": 3.525, + "abstract-flags-classes-parked-browser": 12.125, + "abstract-flags-classes-parked-node": 0.925, + "abstract-flags-classes-stepped-browser": 11.501, + "abstract-flags-classes-stepped-node": 5.411, + "advertised-directories-classes-drain-browser": 7.934, + "advertised-directories-classes-drain-node": 3.916, + "advertised-directories-classes-parked-browser": 8.385, + "advertised-directories-classes-parked-node": 2.12, + "advertised-directories-classes-stepped-browser": 5.409, + "advertised-directories-classes-stepped-node": 4.403, + "arithmetic-dispatch-classes-drain-browser": 6.544, + "arithmetic-dispatch-classes-drain-node": 2.78, + "arithmetic-dispatch-classes-parked-browser": 4.233, + "arithmetic-dispatch-classes-parked-node": 1.949, + "arithmetic-dispatch-classes-stepped-browser": 4.741, + "arithmetic-dispatch-classes-stepped-node": 3.051, + "attribute-hooks-classes-drain-browser": 10.195, + "attribute-hooks-classes-drain-node": 4.412, + "attribute-hooks-classes-stepped-browser": 10.477, + "attribute-hooks-classes-stepped-node": 4.736, + "browser-worker-startup": 1.09, + "buffer-directories-classes-drain-browser": 12.834, + "buffer-directories-classes-drain-node": 6.411, + "buffer-directories-classes-parked-browser": 12.7, + "buffer-directories-classes-parked-node": 4.287, + "buffer-directories-classes-stepped-browser": 12.995, + "buffer-directories-classes-stepped-node": 6.564, + "callable-type-directories-classes-drain-browser": 6.857, + "callable-type-directories-classes-drain-node": 4.959, + "callable-type-directories-classes-parked-browser": 10.624, + "callable-type-directories-classes-parked-node": 2.234, + "callable-type-directories-classes-stepped-browser": 10.551, + "callable-type-directories-classes-stepped-node": 5.107, + "callbacks-browser": 1.772, + "callbacks-node": 0.841, + "class-body-namespaces-classes-drain-browser": 7.125, + "class-body-namespaces-classes-drain-node": 3.187, + "class-body-namespaces-classes-parked-browser": 7.448, + "class-body-namespaces-classes-parked-node": 3.268, + "class-body-namespaces-classes-stepped-browser": 7.693, + "class-body-namespaces-classes-stepped-node": 2.169, + "class-directories-classes-drain-browser": 5.718, + "class-directories-classes-drain-node": 4.555, + "class-directories-classes-parked-browser": 9.44, + "class-directories-classes-parked-node": 4.789, + "class-directories-classes-stepped-browser": 9.301, + "class-directories-classes-stepped-node": 4.787, + "class-mapping-proxies-classes-drain-browser": 21.869, + "class-mapping-proxies-classes-drain-node": 20.472, + "class-mapping-proxies-classes-parked-browser": 18.744, + "class-mapping-proxies-classes-parked-node": 16.8, + "class-mapping-proxies-classes-stepped-browser": 38.926, + "class-mapping-proxies-classes-stepped-node": 23.449, + "class-namespaces-classes-drain-browser": 3.5, + "class-namespaces-classes-drain-node": 2.443, + "class-namespaces-classes-parked-browser": 5.278, + "class-namespaces-classes-parked-node": 1.719, + "class-namespaces-classes-stepped-browser": 5.334, + "class-namespaces-classes-stepped-node": 2.447, + "classes-classes-drain-browser": 3.377, + "classes-classes-drain-node": 2.049, + "classes-classes-stepped-browser": 5.129, + "classes-classes-stepped-node": 2.122, + "classes-console-browser": 312.385, + "code-cells-classes-drain-browser": 3.058, + "code-cells-classes-drain-node": 1.354, + "code-cells-classes-parked-browser": 3.628, + "code-cells-classes-parked-node": 0.919, + "code-cells-classes-stepped-browser": 3.091, + "code-cells-classes-stepped-node": 1.371, + "compatibility-browser": 82.418, + "compatibility-node": 38.686, + "compiled-browser": 4.683, + "construction-classes-drain-browser": 12.434, + "construction-classes-drain-node": 3.9, + "construction-classes-stepped-browser": 12.739, + "construction-classes-stepped-node": 5.65, + "container-directories-classes-drain-browser": 9.991, + "container-directories-classes-drain-node": 3.262, + "container-directories-classes-parked-browser": 6.433, + "container-directories-classes-parked-node": 5.015, + "container-directories-classes-stepped-browser": 10.204, + "container-directories-classes-stepped-node": 4.808, + "container-type-directories-classes-drain-browser": 9.486, + "container-type-directories-classes-drain-node": 8.15, + "container-type-directories-classes-parked-browser": 15.663, + "container-type-directories-classes-parked-node": 8.468, + "container-type-directories-classes-stepped-browser": 9.516, + "container-type-directories-classes-stepped-node": 8.116, + "containers-browser": 4.447, + "custom-mro-classes-drain-browser": 4.996, + "custom-mro-classes-drain-node": 2.053, + "custom-mro-classes-parked-browser": 3.296, + "custom-mro-classes-parked-node": 1.03, + "custom-mro-classes-stepped-browser": 4.856, + "custom-mro-classes-stepped-node": 2.246, + "descriptor-binding-classes-drain-browser": 5.214, + "descriptor-binding-classes-drain-node": 3.499, + "descriptor-binding-classes-parked-browser": 7.931, + "descriptor-binding-classes-parked-node": 3.568, + "descriptor-binding-classes-stepped-browser": 7.91, + "descriptor-binding-classes-stepped-node": 2.428, + "descriptors-classes-drain-browser": 8.215, + "descriptors-classes-drain-node": 2.399, + "descriptors-classes-stepped-browser": 8.32, + "descriptors-classes-stepped-node": 3.653, + "documentation-classes-drain-browser": 11.338, + "documentation-classes-drain-node": 5.266, + "documentation-classes-parked-browser": 7.715, + "documentation-classes-parked-node": 5.197, + "documentation-classes-stepped-browser": 11.998, + "documentation-classes-stepped-node": 3.671, + "dynamic-compile-classes-drain-browser": 3.543, + "dynamic-compile-classes-drain-node": 2.004, + "dynamic-compile-classes-parked-browser": 5.361, + "dynamic-compile-classes-parked-node": 2.079, + "dynamic-compile-classes-stepped-browser": 3.521, + "dynamic-compile-classes-stepped-node": 2.147, + "dynamic-eval-classes-drain-browser": 9.151, + "dynamic-eval-classes-drain-node": 5.419, + "dynamic-eval-classes-parked-browser": 9.542, + "dynamic-eval-classes-parked-node": 5.45, + "dynamic-eval-classes-stepped-browser": 9.479, + "dynamic-eval-classes-stepped-node": 3.409, + "dynamic-exec-classes-drain-browser": 6.844, + "dynamic-exec-classes-drain-node": 4.117, + "dynamic-exec-classes-parked-browser": 5.499, + "dynamic-exec-classes-parked-node": 6.642, + "dynamic-exec-classes-stepped-browser": 10.629, + "dynamic-exec-classes-stepped-node": 6.541, + "dynamic-imports-classes-drain-browser": 5.148, + "dynamic-imports-classes-drain-node": 3.819, + "dynamic-imports-classes-parked-browser": 8.145, + "dynamic-imports-classes-parked-node": 3.937, + "dynamic-imports-classes-stepped-browser": 4.985, + "dynamic-imports-classes-stepped-node": 3.871, + "equality-classes-drain-browser": 10.936, + "equality-classes-drain-node": 5.051, + "equality-classes-stepped-browser": 10.824, + "equality-classes-stepped-node": 5.056, + "exceptions-browser": 59.96, + "exceptions-classes-drain-browser": 218.763, + "exceptions-classes-drain-node": 94.159, + "exceptions-classes-parked-browser": 225.899, + "exceptions-classes-parked-node": 102.897, + "exceptions-classes-stepped-browser": 219.495, + "exceptions-classes-stepped-node": 64.991, + "exceptions-core-drain-browser": 54.459, + "exceptions-core-drain-node": 63.221, + "exceptions-core-parked-browser": 82.899, + "exceptions-core-parked-node": 1.357, + "exceptions-core-stepped-browser": 53.236, + "exceptions-core-stepped-node": 2.144, + "execution-classes-drain-browser": 1.999, + "execution-classes-drain-node": 0.768, + "execution-classes-parked-browser": 1.145, + "execution-classes-parked-node": 0.845, + "execution-classes-stepped-browser": 1.17, + "execution-classes-stepped-node": 0.528, + "expanded-class-bases-classes-drain-browser": 4.407, + "expanded-class-bases-classes-drain-node": 1.169, + "expanded-class-bases-classes-parked-browser": 4.358, + "expanded-class-bases-classes-parked-node": 1.837, + "expanded-class-bases-classes-stepped-browser": 4.29, + "expanded-class-bases-classes-stepped-node": 1.845, + "formatting-classes-drain-browser": 8.325, + "formatting-classes-drain-node": 3.175, + "formatting-classes-stepped-browser": 7.058, + "formatting-classes-stepped-node": 3.223, + "function-binding-classes-drain-browser": 7.181, + "function-binding-classes-drain-node": 5.082, + "function-binding-classes-stepped-browser": 10.857, + "function-binding-classes-stepped-node": 5.168, + "function-binding-core-drain-browser": 8.954, + "function-binding-core-drain-node": 0.504, + "function-binding-core-stepped-browser": 8.212, + "function-binding-core-stepped-node": 3.167, + "function-dictionaries-classes-drain-browser": 9.243, + "function-dictionaries-classes-drain-node": 3.995, + "function-dictionaries-classes-parked-browser": 5.974, + "function-dictionaries-classes-parked-node": 4.009, + "function-dictionaries-classes-stepped-browser": 9.139, + "function-dictionaries-classes-stepped-node": 2.793, + "function-directories-classes-drain-browser": 3.965, + "function-directories-classes-drain-node": 2.114, + "function-directories-classes-parked-browser": 6.242, + "function-directories-classes-parked-node": 3.466, + "function-directories-classes-stepped-browser": 6.303, + "function-directories-classes-stepped-node": 3.474, + "function-metadata-classes-drain-browser": 8.811, + "function-metadata-classes-drain-node": 3.828, + "function-metadata-classes-parked-browser": 6.022, + "function-metadata-classes-parked-node": 4.049, + "function-metadata-classes-stepped-browser": 9.215, + "function-metadata-classes-stepped-node": 4.052, + "function-state-classes-drain-browser": 3.756, + "function-state-classes-drain-node": 1.549, + "function-state-classes-parked-browser": 2.643, + "function-state-classes-parked-node": 0.706, + "function-state-classes-stepped-browser": 2.605, + "function-state-classes-stepped-node": 1.629, + "functions-browser": 6.044, + "functions-node": 1.752, + "generate-identifiers": 0.212, + "generate-native-namespaces": 0.04, + "generate-printability": 0.117, + "generator-directories-classes-drain-browser": 5.228, + "generator-directories-classes-drain-node": 2.323, + "generator-directories-classes-parked-browser": 5.114, + "generator-directories-classes-parked-node": 2.293, + "generator-directories-classes-stepped-browser": 5.212, + "generator-directories-classes-stepped-node": 1.625, + "generator-frames-classes-drain-browser": 4.242, + "generator-frames-classes-drain-node": 2.028, + "generator-frames-classes-parked-browser": 4.287, + "generator-frames-classes-parked-node": 3.171, + "generator-frames-classes-stepped-browser": 6.583, + "generator-frames-classes-stepped-node": 3.116, + "generators-classes-drain-browser": 4.594, + "generators-classes-drain-node": 1.337, + "generators-classes-stepped-browser": 4.501, + "generators-classes-stepped-node": 1.924, + "generators-core-drain-browser": 2.875, + "generators-core-drain-node": 1.846, + "generators-core-stepped-browser": 2.836, + "generators-core-stepped-node": 1.86, + "immutable-subclasses-classes-drain-browser": 11.302, + "immutable-subclasses-classes-drain-node": 6.324, + "immutable-subclasses-classes-parked-browser": 7.792, + "immutable-subclasses-classes-parked-node": 4.113, + "immutable-subclasses-classes-stepped-browser": 12.019, + "immutable-subclasses-classes-stepped-node": 6.33, + "indexing-classes-drain-browser": 10.119, + "indexing-classes-drain-node": 4.59, + "indexing-classes-stepped-browser": 6.728, + "indexing-classes-stepped-node": 2.947, + "instance-dictionaries-classes-drain-browser": 8.14, + "instance-dictionaries-classes-drain-node": 2.46, + "instance-dictionaries-classes-parked-browser": 5.599, + "instance-dictionaries-classes-parked-node": 3.637, + "instance-dictionaries-classes-stepped-browser": 5.555, + "instance-dictionaries-classes-stepped-node": 3.562, + "instance-directories-classes-drain-browser": 8.502, + "instance-directories-classes-drain-node": 2.71, + "instance-directories-classes-parked-browser": 8.318, + "instance-directories-classes-parked-node": 4.244, + "instance-directories-classes-stepped-browser": 5.374, + "instance-directories-classes-stepped-node": 4.156, + "integer-identity-classes-drain-browser": 3.544, + "integer-identity-classes-drain-node": 1.533, + "integer-identity-classes-parked-browser": 2.582, + "integer-identity-classes-parked-node": 1.723, + "integer-identity-classes-stepped-browser": 3.584, + "integer-identity-classes-stepped-node": 1.133, + "interpreter-browser": 0.924, + "interpreter-console-browser": 7.914, + "interpreter-node": 0.182, + "iteration-classes-drain-browser": 10.998, + "iteration-classes-drain-node": 4.768, + "iteration-classes-stepped-browser": 10.857, + "iteration-classes-stepped-node": 3.437, + "iterator-directories-classes-drain-browser": 10.412, + "iterator-directories-classes-drain-node": 3.075, + "iterator-directories-classes-parked-browser": 6.521, + "iterator-directories-classes-parked-node": 0.972, + "iterator-directories-classes-stepped-browser": 6.571, + "iterator-directories-classes-stepped-node": 4.695, + "iterator-type-directories-classes-drain-browser": 15.078, + "iterator-type-directories-classes-drain-node": 7.039, + "iterator-type-directories-classes-parked-browser": 9.914, + "iterator-type-directories-classes-parked-node": 7.129, + "iterator-type-directories-classes-stepped-browser": 9.773, + "iterator-type-directories-classes-stepped-node": 7.238, + "length-truth-classes-drain-browser": 6.7, + "length-truth-classes-drain-node": 4.32, + "length-truth-classes-stepped-browser": 10.649, + "length-truth-classes-stepped-node": 4.561, + "library-console-browser": 4.187, + "library-profiles-browser": 2.454, + "library-profiles-node": 0.801, + "memory-browser": 1.142, + "memory-node": 0.188, + "metaclass-attributes-classes-drain-browser": 11.373, + "metaclass-attributes-classes-drain-node": 3.558, + "metaclass-attributes-classes-parked-browser": 7.376, + "metaclass-attributes-classes-parked-node": 4.957, + "metaclass-attributes-classes-stepped-browser": 7.4, + "metaclass-attributes-classes-stepped-node": 5.066, + "metaclass-factories-classes-drain-browser": 3.711, + "metaclass-factories-classes-drain-node": 0.973, + "metaclass-factories-classes-parked-browser": 2.63, + "metaclass-factories-classes-parked-node": 1.585, + "metaclass-factories-classes-stepped-browser": 3.731, + "metaclass-factories-classes-stepped-node": 1.505, + "metaclass-protocols-classes-drain-browser": 11.649, + "metaclass-protocols-classes-drain-node": 5.097, + "metaclass-protocols-classes-parked-browser": 11.311, + "metaclass-protocols-classes-parked-node": 5.036, + "metaclass-protocols-classes-stepped-browser": 12.182, + "metaclass-protocols-classes-stepped-node": 5.099, + "method-directories-classes-drain-browser": 4.597, + "method-directories-classes-drain-node": 2.268, + "method-directories-classes-parked-browser": 7.023, + "method-directories-classes-parked-node": 3.628, + "method-directories-classes-stepped-browser": 3.483, + "method-directories-classes-stepped-node": 3.517, + "micropython-browser": 1.797, + "module-namespaces-classes-drain-browser": 4.395, + "module-namespaces-classes-drain-node": 3.11, + "module-namespaces-classes-parked-browser": 6.987, + "module-namespaces-classes-parked-node": 3.149, + "module-namespaces-classes-stepped-browser": 6.862, + "module-namespaces-classes-stepped-node": 2.189, + "modules-browser": 2.204, + "modules-node": 1.25, + "mutable-subclasses-classes-drain-browser": 8.67, + "mutable-subclasses-classes-drain-node": 4.572, + "mutable-subclasses-classes-parked-browser": 8.704, + "mutable-subclasses-classes-parked-node": 4.848, + "mutable-subclasses-classes-stepped-browser": 8.505, + "mutable-subclasses-classes-stepped-node": 4.791, + "namespace-view-directories-classes-drain-browser": 4.378, + "namespace-view-directories-classes-drain-node": 1.4, + "namespace-view-directories-classes-parked-browser": 2.728, + "namespace-view-directories-classes-parked-node": 2.096, + "namespace-view-directories-classes-stepped-browser": 2.734, + "namespace-view-directories-classes-stepped-node": 2.049, + "native-attributes-classes-drain-browser": 15.017, + "native-attributes-classes-drain-node": 6.711, + "native-attributes-classes-parked-browser": 15.907, + "native-attributes-classes-parked-node": 6.98, + "native-attributes-classes-stepped-browser": 16.208, + "native-attributes-classes-stepped-node": 4.759, + "native-callable-directories-classes-drain-browser": 5.891, + "native-callable-directories-classes-drain-node": 2.923, + "native-callable-directories-classes-parked-browser": 4.158, + "native-callable-directories-classes-parked-node": 2.884, + "native-callable-directories-classes-stepped-browser": 6.528, + "native-callable-directories-classes-stepped-node": 2.109, + "native-descriptors-classes-drain-browser": 5.958, + "native-descriptors-classes-drain-node": 4.267, + "native-descriptors-classes-parked-browser": 9.931, + "native-descriptors-classes-parked-node": 3.035, + "native-descriptors-classes-stepped-browser": 9.831, + "native-descriptors-classes-stepped-node": 4.296, + "native-directory-closure-classes-drain-browser": 33.818, + "native-directory-closure-classes-drain-node": 15.748, + "native-directory-closure-classes-parked-browser": 21.112, + "native-directory-closure-classes-parked-node": 9.004, + "native-directory-closure-classes-stepped-browser": 26.868, + "native-directory-closure-classes-stepped-node": 16.598, + "native-namespaces-classes-drain-browser": 16.349, + "native-namespaces-classes-drain-node": 18.897, + "native-namespaces-classes-parked-browser": 28.74, + "native-namespaces-classes-parked-node": 12.501, + "native-namespaces-classes-stepped-browser": 18.043, + "native-namespaces-classes-stepped-node": 20.833, + "native-object-subclasses-classes-drain-browser": 7.618, + "native-object-subclasses-classes-drain-node": 2.635, + "native-object-subclasses-classes-parked-browser": 8.304, + "native-object-subclasses-classes-parked-node": 4.022, + "native-object-subclasses-classes-stepped-browser": 8.268, + "native-object-subclasses-classes-stepped-node": 4.052, + "native-protocols-classes-drain-browser": 57.429, + "native-protocols-classes-drain-node": 25.414, + "native-protocols-classes-parked-browser": 59.857, + "native-protocols-classes-parked-node": 3.599, + "native-protocols-classes-stepped-browser": 57.225, + "native-protocols-classes-stepped-node": 24.972, + "native-rebinding-classes-drain-browser": 10.214, + "native-rebinding-classes-drain-node": 4.493, + "native-rebinding-classes-parked-browser": 10.087, + "native-rebinding-classes-parked-node": 4.683, + "native-rebinding-classes-stepped-browser": 10.343, + "native-rebinding-classes-stepped-node": 3.105, + "nested-descriptors-classes-drain-browser": 5.692, + "nested-descriptors-classes-drain-node": 1.838, + "nested-descriptors-classes-parked-browser": 3.99, + "nested-descriptors-classes-parked-node": 2.737, + "nested-descriptors-classes-stepped-browser": 5.766, + "nested-descriptors-classes-stepped-node": 2.632, + "node-unit-abstract-flags": 2.418, + "node-unit-advertised-directories": 1.994, + "node-unit-arithmetic-dispatch": 1.431, + "node-unit-attribute-hooks": 2.154, + "node-unit-browser-host": 0.685, + "node-unit-browser-pool": 0.225, + "node-unit-buffer-directories": 5.529, + "node-unit-callable-type-directories": 2.953, + "node-unit-class-body-namespaces": 0.6, + "node-unit-class-directories": 2.848, + "node-unit-class-mapping-proxies": 13.601, + "node-unit-class-namespaces": 1.446, + "node-unit-class-policy": 0.083, + "node-unit-class-values": 0.171, + "node-unit-classes": 0.252, + "node-unit-classes-source": 1.082, + "node-unit-code-cells": 0.307, + "node-unit-construction": 1.696, + "node-unit-container-directories": 3.94, + "node-unit-container-type-directories": 6.578, + "node-unit-custom-mro": 0.633, + "node-unit-descriptor-binding": 1.365, + "node-unit-descriptors": 0.498, + "node-unit-documentation": 1.213, + "node-unit-dynamic-compile": 0.328, + "node-unit-dynamic-eval": 4.326, + "node-unit-dynamic-exec": 5.001, + "node-unit-dynamic-imports": 1.801, + "node-unit-equality": 1.276, + "node-unit-exceptions": 47.149, + "node-unit-execution": 0.738, + "node-unit-execution-stepping": 1.278, + "node-unit-expanded-class-bases": 0.453, + "node-unit-formatting": 2.704, + "node-unit-function-binding": 1.084, + "node-unit-function-dictionaries": 1.462, + "node-unit-function-directories": 1.944, + "node-unit-function-metadata": 1.391, + "node-unit-function-state": 0.709, + "node-unit-functions": 1.248, + "node-unit-generator-directories": 0.715, + "node-unit-generator-frames": 1.823, + "node-unit-generators": 0.499, + "node-unit-hashing": 1.555, + "node-unit-immutable-subclasses": 2.134, + "node-unit-indexing": 2.848, + "node-unit-instance-dictionaries": 1.513, + "node-unit-instance-directories": 2.693, + "node-unit-integer-identity": 0.612, + "node-unit-iteration": 3.198, + "node-unit-iterator-directories": 2.058, + "node-unit-iterator-type-directories": 3.892, + "node-unit-iterator-types": 0.209, + "node-unit-length-truth": 1.248, + "node-unit-library-profiles": 0.271, + "node-unit-memory": 0.169, + "node-unit-metaclass-attributes": 1.593, + "node-unit-metaclass-factories": 0.711, + "node-unit-metaclass-protocols": 1.856, + "node-unit-method-directories": 3.923, + "node-unit-module-namespaces": 0.721, + "node-unit-module-registry": 0.172, + "node-unit-modules": 0.673, + "node-unit-mutable-subclasses": 2.942, + "node-unit-namespace-view-directories": 0.64, + "node-unit-native-attributes": 1.757, + "node-unit-native-callable-directories": 1.857, + "node-unit-native-descriptors": 1.812, + "node-unit-native-directory-closure": 5.944, + "node-unit-native-namespaces": 20.195, + "node-unit-native-object-subclasses": 2.113, + "node-unit-native-protocols": 3.412, + "node-unit-native-rebinding": 1.411, + "node-unit-nested-descriptors": 2.851, + "node-unit-numeric": 0.335, + "node-unit-numeric-callbacks": 3.373, + "node-unit-numeric-directories": 2.898, + "node-unit-numeric-type-directories": 2.617, + "node-unit-object-hashing": 0.922, + "node-unit-object-integration": 5.712, + "node-unit-object-protocols": 0.556, + "node-unit-operations": 0.111, + "node-unit-ordering": 2.14, + "node-unit-parked-executions": 0.611, + "node-unit-persistent-callbacks": 0.671, + "node-unit-practical-containers": 3.035, + "node-unit-prepared-class-namespaces": 1.373, + "node-unit-private-names": 0.261, + "node-unit-property-diagnostics": 1.512, + "node-unit-property-directories": 1.081, + "node-unit-property-names": 1.156, + "node-unit-prototype": 0.093, + "node-unit-reassignment": 4.179, + "node-unit-reflection": 5.175, + "node-unit-repetition": 2.072, + "node-unit-sequence-directories": 1.59, + "node-unit-sequences": 3.038, + "node-unit-slots": 1.66, + "node-unit-source": 0.566, + "node-unit-source-tokens": 0.17, + "node-unit-special-calls": 0.628, + "node-unit-str-identity": 0.681, + "node-unit-subclass-initialization": 2.0, + "node-unit-syntax": 2.532, + "node-unit-type-checks": 1.111, + "node-unit-type-reflection": 1.578, + "node-unit-value-families": 1.706, + "node-unit-values": 0.122, + "node-unit-vm": 0.071, + "node-unit-wrapper-metadata": 0.727, + "node-unit-wrapper-super-directories": 2.009, + "numeric-classes-drain-browser": 10.781, + "numeric-classes-drain-node": 7.339, + "numeric-classes-stepped-browser": 16.681, + "numeric-classes-stepped-node": 7.57, + "numeric-directories-classes-drain-browser": 12.246, + "numeric-directories-classes-drain-node": 5.63, + "numeric-directories-classes-parked-browser": 7.787, + "numeric-directories-classes-parked-node": 2.321, + "numeric-directories-classes-stepped-browser": 7.808, + "numeric-directories-classes-stepped-node": 5.829, + "numeric-type-directories-classes-drain-browser": 5.965, + "numeric-type-directories-classes-drain-node": 3.062, + "numeric-type-directories-classes-parked-browser": 9.054, + "numeric-type-directories-classes-parked-node": 4.934, + "numeric-type-directories-classes-stepped-browser": 9.585, + "numeric-type-directories-classes-stepped-node": 4.743, + "object-console-browser": 8.97, + "object-hashing-classes-drain-browser": 5.206, + "object-hashing-classes-drain-node": 2.203, + "object-hashing-classes-parked-browser": 3.524, + "object-hashing-classes-parked-node": 2.235, + "object-hashing-classes-stepped-browser": 5.258, + "object-hashing-classes-stepped-node": 2.294, + "object-integration-classes-drain-browser": 6.5, + "object-integration-classes-drain-node": 4.759, + "object-integration-classes-parked-browser": 3.943, + "object-integration-classes-parked-node": 4.192, + "object-integration-classes-stepped-browser": 3.944, + "object-integration-classes-stepped-node": 4.899, + "object-protocols-classes-drain-browser": 5.385, + "object-protocols-classes-drain-node": 2.354, + "object-protocols-classes-stepped-browser": 4.244, + "object-protocols-classes-stepped-node": 2.452, + "object-scope-inventory": 0.186, + "ordering-classes-drain-browser": 12.269, + "ordering-classes-drain-node": 5.185, + "ordering-classes-stepped-browser": 8.198, + "ordering-classes-stepped-node": 5.532, + "ordering-core-drain-browser": 3.467, + "ordering-core-drain-node": 0.267, + "ordering-core-stepped-browser": 5.252, + "ordering-core-stepped-node": 4.43, + "ordinary-syntax-classes-drain-browser": 9.964, + "ordinary-syntax-classes-drain-node": 4.558, + "ordinary-syntax-classes-stepped-browser": 6.51, + "ordinary-syntax-classes-stepped-node": 4.672, + "ordinary-syntax-core-drain-browser": 5.451, + "ordinary-syntax-core-drain-node": 4.174, + "ordinary-syntax-core-stepped-browser": 9.188, + "ordinary-syntax-core-stepped-node": 2.914, + "practical-containers-classes-drain-browser": 12.463, + "practical-containers-classes-drain-node": 8.676, + "practical-containers-classes-parked-browser": 19.519, + "practical-containers-classes-parked-node": 9.051, + "practical-containers-classes-stepped-browser": 20.149, + "practical-containers-classes-stepped-node": 6.358, + "practical-containers-core-drain-browser": 14.541, + "practical-containers-core-drain-node": 7.8, + "practical-containers-core-parked-browser": 4.499, + "practical-containers-core-parked-node": 0.524, + "practical-containers-core-stepped-browser": 14.489, + "practical-containers-core-stepped-node": 0.75, + "prepare-browser": 9.662, + "prepared-class-namespaces-classes-drain-browser": 5.337, + "prepared-class-namespaces-classes-drain-node": 2.366, + "prepared-class-namespaces-classes-parked-browser": 5.279, + "prepared-class-namespaces-classes-parked-node": 2.524, + "prepared-class-namespaces-classes-stepped-browser": 5.251, + "prepared-class-namespaces-classes-stepped-node": 1.527, + "private-names-classes-drain-browser": 4.449, + "private-names-classes-drain-node": 1.206, + "private-names-classes-stepped-browser": 4.49, + "private-names-classes-stepped-node": 1.833, + "property-diagnostics-classes-drain-browser": 12.407, + "property-diagnostics-classes-drain-node": 5.559, + "property-diagnostics-classes-parked-browser": 8.495, + "property-diagnostics-classes-parked-node": 3.898, + "property-diagnostics-classes-stepped-browser": 7.905, + "property-diagnostics-classes-stepped-node": 5.545, + "property-directories-classes-drain-browser": 3.355, + "property-directories-classes-drain-node": 1.543, + "property-directories-classes-parked-browser": 4.633, + "property-directories-classes-parked-node": 2.293, + "property-directories-classes-stepped-browser": 4.526, + "property-directories-classes-stepped-node": 2.206, + "property-names-classes-drain-browser": 5.108, + "property-names-classes-drain-node": 3.308, + "property-names-classes-parked-browser": 7.708, + "property-names-classes-parked-node": 3.295, + "property-names-classes-stepped-browser": 5.039, + "property-names-classes-stepped-node": 2.347, + "reassignment-classes-drain-browser": 7.587, + "reassignment-classes-drain-node": 3.911, + "reassignment-classes-parked-browser": 11.957, + "reassignment-classes-parked-node": 5.891, + "reassignment-classes-stepped-browser": 11.798, + "reassignment-classes-stepped-node": 5.98, + "reflection-classes-drain-browser": 18.559, + "reflection-classes-drain-node": 12.753, + "reflection-classes-stepped-browser": 18.961, + "reflection-classes-stepped-node": 3.764, + "repetition-classes-drain-browser": 8.698, + "repetition-classes-drain-node": 5.755, + "repetition-classes-stepped-browser": 13.608, + "repetition-classes-stepped-node": 6.076, + "repetition-core-drain-browser": 8.818, + "repetition-core-drain-node": 5.394, + "repetition-core-stepped-browser": 8.441, + "repetition-core-stepped-node": 3.914, + "sequence-directories-classes-drain-browser": 5.828, + "sequence-directories-classes-drain-node": 2.544, + "sequence-directories-classes-parked-browser": 4.263, + "sequence-directories-classes-parked-node": 1.741, + "sequence-directories-classes-stepped-browser": 3.506, + "sequence-directories-classes-stepped-node": 2.664, + "sequences-classes-drain-browser": 3.75, + "sequences-classes-drain-node": 2.611, + "sequences-classes-stepped-browser": 5.655, + "sequences-classes-stepped-node": 1.762, + "sequences-core-drain-browser": 3.259, + "sequences-core-drain-node": 2.275, + "sequences-core-stepped-browser": 4.59, + "sequences-core-stepped-node": 2.408, + "shared-application-browser": 3.308, + "slots-classes-drain-browser": 6.727, + "slots-classes-drain-node": 2.819, + "slots-classes-parked-browser": 4.312, + "slots-classes-parked-node": 2.793, + "slots-classes-stepped-browser": 6.74, + "slots-classes-stepped-node": 1.874, + "source-browser": 24.263, + "source-node": 8.656, + "source-tokens-browser": 2.7, + "source-tokens-node": 0.175, + "special-calls-classes-drain-browser": 7.436, + "special-calls-classes-drain-node": 3.605, + "special-calls-classes-stepped-browser": 7.493, + "special-calls-classes-stepped-node": 2.397, + "subclass-initialization-classes-drain-browser": 7.527, + "subclass-initialization-classes-drain-node": 3.678, + "subclass-initialization-classes-stepped-browser": 5.054, + "subclass-initialization-classes-stepped-node": 3.746, + "type-checks-classes-drain-browser": 9.414, + "type-checks-classes-drain-node": 2.902, + "type-checks-classes-stepped-browser": 5.972, + "type-checks-classes-stepped-node": 4.232, + "type-reflection-classes-drain-browser": 8.894, + "type-reflection-classes-drain-node": 3.919, + "type-reflection-classes-parked-browser": 9.26, + "type-reflection-classes-parked-node": 4.13, + "type-reflection-classes-stepped-browser": 9.455, + "type-reflection-classes-stepped-node": 4.183, + "value-families-classes-drain-browser": 12.84, + "value-families-classes-drain-node": 5.655, + "value-families-classes-stepped-browser": 13.275, + "value-families-classes-stepped-node": 5.973, + "value-families-core-drain-browser": 11.143, + "value-families-core-drain-node": 5.673, + "value-families-core-stepped-browser": 7.25, + "value-families-core-stepped-node": 3.894, + "vm1-browser": 2.883, + "vm1-node": 0.782, + "vm2-browser": 8.822, + "vm2-node": 2.273, + "vm3-browser": 10.579, + "vm3-node": 3.23, + "wrapper-metadata-classes-drain-browser": 5.385, + "wrapper-metadata-classes-drain-node": 2.186, + "wrapper-metadata-classes-parked-browser": 3.627, + "wrapper-metadata-classes-parked-node": 2.354, + "wrapper-metadata-classes-stepped-browser": 5.216, + "wrapper-metadata-classes-stepped-node": 1.543, + "wrapper-super-directories-classes-drain-browser": 4.073, + "wrapper-super-directories-classes-drain-node": 2.831, + "wrapper-super-directories-classes-parked-browser": 6.287, + "wrapper-super-directories-classes-parked-node": 2.871, + "wrapper-super-directories-classes-stepped-browser": 6.411, + "wrapper-super-directories-classes-stepped-node": 1.981 + } +} From f562c58507433b62bef23a5159762f7c3bd19b47 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:02:18 -0800 Subject: [PATCH 6/6] fix(CI-05A): separate oracle setup from worker timings --- docs/testing.md | 2 ++ scripts/benchmark_browser_workers.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 430237c..2d472ff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -58,6 +58,8 @@ terminates the active pool before the browser closes. Set the complete exceptions corpus and requires identical ordered results, including resource/teardown observations. Use `--corpus PATH` for another corpus. It writes timings and host/corpus identities to `build/browser-worker-benchmark.json`. +Oracle validation/cache warming runs once before the timed samples and appears +separately as `reference_setup`, so a cold cache cannot bias the serial sample. The classes console compares its bulk successful language programs through two installed console host/worker sessions. Each run still creates its own module diff --git a/scripts/benchmark_browser_workers.py b/scripts/benchmark_browser_workers.py index d4d0679..2d37bc6 100644 --- a/scripts/benchmark_browser_workers.py +++ b/scripts/benchmark_browser_workers.py @@ -11,7 +11,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'runtime')) -from check_classes import check +from check_classes import cached_references, check def main(): @@ -23,6 +23,11 @@ def main(): args = parser.parse_args() if args.repeats < 1: parser.error('--repeats must be positive') + reference_start = time.monotonic() + reference = cached_references(args.corpus) + if reference['summary']['fail']: + raise AssertionError(reference['summary']) + reference_seconds = round(time.monotonic() - reference_start, 3) baseline, measurements = None, [] original = os.environ.get('POCKET_PYTHON_BROWSER_WORKERS') try: @@ -53,6 +58,8 @@ def main(): args.report.write_text(json.dumps({'schema_version': 1, 'python': platform.python_version(), 'host': platform.platform(), 'browser': report['host_version'], 'corpus_sha256': report['corpus_sha256'], 'repeats': args.repeats, + 'reference_setup': {'seconds': reference_seconds, 'summary': reference['summary'], + 'cache': reference['reference_cache']}, 'measurements': measurements}, indent=2) + '\n', encoding='utf-8')