From 997e9a88a4b5d64c5d988588bdb7dafd4c969604 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:36:46 -0800 Subject: [PATCH 1/3] perf(CI-04): reuse validated CPython corpus references --- .github/workflows/ci.yml | 11 ++++++ docs/testing.md | 9 +++++ runtime/check_classes.py | 49 ++++++++++++++++++++++++- tests/test_reference_diagnostics.py | 57 +++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2affde2..dabebae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,11 @@ jobs: version: "0.12.19" enable-cache: true - run: uv sync --locked + - uses: actions/cache@v5 + with: + path: build/reference-cache + key: oracle-v1-${{ runner.os }}-node-${{ matrix.shard }}-${{ hashFiles('tests/fixtures/interpreter/*.json', 'runtime/check_classes.py', 'runtime/check_functions.py') }} + restore-keys: oracle-v1-${{ runner.os }}-node-${{ matrix.shard }}- - run: uv run --no-sync python scripts/run_tests.py interpreter --shard ${{ matrix.shard }} --shards 4 - uses: actions/upload-artifact@v5 if: always() @@ -128,6 +133,12 @@ jobs: - run: uv run --no-sync python scripts/run_tests.py prepare-browser - if: matrix.suite == 'browser-integration' run: uv run --no-sync python scripts/prepare_runtime.py + - uses: actions/cache@v5 + if: matrix.suite == 'browser-conformance' + with: + path: build/reference-cache + key: oracle-v1-${{ runner.os }}-browser-${{ matrix.shard }}-${{ hashFiles('tests/fixtures/interpreter/*.json', 'runtime/check_classes.py', 'runtime/check_functions.py') }} + restore-keys: oracle-v1-${{ runner.os }}-browser-${{ matrix.shard }}- - run: uv run --no-sync python scripts/run_tests.py ${{ matrix.suite }} --shard ${{ matrix.shard }} --shards ${{ matrix.shards }} - uses: actions/upload-artifact@v5 if: always() diff --git a/docs/testing.md b/docs/testing.md index c6658c7..613a566 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -41,6 +41,15 @@ selected suites/features, command results and durations. Use `--report PATH` for a separate report location. Failures print the end of their log and return a nonzero exit status after the remaining selected checks finish. +Class-corpus differential checks reuse validated CPython reports under +`build/reference-cache` (override with `POCKET_PYTHON_REFERENCE_CACHE`). Cache +identity includes the complete corpus, Python build/host and both reference +drivers; failed or damaged entries are recomputed. Reports record cache hits. +CI restores each shard's cache across PR updates, including unchanged corpora +when another corpus changes. Standalone `--reference-only` checks always execute +CPython again, so nightly/manual/release validation independently verifies the +recorded expectations. Remove the directory to benchmark a cold run. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/runtime/check_classes.py b/runtime/check_classes.py index c423cde..b948e68 100644 --- a/runtime/check_classes.py +++ b/runtime/check_classes.py @@ -3,9 +3,11 @@ import argparse import hashlib import json +import os from pathlib import Path import platform import subprocess +import sys import tempfile from check_conformance import run_browser @@ -65,8 +67,52 @@ def references(corpus_path=CORPUS): for status in ['pass', 'fail']}} -def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, profile=None, parked=False): +def cached_references(corpus_path, cache_dir=None): + """Reuse only successful, intact reports for these exact oracle inputs.""" + cache_dir = Path(cache_dir or os.environ.get('POCKET_PYTHON_REFERENCE_CACHE', + ROOT.parent / 'build/reference-cache')) + identity = { + 'corpus': hashlib.sha256(corpus_path.read_bytes()).hexdigest(), + 'python': sys.version, 'implementation': platform.python_implementation(), + 'host': platform.platform(), + 'harness': {path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in (Path(__file__), ROOT / 'check_functions.py')}, + } + def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode('utf-8')).hexdigest() + + key = digest(identity) + path = cache_dir / f'{corpus_path.stem}-{key}.json' + try: + saved = json.loads(path.read_bytes()) + report = saved['report'] + if (saved['identity'] == identity and saved['report_sha256'] == digest(report) and + report['corpus_sha256'] == identity['corpus'] and + report['summary'] == {'pass': len(report['cases']), 'fail': 0} and + all(case['reference_status'] == 'pass' for case in report['cases'])): + return {**report, 'reference_cache': {'key': key, 'hit': True}} + except (FileNotFoundError, ValueError, KeyError, TypeError): + pass report = references(corpus_path) + if not report['summary']['fail']: + cache_dir.mkdir(parents=True, exist_ok=True) + # Atomic replacement also permits independent processes to share a cache. + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=cache_dir, + delete=False) as output: + json.dump({'identity': identity, 'report_sha256': digest(report), 'report': report}, output) + temporary = Path(output.name) + try: + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + for previous in cache_dir.glob(corpus_path.stem + '-*.json'): + if previous != path: + previous.unlink(missing_ok=True) + return {**report, 'reference_cache': {'key': key, 'hit': False}} + + +def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, profile=None, parked=False): + report = cached_references(corpus_path) if report['summary']['fail']: raise AssertionError('The class reference corpus differs from its recorded expectations.') corpus = json.loads(corpus_path.read_bytes()) @@ -130,6 +176,7 @@ def check(target, channel=None, corpus_path=CORPUS, stages=None, stepped=False, 'omitted_stages': sorted(available - {case['stage'] for case in cases}), 'omitted_profile_cases': [case['id'] for case in corpus['cases'] if selected_profile not in case.get('profiles', [corpus['profile']])], 'reference_summary': report['summary'], + 'reference_cache': report['reference_cache'], 'implementation_status': 'Implemented stages: ' + ', '.join(adopted) + '. Other recorded stages must refuse.', 'comparison': 'Adopted stdout and error types match CPython; messages/repr identity are not compared. Later stages must refuse explicitly.', 'inputs_sha256': {p.name: hashlib.sha256(p.read_bytes().replace(b'\r\n', b'\n')).hexdigest() diff --git a/tests/test_reference_diagnostics.py b/tests/test_reference_diagnostics.py index 1560208..62fe0ba 100644 --- a/tests/test_reference_diagnostics.py +++ b/tests/test_reference_diagnostics.py @@ -12,6 +12,63 @@ class ReferenceDiagnosticsTests(unittest.TestCase): + def test_cache_reuses_success_and_invalidates_corpus_python_and_harness(self): + corpus = {'profile': 'classes-source/1', 'cases': [ + {'id': 'one', 'stage': 'C1', 'source': 'print(7)', 'stdout': '7\n'}]} + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + path, cache = root / 'corpus.json', root / 'cache' + path.write_text(json.dumps(corpus), encoding='utf-8') + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': '7\n'}) as reference: + first = check_classes.cached_references(path, cache) + second = check_classes.cached_references(path, cache) + self.assertFalse(first['reference_cache']['hit']) + self.assertTrue(second['reference_cache']['hit']) + self.assertEqual(first['cases'], second['cases']) + self.assertEqual(reference.call_count, 1) + # Metadata and expectation changes also invalidate the whole corpus. + corpus['cases'][0]['stage'] = 'C2' + path.write_text(json.dumps(corpus), encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + with patch.object(check_classes.sys, 'version', sys.version.replace(sys.version.split()[0], '9.9.9', 1)): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + with patch.object(check_classes.platform, 'platform', return_value='another host'): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + original = Path.read_bytes + def changed_harness(file): + data = original(file) + return data + b'\n# changed reference' if file.name == 'check_functions.py' else data + with patch.object(Path, 'read_bytes', changed_harness): + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + self.assertEqual(reference.call_count, 5) + self.assertEqual(len(list(cache.glob('*.json'))), 1) + + def test_damaged_cache_is_recomputed_and_failed_reports_are_not_saved(self): + corpus = {'profile': 'classes-source/1', 'cases': [ + {'id': 'one', 'stage': 'C1', 'source': 'print(7)', 'stdout': '7\n'}]} + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + path, cache = root / 'corpus.json', root / 'cache' + path.write_text(json.dumps(corpus), encoding='utf-8') + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': '7\n'}) as reference: + check_classes.cached_references(path, cache) + saved_path, = cache.glob('*.json') + for damaged in ('{truncated', 'null', '{}'): + saved_path.write_text(damaged, encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + saved = json.loads(saved_path.read_bytes()) + saved['report']['cases'][0]['reference']['stdout'] = 'tampered\n' + saved_path.write_text(json.dumps(saved), encoding='utf-8') + self.assertFalse(check_classes.cached_references(path, cache)['reference_cache']['hit']) + self.assertEqual(reference.call_count, 5) + # Reference-only validation always executes the oracle afresh. + check_classes.references(path) + self.assertEqual(reference.call_count, 6) + saved_path.unlink() + with patch.object(check_classes, 'reference_case', return_value={'ok': True, 'stdout': 'wrong\n'}): + self.assertEqual(check_classes.cached_references(path, cache)['summary']['fail'], 1) + self.assertFalse(list(cache.iterdir())) + def test_failed_process_identifies_the_case_and_preserves_the_cause(self): corpus = {"profile": "classes-source/1", "cases": [{"id": "failing-input", "stage": "test", "source": "pass"}]} for failure, detail in [(subprocess.CalledProcessError(-11, ["reference"]), "exit -11"), From a53ed85c81bb14462bb8350694e135893b19e918 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:41:56 -0800 Subject: [PATCH 2/3] perf(CI-05): bound concurrent browser conformance workers --- docs/testing.md | 9 ++++ runtime/browser-pool.mjs | 50 ++++++++++++++++++++ runtime/browser-pool.test.mjs | 69 ++++++++++++++++++++++++++++ runtime/check_conformance.py | 31 ++++--------- scripts/benchmark_browser_workers.py | 60 ++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 21 deletions(-) create mode 100644 runtime/browser-pool.mjs create mode 100644 runtime/browser-pool.test.mjs create mode 100644 scripts/benchmark_browser_workers.py diff --git a/docs/testing.md b/docs/testing.md index 613a566..01006b2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -50,6 +50,15 @@ when another corpus changes. Standalone `--reference-only` checks always execute CPython again, so nightly/manual/release validation independently verifies the recorded expectations. Remove the directory to benchmark a cold run. +Browser differential requests use two concurrent fresh workers, with the same +10-second per-case deadline and results retained in input order. A failure +terminates the active pool before the browser closes. Set +`POCKET_PYTHON_BROWSER_WORKERS=1` for a serial comparison (accepted range: 1–8). +`scripts/benchmark_browser_workers.py --repeats 3` compares 1/2/4/8 workers on +the complete exceptions corpus and requires identical ordered results, including +resource/teardown observations. Use `--corpus PATH` for another corpus. It writes +timings and host/corpus identities to `build/browser-worker-benchmark.json`. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/runtime/browser-pool.mjs b/runtime/browser-pool.mjs new file mode 100644 index 0000000..8fdc11e --- /dev/null +++ b/runtime/browser-pool.mjs @@ -0,0 +1,50 @@ +// Test transport: each request owns a fresh worker, even when requests overlap. +export async function runWorkerRequests(workerPath, requests, { + concurrency = 2, timeout = 10000, WorkerClass = globalThis.Worker, +} = {}) { + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) { + throw new RangeError('Browser concurrency must be an integer from 1 to 8.'); + } + if (!Number.isFinite(timeout) || timeout <= 0) throw new RangeError('Invalid worker timeout.'); + const results = new Array(requests.length); + const active = new Set(); + let next = 0, failed = false; + const run = request => new Promise((resolve, reject) => { + let worker, timer; + const cleanup = () => { + clearTimeout(timer); + active.delete(cleanup); + if (worker) { + worker.onmessage = worker.onerror = worker.onmessageerror = null; + worker.terminate(); + } + }; + const fail = error => { + failed = true; + cleanup(); + for (const close of active) close(); + reject(new Error(`Conformance worker ${request.id}: ${error.message}`, {cause: error})); + }; + try { + worker = new WorkerClass(workerPath, {type: 'module'}); + active.add(cleanup); + timer = setTimeout(() => fail(new Error(`timed out after ${timeout}ms`)), timeout); + worker.onmessage = event => { + cleanup(); + resolve({...event.data, id: request.id}); + }; + worker.onerror = event => fail(new Error(event.message)); + worker.onmessageerror = () => fail(new Error('could not deserialize response')); + worker.postMessage(request); + } catch (error) { + fail(error); + } + }); + await Promise.all(Array.from({length: Math.min(concurrency, requests.length)}, async () => { + while (!failed && next < requests.length) { + const index = next++; + results[index] = await run(requests[index]); + } + })); + return results; +} diff --git a/runtime/browser-pool.test.mjs b/runtime/browser-pool.test.mjs new file mode 100644 index 0000000..4bdd286 --- /dev/null +++ b/runtime/browser-pool.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import {runWorkerRequests} from './browser-pool.mjs'; + +function workers() { + const instances = []; + let live = 0, peak = 0; + class FakeWorker { + constructor(path, options) { + assert.equal(path, '/fixture.mjs'); + assert.deepEqual(options, {type: 'module'}); + instances.push(this); + peak = Math.max(peak, ++live); + } + postMessage(request) { + this.request = request; + if (request.throw) throw new Error('post failed'); + if (request.hang) return; + this.timer = setTimeout(() => { + if (request.error) this.onerror?.({message: 'worker failed'}); + else if (request.messageerror) this.onmessageerror?.(); + else this.onmessage?.({data: {ok: true, value: request.id}}); + }, request.delay ?? 1); + } + terminate() { + assert.equal(this.closed, undefined, 'worker terminated once'); + this.closed = true; + clearTimeout(this.timer); + live--; + } + } + return {WorkerClass: FakeWorker, instances, live: () => live, peak: () => peak}; +} + +test('bounded fresh workers preserve input order despite out-of-order completion', async () => { + for (const concurrency of [1, 2, 4, 8]) { + const fixture = workers(); + const requests = Array.from({length: 12}, (_, id) => ({id, delay: id === 0 ? 20 : 1})); + const results = await runWorkerRequests('/fixture.mjs', requests, {...fixture, concurrency}); + assert.deepEqual(results, requests.map(({id}) => ({id, ok: true, value: id}))); + assert.equal(fixture.instances.length, requests.length); + assert.equal(fixture.peak(), concurrency); + assert.equal(fixture.live(), 0); + } +}); + +test('errors, serialization failures and per-case deadlines close all workers and stop scheduling', async () => { + for (const failure of [{error: true}, {messageerror: true}, {throw: true}, {hang: true}]) { + const fixture = workers(); + const requests = [{id: 'blocked', hang: true}, {id: 'fails', ...failure}, {id: 'never'}]; + await assert.rejects(runWorkerRequests('/fixture.mjs', requests, + {...fixture, concurrency: 2, timeout: 25}), /Conformance worker (fails|blocked):/); + assert.equal(fixture.live(), 0); + assert.equal(fixture.instances.length, 2); + } +}); + +test('worker construction failure is reported and invalid bounds are rejected', async () => { + await assert.rejects(runWorkerRequests('/fixture.mjs', [{id: 'one'}], { + WorkerClass: class {constructor() {throw new Error('construction failed');}}, + }), /one: construction failed/); + for (const concurrency of [0, 9, 1.5, NaN]) { + await assert.rejects(runWorkerRequests('', [], {concurrency}), RangeError); + } + for (const timeout of [0, -1, Infinity]) { + await assert.rejects(runWorkerRequests('', [], {timeout}), RangeError); + } + assert.deepEqual(await runWorkerRequests('', []), []); +}); diff --git a/runtime/check_conformance.py b/runtime/check_conformance.py index 3b26961..37ad37c 100644 --- a/runtime/check_conformance.py +++ b/runtime/check_conformance.py @@ -5,6 +5,7 @@ import hashlib from http.server import ThreadingHTTPServer import json +import os from pathlib import Path import platform import random @@ -54,10 +55,13 @@ def run_node(engine, requests): return response["host"], response["results"] -def run_browser(engine, requests, channel): +def run_browser(engine, requests, channel, concurrency=None): from browser_smoke import Handler from playwright.sync_api import sync_playwright + concurrency = int(os.environ.get('POCKET_PYTHON_BROWSER_WORKERS', '2')) if concurrency is None else concurrency + if not 1 <= concurrency <= 8: + raise ValueError('Browser concurrency must be between 1 and 8.') server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(ROOT))) thread = Thread(target=server.serve_forever, daemon=True) thread.start() @@ -68,26 +72,11 @@ def run_browser(engine, requests, channel): try: page = browser.new_page() page.goto(origin) - responses = page.evaluate("""async ({workerPath, requests}) => { - const results = []; - for (const request of requests) { - const result = await new Promise((resolve, reject) => { - const worker = new Worker(workerPath, {type: 'module'}); - const timer = setTimeout(() => { - worker.terminate(); reject(new Error('Conformance worker timed out.')); - }, 10000); - worker.onmessage = event => { - clearTimeout(timer); worker.terminate(); resolve(event.data); - }; - worker.onerror = event => { - clearTimeout(timer); worker.terminate(); reject(new Error(event.message)); - }; - worker.postMessage(request); - }); - results.push({id: request.id, ...result}); - } - return results; - }""", {"workerPath": "/" + engine["worker"], "requests": requests}) + responses = page.evaluate("""async ({workerPath, requests, concurrency}) => { + const {runWorkerRequests} = await import('/browser-pool.mjs'); + return runWorkerRequests(workerPath, requests, {concurrency}); + }""", {"workerPath": "/" + engine["worker"], "requests": requests, + "concurrency": concurrency}) return browser.version, responses finally: browser.close() diff --git a/scripts/benchmark_browser_workers.py b/scripts/benchmark_browser_workers.py new file mode 100644 index 0000000..d4d0679 --- /dev/null +++ b/scripts/benchmark_browser_workers.py @@ -0,0 +1,60 @@ +"""Compare isolated browser worker pool sizes on the same differential corpus.""" + +import argparse +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'runtime')) +from check_classes import check + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--corpus', type=Path, default=ROOT / 'tests/fixtures/interpreter/exceptions.json') + parser.add_argument('--repeats', type=int, default=3) + parser.add_argument('--channel') + parser.add_argument('--report', type=Path, default=ROOT / 'build/browser-worker-benchmark.json') + args = parser.parse_args() + if args.repeats < 1: + parser.error('--repeats must be positive') + baseline, measurements = None, [] + original = os.environ.get('POCKET_PYTHON_BROWSER_WORKERS') + try: + for concurrency in (1, 2, 4, 8): + durations = [] + os.environ['POCKET_PYTHON_BROWSER_WORKERS'] = str(concurrency) + for _ in range(args.repeats): + start = time.monotonic() + report = check('browser', args.channel, args.corpus) + durations.append(round(time.monotonic() - start, 3)) + if report['summary']['fail']: + raise AssertionError(report['summary']) + observed = [(case['id'], case['status'], case['actual']) for case in report['cases']] + if baseline is None: + baseline = observed + if observed != baseline: + raise AssertionError('Pool size changed the ordered differential results') + row = {'workers': concurrency, 'seconds': durations, + 'median_seconds': statistics.median(durations), 'cases': len(baseline)} + measurements.append(row) + print(json.dumps(row), flush=True) + finally: + if original is None: + os.environ.pop('POCKET_PYTHON_BROWSER_WORKERS', None) + else: + os.environ['POCKET_PYTHON_BROWSER_WORKERS'] = original + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps({'schema_version': 1, 'python': platform.python_version(), + 'host': platform.platform(), 'browser': report['host_version'], + 'corpus_sha256': report['corpus_sha256'], 'repeats': args.repeats, + 'measurements': measurements}, indent=2) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + main() From 44417a117246464c8b3f0f3024a0b41b2a3b8ee9 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:49:09 -0800 Subject: [PATCH 3/3] perf(CI-06): batch bulk console language acceptance --- docs/testing.md | 9 +++++ scripts/classes_console_browser_smoke.py | 51 +++++++++++++++++++++--- scripts/console_cases.py | 30 ++++++++++++++ tests/test_console_cases.py | 40 +++++++++++++++++++ 4 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 scripts/console_cases.py create mode 100644 tests/test_console_cases.py diff --git a/docs/testing.md b/docs/testing.md index 01006b2..31caeaa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -59,6 +59,15 @@ the complete exceptions corpus and requires identical ordered results, including resource/teardown observations. Use `--corpus PATH` for another corpus. It writes timings and host/corpus identities to `build/browser-worker-benchmark.json`. +The classes console compares its bulk successful language programs through two +installed console host/worker sessions. Each run still creates its own module +session with the shipped profile, libraries, output grant and resource limits. +Corpus stdout comes from the same committed expectations checked independently +by the differential matrix; standalone examples still run isolated CPython. +Representative form submissions, all source-error/diagnostic checks, offline +reloads, service-worker/profile updates and every Stop/Restart cycle retain +their UI paths. The console report records UI/worker counts and elapsed time. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/scripts/classes_console_browser_smoke.py b/scripts/classes_console_browser_smoke.py index 31149e2..92d50b2 100644 --- a/scripts/classes_console_browser_smoke.py +++ b/scripts/classes_console_browser_smoke.py @@ -15,6 +15,7 @@ from playwright.sync_api import sync_playwright from interpreter_browser_smoke import EXAMPLE, Handler, Server +from console_cases import program_expectations from pocket_python.interpreter import build_interpreter, read_interpreter ROOT = Path(__file__).resolve().parents[1] @@ -109,6 +110,8 @@ def __getattribute__(self,n): def smoke(channel=None): checks, cases = [], [] + started = time.perf_counter() + transport = {'ui': 0, 'worker': 0} with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) source = EXAMPLE.write_source(root / 'source', source=SOURCE) @@ -140,6 +143,7 @@ def ready(): page.wait_for_function('()=>navigator.serviceWorker.controller!==null') def run(source, expected): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() # Repeated corpus runs need the normal click handler, @@ -150,6 +154,7 @@ def run(source, expected): assert page.locator('#result').inner_text() == expected def refused(source, prefix): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() page.locator('#run').dispatch_event('click') @@ -365,10 +370,45 @@ def refused(source, prefix): programs.append(PROPERTY_NAMES_SOURCE) programs.append(ABSTRACT_FLAGS_SOURCE) programs.append(DOCUMENTATION_SOURCE) - for program in programs: - expected = subprocess.check_output([sys.executable, '-I', '-S', '-X', 'utf8', '-c', program], - text=True, encoding='utf-8', timeout=5) - run(program, expected) + expectations = program_expectations(programs) + # Keep a representative real form submission before the bulk + # comparisons; error, offline/update and Stop/Restart checks + # below continue through their existing UI paths. + run(programs[0], expectations[0]) + config = json.loads((root / 'site/nested/classes' / selected['configuration']).read_bytes()) + config['library_sources'] = [{**{key: module[key] for key in ('name', 'package', 'imports')}, + **{key: (root / 'site/nested/classes' / module[key]['path']).read_text(encoding='utf-8') + for key in ('source', 'notice')}} for module in config['libraries']] + results = page.evaluate('''async ({host,config,programs})=>{ + const {mountInterpreter}=await import('./'+host); + const results=new Array(programs.length),active=new Set();let next=0,failed=false; + try{ + await Promise.all(Array.from({length:Math.min(2,programs.length)},async()=>{ + let output=''; + const app=mountInterpreter({application:config.application,capabilities:config.capabilities, + limits:config.limits,libraryProfile:config.library_profile,libraries:config.library_sources, + hostProfile:config.host_profile,onOutput:text=>{output+=text;}}); + active.add(app); + try{ + await app.ready; + while(!failed&&next