diff --git a/docs/testing.md b/docs/testing.md index 01006b2..31caeaa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -59,6 +59,15 @@ the complete exceptions corpus and requires identical ordered results, including resource/teardown observations. Use `--corpus PATH` for another corpus. It writes timings and host/corpus identities to `build/browser-worker-benchmark.json`. +The classes console compares its bulk successful language programs through two +installed console host/worker sessions. Each run still creates its own module +session with the shipped profile, libraries, output grant and resource limits. +Corpus stdout comes from the same committed expectations checked independently +by the differential matrix; standalone examples still run isolated CPython. +Representative form submissions, all source-error/diagnostic checks, offline +reloads, service-worker/profile updates and every Stop/Restart cycle retain +their UI paths. The console report records UI/worker counts and elapsed time. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/scripts/classes_console_browser_smoke.py b/scripts/classes_console_browser_smoke.py index 31149e2..92d50b2 100644 --- a/scripts/classes_console_browser_smoke.py +++ b/scripts/classes_console_browser_smoke.py @@ -15,6 +15,7 @@ from playwright.sync_api import sync_playwright from interpreter_browser_smoke import EXAMPLE, Handler, Server +from console_cases import program_expectations from pocket_python.interpreter import build_interpreter, read_interpreter ROOT = Path(__file__).resolve().parents[1] @@ -109,6 +110,8 @@ def __getattribute__(self,n): def smoke(channel=None): checks, cases = [], [] + started = time.perf_counter() + transport = {'ui': 0, 'worker': 0} with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) source = EXAMPLE.write_source(root / 'source', source=SOURCE) @@ -140,6 +143,7 @@ def ready(): page.wait_for_function('()=>navigator.serviceWorker.controller!==null') def run(source, expected): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() # Repeated corpus runs need the normal click handler, @@ -150,6 +154,7 @@ def run(source, expected): assert page.locator('#result').inner_text() == expected def refused(source, prefix): + transport['ui'] += 1 page.fill('#input', source) assert page.locator('#run').is_enabled() page.locator('#run').dispatch_event('click') @@ -365,10 +370,45 @@ def refused(source, prefix): programs.append(PROPERTY_NAMES_SOURCE) programs.append(ABSTRACT_FLAGS_SOURCE) programs.append(DOCUMENTATION_SOURCE) - for program in programs: - expected = subprocess.check_output([sys.executable, '-I', '-S', '-X', 'utf8', '-c', program], - text=True, encoding='utf-8', timeout=5) - run(program, expected) + expectations = program_expectations(programs) + # Keep a representative real form submission before the bulk + # comparisons; error, offline/update and Stop/Restart checks + # below continue through their existing UI paths. + run(programs[0], expectations[0]) + config = json.loads((root / 'site/nested/classes' / selected['configuration']).read_bytes()) + config['library_sources'] = [{**{key: module[key] for key in ('name', 'package', 'imports')}, + **{key: (root / 'site/nested/classes' / module[key]['path']).read_text(encoding='utf-8') + for key in ('source', 'notice')}} for module in config['libraries']] + results = page.evaluate('''async ({host,config,programs})=>{ + const {mountInterpreter}=await import('./'+host); + const results=new Array(programs.length),active=new Set();let next=0,failed=false; + try{ + await Promise.all(Array.from({length:Math.min(2,programs.length)},async()=>{ + let output=''; + const app=mountInterpreter({application:config.application,capabilities:config.capabilities, + limits:config.limits,libraryProfile:config.library_profile,libraries:config.library_sources, + hostProfile:config.host_profile,onOutput:text=>{output+=text;}}); + active.add(app); + try{ + await app.ready; + while(!failed&&next