perf(CI-06): batch bulk console language acceptance

docs/testing.md

@@ -59,6 +59,15 @@ the complete exceptions corpus and requires identical ordered results, including
 resource/teardown observations. Use `--corpus PATH` for another corpus. It writes
 timings and host/corpus identities to `build/browser-worker-benchmark.json`.
 
+The classes console compares its bulk successful language programs through two
+installed console host/worker sessions. Each run still creates its own module
+session with the shipped profile, libraries, output grant and resource limits.
+Corpus stdout comes from the same committed expectations checked independently
+by the differential matrix; standalone examples still run isolated CPython.
+Representative form submissions, all source-error/diagnostic checks, offline
+reloads, service-worker/profile updates and every Stop/Restart cycle retain
+their UI paths. The console report records UI/worker counts and elapsed time.
+
 ## Fast development and merge policy
 
 Run the smallest meaningful local checks for the behavior being changed. A

scripts/classes_console_browser_smoke.py

@@ -15,6 +15,7 @@
 from playwright.sync_api import sync_playwright
 
 from interpreter_browser_smoke import EXAMPLE, Handler, Server
+from console_cases import program_expectations
 from pocket_python.interpreter import build_interpreter, read_interpreter
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -109,6 +110,8 @@ def __getattribute__(self,n):
 
 def smoke(channel=None):
     checks, cases = [], []
+    started = time.perf_counter()
+    transport = {'ui': 0, 'worker': 0}
     with tempfile.TemporaryDirectory() as temporary:
         root = Path(temporary)
         source = EXAMPLE.write_source(root / 'source', source=SOURCE)
@@ -140,6 +143,7 @@ def ready():
                         page.wait_for_function('()=>navigator.serviceWorker.controller!==null')
 
                     def run(source, expected):
+                        transport['ui'] += 1
                         page.fill('#input', source)
                         assert page.locator('#run').is_enabled()
                         # Repeated corpus runs need the normal click handler,
@@ -150,6 +154,7 @@ def run(source, expected):
                         assert page.locator('#result').inner_text() == expected
 
                     def refused(source, prefix):
+                        transport['ui'] += 1
                         page.fill('#input', source)
                         assert page.locator('#run').is_enabled()
                         page.locator('#run').dispatch_event('click')
@@ -365,10 +370,45 @@ def refused(source, prefix):
                     programs.append(PROPERTY_NAMES_SOURCE)
                     programs.append(ABSTRACT_FLAGS_SOURCE)
                     programs.append(DOCUMENTATION_SOURCE)
-                    for program in programs:
-                        expected = subprocess.check_output([sys.executable, '-I', '-S', '-X', 'utf8', '-c', program],
-                                                           text=True, encoding='utf-8', timeout=5)
-                        run(program, expected)
+                    expectations = program_expectations(programs)
+                    # Keep a representative real form submission before the bulk
+                    # comparisons; error, offline/update and Stop/Restart checks
+                    # below continue through their existing UI paths.
+                    run(programs[0], expectations[0])
+                    config = json.loads((root / 'site/nested/classes' / selected['configuration']).read_bytes())
+                    config['library_sources'] = [{**{key: module[key] for key in ('name', 'package', 'imports')},
+                        **{key: (root / 'site/nested/classes' / module[key]['path']).read_text(encoding='utf-8')
+                           for key in ('source', 'notice')}} for module in config['libraries']]
+                    results = page.evaluate('''async ({host,config,programs})=>{
+                      const {mountInterpreter}=await import('./'+host);
+                      const results=new Array(programs.length),active=new Set();let next=0,failed=false;
+                      try{
+                        await Promise.all(Array.from({length:Math.min(2,programs.length)},async()=>{
+                          let output='';
+                          const app=mountInterpreter({application:config.application,capabilities:config.capabilities,
+                            limits:config.limits,libraryProfile:config.library_profile,libraries:config.library_sources,
+                            hostProfile:config.host_profile,onOutput:text=>{output+=text;}});
+                          active.add(app);
+                          try{
+                            await app.ready;
+                            while(!failed&&next<programs.length){
+                              const index=next++;output='';
+                              await app.run(programs[index]);results[index]=output;
+                            }
+                          }catch(error){failed=true;throw error;}
+                          finally{app.stop('completed');active.delete(app);
+                            const stats=app.stats();
+                            if(stats.pending||stats.listeners||stats.handles||stats.subscriptions)
+                              throw new Error('Bulk console worker retained resources.');}
+                        }));
+                        return results;
+                      }finally{failed=true;for(const app of active)app.stop('error');}
+                    }''', {'host': selected['engine']['entrypoint'].replace('interpreter-worker.mjs', 'interpreter-host.mjs'),
+                           'config': config, 'programs': programs})
+                    assert len(results) == len(programs)
+                    transport['worker'] += len(results)
+                    for program, expected, actual in zip(programs, expectations, results, strict=True):
+                        assert actual == expected, (program, expected, actual)
                         cases.append({'source_sha256': hashlib.sha256(program.encode()).hexdigest(), 'stdout': expected, 'status': 'pass'})
                     for corpus in [native_directory_closure, wrapper_super_directories, advertised_directories, integer_identity, native_namespaces, class_mapping_proxies]:
                         for case in corpus['cases']:
@@ -1434,7 +1474,8 @@ def refused(source, prefix):
     return {'schema_version': 1, 'suite': 'class-console-browser/1', 'source_execution': True,
             'language_profile': 'classes-source/1', 'browser': browser_version, 'engine': selected['engine'],
             'script_sha256': hashlib.sha256(Path(__file__).read_bytes().replace(b'\r\n', b'\n')).hexdigest(),
-            'cases': cases, 'checks': checks, 'stop_ms': stopped,
+            'cases': cases, 'checks': checks, 'stop_ms': stopped, 'transport': transport,
+            'seconds': round(time.perf_counter() - started, 3),
             'summary': {'pass': len(cases) + len(checks), 'fail': 0}}
 
 

scripts/console_cases.py

@@ -0,0 +1,30 @@
+"""Resolve bulk console expectations from the independently validated corpus."""
+
+from functools import lru_cache
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+@lru_cache
+def reference_stdout(source):
+    return subprocess.check_output([sys.executable, '-I', '-S', '-X', 'utf8', '-c', source],
+                                   text=True, encoding='utf-8', timeout=5)
+
+
+def program_expectations(programs, corpus_root=ROOT / 'tests/fixtures/interpreter'):
+    recorded = {}
+    for path in sorted(corpus_root.glob('*.json')):
+        corpus = json.loads(path.read_bytes())
+        for case in corpus.get('cases', []):
+            if ('source' not in case or 'stdout' not in case or 'error_type' in case or
+                    case.get('modules') or case.get('stage') not in corpus.get('implemented_stages', [])):
+                continue
+            source, stdout = case['source'], case['stdout']
+            if source in recorded and recorded[source] != stdout:
+                raise ValueError('Conflicting corpus expectations for identical source')
+            recorded[source] = stdout
+    return [recorded[source] if source in recorded else reference_stdout(source) for source in programs]

tests/test_console_cases.py

@@ -0,0 +1,40 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from scripts.console_cases import program_expectations, reference_stdout
+
+
+class ConsoleCasesTests(unittest.TestCase):
+    def test_bulk_expectations_reuse_adopted_cases_and_reference_only_new_examples(self):
+        corpus = {'implemented_stages': ['C1'], 'cases': [
+            {'source': 'adopted', 'stage': 'C1', 'stdout': '7\n'},
+            {'source': 'later', 'stage': 'Future', 'stdout': 'future\n'},
+            {'source': 'error', 'stage': 'C1', 'stdout': 'prefix\n', 'error_type': 'ValueError'},
+            {'source': 'module', 'stage': 'C1', 'stdout': 'module\n', 'modules': [{'name': 'app'}]},
+        ]}
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            (root / 'one.json').write_text(json.dumps(corpus), encoding='utf-8')
+            with patch('scripts.console_cases.reference_stdout', side_effect=lambda source: source + '\n') as oracle:
+                observed = program_expectations(['adopted', 'new', 'later', 'error', 'module', 'adopted'], root)
+            self.assertEqual(observed, ['7\n', 'new\n', 'later\n', 'error\n', 'module\n', '7\n'])
+            self.assertEqual([call.args[0] for call in oracle.call_args_list], ['new', 'later', 'error', 'module'])
+            (root / 'two.json').write_text(json.dumps({'implemented_stages': ['C1'], 'cases': [
+                {'source': 'adopted', 'stage': 'C1', 'stdout': 'different'}]}), encoding='utf-8')
+            with self.assertRaisesRegex(ValueError, 'Conflicting'):
+                program_expectations(['adopted'], root)
+
+    def test_examples_run_isolated_cpython_only_once_per_source(self):
+        reference_stdout.cache_clear()
+        try:
+            with patch('scripts.console_cases.subprocess.check_output', return_value='7\n') as oracle:
+                self.assertEqual(reference_stdout('print(7)'), '7\n')
+                self.assertEqual(reference_stdout('print(7)'), '7\n')
+                oracle.assert_called_once()
+                self.assertIn('-I', oracle.call_args.args[0])
+                self.assertEqual(oracle.call_args.kwargs['timeout'], 5)
+        finally:
+            reference_stdout.cache_clear()