From 47af78ffd0464f7494fb1808fe0414f20c744f4b Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Thu, 22 Oct 2026 08:32:24 +0800 Subject: [PATCH 1/9] test(ci): pin equivalence CTest row and prerequisite contract (CR-223-ROW) --- scripts/test_ci_verify.py | 151 +++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index b8e19cd9a..d580ebe21 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -892,7 +892,11 @@ def test_live_tsan_is_separate_and_requires_complete_runner_coverage(self): with self.assertRaisesRegex(ConfigError, 'must run every runner row'): self.definitions('live-tsan', ['--exclude-label', 'slow']) self.assertIn('native_live_routing_e2e', ci_verify.runner_e2e_test_names(ROOT)) - self.assertNotIn('native_live_equivalence_e2e', ci_verify.runner_e2e_test_names(ROOT)) + # The prerequisite-gated row is registered, but public native-live + # does not provision its corpus. Keep the mandatory inventory honest. + self.assertEqual(ci_verify.runner_e2e_test_names(ROOT), + {path.stem for path in (ROOT / 'tests').glob('native_live*_e2e.py')} + - {'native_live_equivalence_e2e'}) def test_kernel_drops_the_source_layer_and_keeps_the_live_runner(self): values, argv = self.definitions('kernel') @@ -4120,5 +4124,150 @@ def test_failed_and_skipped_labelled_rows_keep_their_counts(self): self.assertEqual(rows.not_run, ('labelled_not_run',), output) +class NativeLiveEquivalenceCTest(unittest.TestCase): + """CR-223-ROW: source wiring and compiler-free adapter controls. + + The tiny driver below is a fixture, never the corpus e2e or a compiler. + The supervisor separately proves the registered row with the real driver. + """ + + def test_registered_row_has_build_tree_outputs_and_status_only_skip(self): + cmake = (ROOT / 'runner' / 'CMakeLists.txt').read_text() + row = re.search(r'add_test\(NAME native_live_equivalence_e2e\s+(.*?)\)', + cmake, re.S) + self.assertIsNotNone(row, 'the equivalence driver needs a permanent runner CTest row') + command = row.group(1) + self.assertIn('${Python3_EXECUTABLE} -B', command) + self.assertIn('${PROJECT_SOURCE_DIR}/tests/native_live_equivalence_ctest.py', command) + self.assertIn('--script ${PROJECT_SOURCE_DIR}/tests/native_live_equivalence_e2e.py', command) + self.assertIn('--build-dir ${PROJECT_BINARY_DIR}', command) + for option in ('--libraries-dir', '--out'): + self.assertRegex(command, re.escape(option) + r'\s+\$\{CMAKE_CURRENT_BINARY_DIR\}/') + properties = re.search( + r'set_tests_properties\(native_live_equivalence_e2e\s+PROPERTIES\s+(.*?)\)', + cmake, re.S) + self.assertIsNotNone(properties) + self.assertRegex(properties.group(1), r'\bSKIP_RETURN_CODE\s+77\b') + self.assertRegex(properties.group(1), r'\bLABELS\s+private_fixtures\b') + self.assertNotRegex(properties.group(1), + r'\b(SKIP_REGULAR_EXPRESSION|PASS_REGULAR_EXPRESSION|WILL_FAIL|DISABLED)\b') + self.assertIn('native_live_equivalence_e2e', ci_verify.private_fixture_rows(ROOT)) + # The same runtime preload/finalization contract must reach both + # ctypes witnesses in ASan and TSan builds. + self.assertRegex(cmake, r'set_tests_properties\(native_live_genuine_retry_e2e\s+' + r'native_live_equivalence_e2e\s+PROPERTIES ENVIRONMENT') + + def test_exclusion_reason_is_next_to_the_filter_and_in_ci_inventory_docs(self): + source = (ROOT / 'scripts' / 'ci_verify.py').read_text() + function = source.split('def runner_e2e_test_names(', 1)[1].split('\n\n', 1)[0] + self.assertIn('native-live.yml', function) + self.assertIn('corpus', function) + self.assertIn('docs/ci.md', function) + docs = (ROOT / 'docs' / 'ci.md').read_text() + self.assertIn('`native_live_equivalence_e2e`', docs) + self.assertIn('runner_e2e_test_names()', docs) + self.assertNotIn('corpus equivalence driver is not a runner CTest row', docs) + + def test_conditional_row_does_not_lower_or_inflate_existing_live_run_floor(self): + # This row can SKIP; the pre-existing public rows must all still run. + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) + for transport_present in (False, True): + if transport_present: + (source / 'runner').mkdir() + (source / 'runner' / 'transport.cpp').write_text('// fixture\n') + for overlay in ('', '1'): + with self.subTest(transport=transport_present, overlay=overlay), \ + mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': overlay}): + for profile in ci_verify.LIVE_ONLY_PROFILES: + self.assertEqual(ci_verify.profile_min_tests(ci_verify.PROFILE[profile], source), + 12 if transport_present else 11) + + def fixture(self, temporary, *, missing=None, exit_code=0, exception=False): + root = Path(temporary) / 'source' + build = Path(temporary) / 'build' + script = root / 'tests' / 'native_live_equivalence_e2e.py' + script.parent.mkdir(parents=True) + script.write_text( + "PROBES = {'probe-alpha': {'market'}, 'probe-beta': {'limit'}}\n" + "import argparse, json\nfrom pathlib import Path\n" + "parser = argparse.ArgumentParser()\n" + "for option in ('--root', '--build-dir', '--libraries-dir', '--out'):\n" + " parser.add_argument(option, required=True)\n" + "args = parser.parse_args()\n" + "Path(args.out).mkdir(parents=True, exist_ok=False)\n" + "Path(args.libraries_dir).mkdir(parents=True, exist_ok=False)\n" + "print('DRIVER_EXECUTED ' + json.dumps(vars(args)), flush=True)\n" + + ("raise RuntimeError('driver fixture failure')\n" if exception + else f"raise SystemExit({exit_code})\n")) + paths = [ + build / 'bin' / 'pineforge-live', build / 'compile_commands.json', + build / 'lib' / 'libpineforge.a', + root / 'corpus' / 'data' / 'ohlcv_ETH-USDT-USDT_1m.csv', + root / 'tests' / 'native_live_receipts.py', + *(root / 'corpus' / 'validation' / probe / 'generated.cpp' + for probe in ('probe-alpha', 'probe-beta')), + ] + for path in paths: + if missing is None or not path.as_posix().endswith(missing): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('// prerequisite fixture only\n') + command = [sys.executable, '-B', str(ROOT / 'tests' / 'native_live_equivalence_ctest.py'), + '--script', str(script), '--root', str(root), '--build-dir', str(build), + '--libraries-dir', str(build / 'libraries'), '--out', str(build / 'results')] + return command + + def invoke_fixture(self, command): + return subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env={key: value for key, value in os.environ.items() + if key not in ('LD_PRELOAD', 'DYLD_INSERT_LIBRARIES')}) + + def test_each_missing_prerequisite_skips_once_without_executing_driver(self): + for missing in ('bin/pineforge-live', 'compile_commands.json', 'lib/libpineforge.a', + 'corpus/data/ohlcv_ETH-USDT-USDT_1m.csv', 'tests/native_live_receipts.py', + 'corpus/validation/probe-alpha/generated.cpp', + 'corpus/validation/probe-beta/generated.cpp'): + with self.subTest(missing=missing), tempfile.TemporaryDirectory() as temporary: + result = self.invoke_fixture(self.fixture(temporary, missing=missing)) + self.assertEqual(result.returncode, 77, result.stdout + result.stderr) + self.assertEqual(len(result.stdout.splitlines()), 1) + self.assertTrue(result.stdout.startswith( + 'SKIP native_live_equivalence_e2e prerequisites absent: '), result.stdout) + self.assertIn(missing, result.stdout) + self.assertNotIn('DRIVER_EXECUTED', result.stdout) + + def test_ready_driver_executes_and_runtime_failures_never_become_skips(self): + for exit_code in (0, 1, 2, 77): + with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as temporary: + result = self.invoke_fixture(self.fixture(temporary, exit_code=exit_code)) + self.assertIn('DRIVER_EXECUTED ', result.stdout, result.stdout + result.stderr) + self.assertEqual(result.returncode, 1 if exit_code == 77 else exit_code, + result.stdout + result.stderr) + self.assertNotIn('SKIP ', result.stdout) + with tempfile.TemporaryDirectory() as temporary: + result = self.invoke_fixture(self.fixture(temporary, exception=True)) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(result.returncode, 77) + self.assertIn('driver fixture failure', result.stderr) + + def test_reruns_retain_distinct_outputs_and_libraries_under_build_tree(self): + with tempfile.TemporaryDirectory() as temporary: + command = self.fixture(temporary) + receipts = [] + for _ in range(2): + result = self.invoke_fixture(command) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + receipt = json.loads(next(line.removeprefix('DRIVER_EXECUTED ') + for line in result.stdout.splitlines() + if line.startswith('DRIVER_EXECUTED '))) + for option in ('out', 'libraries_dir'): + self.assertTrue(Path(receipt[option]).is_relative_to(Path(temporary) / 'build')) + self.assertTrue(Path(receipt[option]).is_dir()) + receipts.append(receipt) + for option in ('out', 'libraries_dir'): + self.assertNotEqual(receipts[0][option], receipts[1][option]) + self.assertTrue(Path(receipts[0][option]).is_dir()) + + if __name__ == '__main__': unittest.main() From 451a34985119df7f45e9fdbad26d89c45721a2aa Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Thu, 22 Oct 2026 08:38:26 +0800 Subject: [PATCH 2/9] test(ci): require equivalence execution and corpus LFS provisioning --- scripts/test_ci_verify.py | 96 +++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index d580ebe21..ba54f10bd 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -232,6 +232,8 @@ def __call__(self, argv, *, extra_env=None, combine_stderr=True, if 'status' in argv: prefix = str(self.exits.get('corpus-submodule-prefix', ' ')) return Completed(0, (prefix + 'b46cd80c247a53b19e23cb0c12c4451d624ce9a6 corpus\n').encode(), b'') + if argv[0] == 'git' and 'lfs' in argv: + return Completed(int(self.exits.get('corpus-live-feed', 0)), b'corpus LFS feed\n', b'') if argv[0] == 'git' and 'cat-file' in argv: if V18_FROZEN_COMMIT + '^{commit}' in argv: key = 'v18-frozen-cat-file' @@ -892,11 +894,9 @@ def test_live_tsan_is_separate_and_requires_complete_runner_coverage(self): with self.assertRaisesRegex(ConfigError, 'must run every runner row'): self.definitions('live-tsan', ['--exclude-label', 'slow']) self.assertIn('native_live_routing_e2e', ci_verify.runner_e2e_test_names(ROOT)) - # The prerequisite-gated row is registered, but public native-live - # does not provision its corpus. Keep the mandatory inventory honest. + # Every driver, including corpus equivalence, must be a required row. self.assertEqual(ci_verify.runner_e2e_test_names(ROOT), - {path.stem for path in (ROOT / 'tests').glob('native_live*_e2e.py')} - - {'native_live_equivalence_e2e'}) + {path.stem for path in (ROOT / 'tests').glob('native_live*_e2e.py')}) def test_kernel_drops_the_source_layer_and_keeps_the_live_runner(self): values, argv = self.definitions('kernel') @@ -1494,6 +1494,14 @@ def test_uninitialized_corpus_status_fails_before_configure(self): self.assertIn('corpus-submodule-pin', failure_stages(summary)) self.assertFalse(any(argv[0] == 'cmake' and '-S' in argv for argv in scripted.calls)) + def test_live_corpus_lfs_failure_stops_before_guards_and_configure(self): + code, summary, scripted, _ = self.run_profile( + 'live-sanitizers', **{'corpus-live-feed': 1}) + self.assertEqual(code, 1) + self.assertIn('corpus-live-feed', failure_stages(summary)) + self.assertNotIn('configure', scripted.names()) + self.assertNotIn('source-guard-c-abi', stage_names(summary)) + def test_build_failure_skips_ctest_and_install(self): code, summary, scripted, _ = self.run_profile(build=1) self.assertEqual(code, 1) @@ -3387,18 +3395,21 @@ def run_profile(self, profile='live-sanitizers', extra=None, **exits): self, profile, extra=extra, **exits) def pinned_probe(self, count: int) -> list: - pinned = sorted(ci_verify.private_fixture_rows(ROOT)) + # Required e2e rows must execute even when other pinned rows skip. + pinned = sorted(ci_verify.private_fixture_rows(ROOT) + - ci_verify.runner_e2e_test_names(ROOT)) self.assertGreaterEqual(len(pinned), count) return pinned[:count] def test_green_the_absent_runner_placeholders_skip_and_the_profile_passes(self): # Control (a): the public native-live shape. The runner/ tree - # registers its 65 pinned placeholders beside the 12 rows it runs; + # registers its 65 old pinned placeholders beside the required rows; # they skip, ran stays at the floor, and the profile passes with the # acceptance printed. pinned = self.pinned_probe(65) code, summary, _, build_dir = self.run_profile( - 'live-sanitizers', ctest_registered=77, ctest_rows=77, + 'live-sanitizers', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS, + ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS, ctest_skipped=pinned) self.assertEqual(code, 0, summary['failures']) self.assertEqual(summary['ctestRows'], LIVE_SANITIZERS_MIN_TESTS) @@ -3412,7 +3423,8 @@ def test_red_an_unpinned_runner_skip_fails(self): # though the arithmetic alone would tolerate it. pinned = self.pinned_probe(65) code, summary, _, _ = self.run_profile( - 'live-sanitizers', ctest_registered=78, ctest_rows=78, + 'live-sanitizers', ctest_registered=66 + LIVE_SANITIZERS_MIN_TESTS, + ctest_rows=66 + LIVE_SANITIZERS_MIN_TESTS, ctest_skipped=pinned + ['test_unpinned_runner_row']) self.assertEqual(code, 1) self.assertIn('live-test-skips', failure_stages(summary)) @@ -3426,7 +3438,8 @@ def test_red_a_not_run_row_fails_with_the_pinned_skips(self): # pinned skips -- the acceptance covers skips, nothing else. pinned = self.pinned_probe(65) code, summary, _, _ = self.run_profile( - 'live-sanitizers', ctest_registered=78, ctest_rows=78, + 'live-sanitizers', ctest_registered=66 + LIVE_SANITIZERS_MIN_TESTS, + ctest_rows=66 + LIVE_SANITIZERS_MIN_TESTS, ctest_skipped=pinned, ctest_not_run=['test_live_not_run_row']) self.assertEqual(code, 1) self.assertIn('live-test-skips', failure_stages(summary)) @@ -3439,7 +3452,8 @@ def test_red_a_disabled_row_fails_with_the_pinned_skips(self): # own count, so only this rule can refuse it. pinned = self.pinned_probe(65) code, summary, _, _ = self.run_profile( - 'live-sanitizers', ctest_registered=77, ctest_rows=77, + 'live-sanitizers', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS, + ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS, ctest_skipped=pinned, ctest_disabled=['test_live_disabled_row']) self.assertEqual(code, 1) self.assertIn('live-test-skips', failure_stages(summary)) @@ -3450,7 +3464,8 @@ def test_red_overlay_present_a_pinned_skip_still_fails(self): # private-fixtures rule and on this one. with mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': '1'}): code, summary, _, _ = self.run_profile( - 'live-sanitizers', ctest_registered=13, ctest_rows=13, + 'live-sanitizers', ctest_registered=LIVE_SANITIZERS_MIN_TESTS + 1, + ctest_rows=LIVE_SANITIZERS_MIN_TESTS + 1, ctest_skipped=self.pinned_probe(1)) self.assertEqual(code, 1) self.assertIn('live-test-skips', failure_stages(summary)) @@ -3464,11 +3479,29 @@ def test_green_live_tsan_carries_the_same_rule(self): # placeholders the same way. pinned = self.pinned_probe(65) code, summary, _, _ = self.run_profile( - 'live-tsan', ctest_registered=77, ctest_rows=77, + 'live-tsan', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS, + ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS, ctest_skipped=pinned) self.assertEqual(code, 0, summary['failures']) self.assertIn('live-test-skips', stage_names(summary)) + def test_required_equivalence_skip_fails_even_when_other_rows_cover_the_floor(self): + # Inventory membership and a healthy total cannot substitute for this + # required driver's execution, even though its skip is a pinned one. + for profile in ci_verify.LIVE_ONLY_PROFILES: + with self.subTest(profile=profile): + code, summary, _, _ = self.run_profile( + profile, ctest_registered=LIVE_SANITIZERS_MIN_TESTS + 1, + ctest_rows=LIVE_SANITIZERS_MIN_TESTS + 1, + ctest_skipped=['native_live_equivalence_e2e']) + self.assertEqual(code, 1) + self.assertEqual(summary['ctestRows'], LIVE_SANITIZERS_MIN_TESTS) + self.assertIn('live-test-skips', failure_stages(summary)) + error = next(item['error'] for item in summary['failures'] + if item['stage'] == 'live-test-skips') + self.assertIn('required runner e2e rows skipped', error) + self.assertIn('native_live_equivalence_e2e', error) + def test_the_native_profile_is_not_under_this_rule(self): # Control (e): the native profile runs runner rows too but is not a # live-only profile; its shape is unchanged -- the runner-skip rule @@ -4157,7 +4190,7 @@ def test_registered_row_has_build_tree_outputs_and_status_only_skip(self): self.assertRegex(cmake, r'set_tests_properties\(native_live_genuine_retry_e2e\s+' r'native_live_equivalence_e2e\s+PROPERTIES ENVIRONMENT') - def test_exclusion_reason_is_next_to_the_filter_and_in_ci_inventory_docs(self): + def test_inventory_decision_and_prerequisites_are_documented(self): source = (ROOT / 'scripts' / 'ci_verify.py').read_text() function = source.split('def runner_e2e_test_names(', 1)[1].split('\n\n', 1)[0] self.assertIn('native-live.yml', function) @@ -4168,8 +4201,8 @@ def test_exclusion_reason_is_next_to_the_filter_and_in_ci_inventory_docs(self): self.assertIn('runner_e2e_test_names()', docs) self.assertNotIn('corpus equivalence driver is not a runner CTest row', docs) - def test_conditional_row_does_not_lower_or_inflate_existing_live_run_floor(self): - # This row can SKIP; the pre-existing public rows must all still run. + def test_mandatory_row_raises_live_run_floor_in_both_fixture_worlds(self): + # All twelve existing public rows and the equivalence row must RUN. with tempfile.TemporaryDirectory() as temporary: source = Path(temporary) for transport_present in (False, True): @@ -4181,7 +4214,34 @@ def test_conditional_row_does_not_lower_or_inflate_existing_live_run_floor(self) mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': overlay}): for profile in ci_verify.LIVE_ONLY_PROFILES: self.assertEqual(ci_verify.profile_min_tests(ci_verify.PROFILE[profile], source), - 12 if transport_present else 11) + 13 if transport_present else 12) + + def test_live_feed_materialization_follows_profile_and_is_lfs_explicit(self): + # Exercise the prerequisite method alone: no guards, configure or + # compiler is invoked by this control. + for profile in ci_verify.PROFILES: + with self.subTest(profile=profile), tempfile.TemporaryDirectory() as temporary: + config = validate_config(parse_args( + [profile, '--build-dir', temporary], source=ROOT), source=ROOT) + driver = ci_verify.Driver(config) + calls = [] + + def invoke(stage, argv): + calls.append((stage, argv)) + return Completed(0, b' 0123456789abcdef corpus\n', b'') + + with mock.patch.object(driver, 'invoke', side_effect=invoke): + self.assertTrue(driver.ensure_corpus_submodule()) + lfs = [argv for stage, argv in calls if stage == 'corpus-live-feed'] + expected = config.profile.live_runner and config.profile.source_layer + self.assertEqual(bool(lfs), expected) + if expected: + self.assertEqual(lfs, [[ + 'git', '-C', str(ROOT / 'corpus'), 'lfs', 'pull', + '--include=data/ohlcv_ETH-USDT-USDT_1m.csv', '--exclude=']]) + workflow = (ROOT / '.github' / 'workflows' / 'native-live.yml').read_text() + install = next(line for line in workflow.splitlines() if 'apt-get install' in line) + self.assertIn('git-lfs', install.split()) def fixture(self, temporary, *, missing=None, exit_code=0, exception=False): root = Path(temporary) / 'source' @@ -4237,11 +4297,11 @@ def test_each_missing_prerequisite_skips_once_without_executing_driver(self): self.assertNotIn('DRIVER_EXECUTED', result.stdout) def test_ready_driver_executes_and_runtime_failures_never_become_skips(self): - for exit_code in (0, 1, 2, 77): + for exit_code in (0, 1, 2, 77, 256, -256): with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as temporary: result = self.invoke_fixture(self.fixture(temporary, exit_code=exit_code)) self.assertIn('DRIVER_EXECUTED ', result.stdout, result.stdout + result.stderr) - self.assertEqual(result.returncode, 1 if exit_code == 77 else exit_code, + self.assertEqual(result.returncode, 0 if exit_code == 0 else 1, result.stdout + result.stderr) self.assertNotIn('SKIP ', result.stdout) with tempfile.TemporaryDirectory() as temporary: From 0359213e700b6bc3695cc255bd6d9290804a83fc Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Thu, 22 Oct 2026 08:42:52 +0800 Subject: [PATCH 3/9] test(ci): extend verifier partition and native row accounting --- scripts/test_ci_verify.py | 15 ++++++++------- scripts/test_run_unittest_shards.py | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index ba54f10bd..5944072c3 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -1793,7 +1793,7 @@ def test_ctest_label_exclusion_is_forwarded(self): def test_pr_exclusion_proves_registered_minus_labelled_equals_ran(self): self.assertEqual(ci_verify.EXCLUDED_REGISTERED_MIN, - {'debug': 876, 'sanitizers': 867, 'native': 963, + {'debug': 876, 'sanitizers': 867, 'native': 964, 'release': 895, 'kernel': 337}) for profile, registered in ci_verify.EXCLUDED_REGISTERED_MIN.items(): with self.subTest(profile=profile): @@ -2828,7 +2828,7 @@ def test_the_table_is_the_floor_and_profiles_outside_it_keep_their_own(self): self.assertEqual(validate_config(parse_args(['kernel'])).min_tests, FULL_FLOOR['kernel']) self.assertEqual(ci_verify.EXCLUDED_REGISTERED_MIN, - {'debug': 876, 'sanitizers': 867, 'native': 963, + {'debug': 876, 'sanitizers': 867, 'native': 964, 'release': 895, 'kernel': 337}) def test_overlay_flag_moves_the_floor_and_is_the_only_input(self): @@ -3377,10 +3377,10 @@ class LiveProfilesWithFixturesAbsent(unittest.TestCase): skip; without the overlay runner/ registers its pinned private-fixture placeholders and they SKIP, so the old rule failed the required native-live job of every public PR (.github/workflows/ci.yml's build - gate). The branch now accepts exactly the pinned skips -- every skipped - row must be a pinned private-fixture row -- while an unpinned skip, a - not-run row and a disabled row still fail, and with the overlay present - the old rule stands: nothing may skip. + gate). The branch accepts pinned skips outside the required e2e inventory; + every skipped row must be a pinned private-fixture row. A required e2e + skip, an unpinned skip, a not-run row and a disabled row still fail. With + the overlay present the old rule stands: nothing may skip. """ def run_profile(self, profile='live-sanitizers', extra=None, **exits): @@ -4321,7 +4321,8 @@ def test_reruns_retain_distinct_outputs_and_libraries_under_build_tree(self): for line in result.stdout.splitlines() if line.startswith('DRIVER_EXECUTED '))) for option in ('out', 'libraries_dir'): - self.assertTrue(Path(receipt[option]).is_relative_to(Path(temporary) / 'build')) + self.assertTrue(Path(receipt[option]).is_relative_to( + (Path(temporary) / 'build').resolve())) self.assertTrue(Path(receipt[option]).is_dir()) receipts.append(receipt) for option in ('out', 'libraries_dir'): diff --git a/scripts/test_run_unittest_shards.py b/scripts/test_run_unittest_shards.py index 9bcca8b19..3bd5f89b2 100644 --- a/scripts/test_run_unittest_shards.py +++ b/scripts/test_run_unittest_shards.py @@ -112,6 +112,7 @@ def test_verifier_classes_are_an_exhaustive_disjoint_partition(self) -> None: 'PrExclusionWithFixturesAbsent', 'LiveProfilesWithFixturesAbsent', 'FloorDocsPointAtTheTable', 'ExamplesAssertLive', 'StaleBinaries', 'ReceiptRecipe', 'DiagnosticsCollection', 'CTestDeadlines', + 'NativeLiveEquivalenceCTest', }) self.assertEqual(verifier_classes.FAST_CLASSES & verifier_classes.SLOW_CLASSES, set()) self.assertEqual(verifier_classes.SLOW_CLASSES, { @@ -119,6 +120,7 @@ def test_verifier_classes_are_an_exhaustive_disjoint_partition(self) -> None: 'PrivateFixtureFloors', 'PrExclusionWithFixturesAbsent', 'LiveProfilesWithFixturesAbsent', 'ExamplesAssertLive', 'StaleBinaries', 'ReceiptRecipe', + 'NativeLiveEquivalenceCTest', }) self.assertEqual(verifier_classes.FAST_CLASSES | verifier_classes.SLOW_CLASSES, discovered) verifier_classes.validate_class_partition(discovered) From 4d948365ce795b493b8b3875a9cd777014a92f1f Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Thu, 22 Oct 2026 08:44:01 +0800 Subject: [PATCH 4/9] ci(live): run corpus equivalence as a required CTest row (CR-223-ROW) The corpus equivalence driver becomes a permanent runner CTest row, and the runner-e2e equivalence exclusion leaves `ci_verify.py`. Two sentences of `docs/ci.md` that dbd17b3 ("Live runner: normalized feed input and per-action webhook routing (#320)") introduced are retired here because this change makes them false, and the message names them so the doc-revert gate can read the retirement: - "The standalone corpus equivalence driver is not a runner CTest row; its harness unit tests remain in the engine-wide inventory." The driver is now the `native_live_equivalence_e2e` row in `runner/CMakeLists.txt`, and the same sentence's second clause (the harness unit tests staying in the engine-wide inventory) is kept verbatim in the replacement text. - "It refuses label exclusions, missing runner compile commands, missing ASan/UBSan or frame-pointer flags, skipped/disabled tests, and, while `runner/transport.cpp` exists, a missing WebSocket row or transport skip even without `--require-websocket`." Its "skipped/disabled tests" clause is rewritten because a required runner e2e skip now fails outright; every other clause is kept in the replacement paragraph. Restored on purpose: nothing. The rest of the diff only adds text. --- .github/workflows/native-live.yml | 2 +- docs/ci.md | 76 +++++++++++++++--- runner/CMakeLists.txt | 17 +++- scripts/ci_verifier_classes.py | 1 + scripts/ci_verify.py | 21 +++-- tests/native_live_equivalence_ctest.py | 105 +++++++++++++++++++++++++ tests/private_fixture_rows.txt | 3 +- 7 files changed, 207 insertions(+), 18 deletions(-) create mode 100644 tests/native_live_equivalence_ctest.py diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 82582e3cc..54cecb90c 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -145,7 +145,7 @@ jobs: - name: Install native build dependencies run: | sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev tzdata-legacy libsqlite3-dev libssl-dev zlib1g-dev ccache + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev tzdata-legacy libsqlite3-dev libssl-dev zlib1g-dev ccache git-lfs - name: Restore ccache id: ccache-restore diff --git a/docs/ci.md b/docs/ci.md index 2e5afda16..3c1d6f449 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -110,17 +110,19 @@ curl may report the existing unsupported-WebSocket skip; that is not the require Linux transport proof. `live-sanitizers` performs a full all-target build, then runs the complete -`build-ci-live-sanitizers/runner` CTest inventory (at least twelve rows while -`runner/transport.cpp` exists, eleven after its removal), including +`build-ci-live-sanitizers/runner` CTest inventory (at least thirteen rows while +`runner/transport.cpp` exists, twelve after its removal), including `native_live_e2e` and `native_live_startup_e2e`, and requires any additional -runner `tests/native_live*_e2e.py` on the tree to be registered too. The standalone -corpus equivalence driver is not a runner CTest row; its harness unit tests remain -in the engine-wide inventory. The profile does not rerun +runner `tests/native_live*_e2e.py` on the tree to be registered too, including the +corpus equivalence driver described below. Its harness unit tests +remain in the engine-wide inventory. The profile does not rerun the engine-wide CTest set or prepare historical ABI providers; those remain covered by `sanitizers`. It refuses label exclusions, missing runner compile -commands, missing ASan/UBSan or frame-pointer flags, skipped/disabled tests, -and, while `runner/transport.cpp` exists, a missing WebSocket row or transport -skip even without `--require-websocket`. After transport removal those two +commands, missing ASan/UBSan or frame-pointer flags, disabled tests and skipped +required e2e rows. Other unpinned skips also fail (with the fixture overlay +present, every skip fails). While `runner/transport.cpp` exists, a missing +WebSocket row or transport skip fails even without `--require-websocket`. +After transport removal those two automatic gates no longer apply; an explicit `--require-websocket` still requires the WebSocket test. The runner's curl version floor still applies at configure time. @@ -134,6 +136,60 @@ cancel its peers. The `native-live-gate` job requires the matrix to succeed; the caller's `build` aggregate still requires the reusable workflow to succeed. Only the native cell accepts the caller's existing PR-only slow-label exclusion. +`native_live_equivalence_e2e` is a permanent CTest row in `runner/CMakeLists.txt`, +registered when tests, the source layer and the live runner are enabled. It runs +`tests/native_live_equivalence_e2e.py` through a prerequisite adapter, using the +configured live build and fresh, retained libraries/results directories under +`/runner/native_live_equivalence_e2e/`. It executes the driver's default +eight probes and full scenario population: observer qualification, exact +batch/stream/loopback action and report comparisons, restart, delivery retry and +state-selected splits. It uses a loopback receiver with an ephemeral port and +needs no external venue. The previously measured c5.2xlarge run took about 929 s +to build and 983 s for the script (109 scenarios, zero failures). Those are +observations from the CR-223 proof, not timings measured by this registration. + +The row needs `bin/pineforge-live`, `compile_commands.json`, `lib/libpineforge.a` +and the built `native_live_equivalence_observer` object in the selected live +build; the corpus's `data/ohlcv_ETH-USDT-USDT_1m.csv` and +`validation//generated.cpp` for every probe the driver declares; and +`tests/native_live_receipts.py`. The receipt helper is tracked in the engine +repository at base `c32c73238`; no private-fixtures overlay is needed for it. +A tree or overlay that omits it still receives the explicit prerequisite skip. +The adapter reads the probe population from the driver without importing that +helper. A missing binary, compile database, archive, feed, generated probe or +receipt helper prints one reason line starting +`SKIP native_live_equivalence_e2e prerequisites absent: ` followed by all missing +paths, and returns 77. CTest reports **Skipped**, never Passed. A malformed +driver/compile database, a missing observer object, a runtime failure, an exception +or a failed sanitizer check fails the row; a driver exit of 77 after the checks +is a failure too. ASan/TSan builds use the same runtime preload and status-bearing +cleanup as the genuine-retry witness, including cleanup on the skip path. + +The row carries the existing `private_fixtures` label and is added to +`tests/private_fixture_rows.txt`. The live profiles select the runner directory, +so this label does not hide it from either sanitizer profile or the native +profile's `slow` exclusion. Direct CTest records missing prerequisites as a +skip. In `ci_verify.py`, both live sanitizer profiles reject a skip for any +required runner e2e row, even a pinned row and even without +`PINEFORGE_PRIVATE_FIXTURES`. Other pinned-placeholder skips keep their existing +policy. A skip is recorded in `ctestSkipped` and contributes zero to the run +count. The pinned list grows from 375 to 376; `LIVE_SANITIZERS_MIN_TESTS` grows +from 12 to 13, so `profile_min_tests()` returns 13 while transport exists and +12 after its removal, in both fixture worlds. The native PR registration floor +grows from 963 to 964. Release/kernel row floors are unchanged, since this row +needs the live runner and source layer together. + +The `runner_e2e_test_names()` equivalence exclusion is removed. Both live +sanitizer profiles require the row in their inventory and require it to execute; +inventory membership alone is not execution proof. Although the +`native-live.yml` checkout step does not initialize submodules, its verifier +already calls `ensure_corpus_submodule()` before source guards and configure. +That method initializes and checks the recorded corpus gitlink. Source-layer +live profiles now explicitly pull the LFS 1m feed, and `native-live.yml` +installs `git-lfs`, as the corpus-parity workflow already does. A failed fetch +stops verification before configure. With the build, corpus/feed and tracked +receipt helper present, the row runs without a private-fixtures overlay. + The kernel profile also gates the CTest row count: `tests/CMakeLists.txt` drops every test TU whose include closure reaches `pineforge/source/` or `compat/pine/`, so a lane whose native witnesses shared a TU with an adapter @@ -826,6 +882,8 @@ the one place the path list lives, reports that the change touches a path the verifier reads (`include/pineforge`, `src/c_abi.cpp`, `tests/`, `runner/`, the workflows and the CI scripts among them); an unknown or unreadable diff runs the SLOW classes. Push to `main` and `workflow_dispatch` run every class. +`NativeLiveEquivalenceCTest` is SLOW until its passing runtime is measured; +it checks the row wiring, prerequisite skips, driver failures and run-directory isolation. The CTest row `test_ci_verify` stays in the `slow` population: the pull-request legs exclude it by label and every full run executes it. @@ -836,7 +894,7 @@ the selection must remove some rows and leave some, and, with the overlay absent, the rows that ran plus the pinned skips must equal the selection with no other skip (the `--exclude-label` branch of `enforce_test_floor` in `scripts/ci_verify.py`). `EXCLUDED_REGISTERED_MIN` holds 876 for `debug`, 867 -for `sanitizers` and 963 for `native`, and with D1 895 for `release` and 337 +for `sanitizers` and 964 for `native`, and with D1 895 for `release` and 337 for `kernel`. The two new keys are derived, not measured. `release` is the registered count in the sampled public pull-request runs (957) less the 62 rows by which the Debug key sits below the Debug registry in the same runs (938 less diff --git a/runner/CMakeLists.txt b/runner/CMakeLists.txt index 00e30725c..b18ed5551 100644 --- a/runner/CMakeLists.txt +++ b/runner/CMakeLists.txt @@ -466,6 +466,18 @@ if(PINEFORGE_BUILD_TESTS AND PINEFORGE_BUILD_SOURCE_LAYER) if(_pf_genuine_retry_sanitizer) list(APPEND _pf_genuine_retry_sanitizer_args --sanitizer ${_pf_genuine_retry_sanitizer}) endif() + # This corpus-backed row belongs to the same runner inventory selected by + # live-sanitizers/live-tsan. Missing inputs are an explicit CTest SKIP; + # once the driver starts, its failures cannot take the skip exit code. + add_test(NAME native_live_equivalence_e2e COMMAND ${Python3_EXECUTABLE} -B + ${PROJECT_SOURCE_DIR}/tests/native_live_equivalence_ctest.py + --script ${PROJECT_SOURCE_DIR}/tests/native_live_equivalence_e2e.py + --root ${PROJECT_SOURCE_DIR} --build-dir ${PROJECT_BINARY_DIR} + --libraries-dir ${CMAKE_CURRENT_BINARY_DIR}/native_live_equivalence_e2e/libraries + --out ${CMAKE_CURRENT_BINARY_DIR}/native_live_equivalence_e2e/output + ${_pf_genuine_retry_sanitizer_args}) + set_tests_properties(native_live_equivalence_e2e PROPERTIES + SKIP_RETURN_CODE 77 LABELS private_fixtures) add_test(NAME native_live_genuine_retry_e2e COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/native_live_genuine_retry_e2e.py --runner $ --library $ @@ -483,7 +495,7 @@ if(PINEFORGE_BUILD_TESTS AND PINEFORGE_BUILD_SOURCE_LAYER) OUTPUT_VARIABLE _pf_genuine_retry_runtime OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _pf_genuine_retry_runtime_result) if(NOT _pf_genuine_retry_runtime_result EQUAL 0 OR NOT EXISTS "${_pf_genuine_retry_runtime}") - message(FATAL_ERROR "The genuine retry Python test requires the compiler's shared sanitizer runtime") + message(FATAL_ERROR "The live ctypes witnesses require the compiler's shared sanitizer runtime") endif() if(APPLE) set(_pf_genuine_retry_environment "DYLD_INSERT_LIBRARIES=${_pf_genuine_retry_runtime}") @@ -505,6 +517,7 @@ if(PINEFORGE_BUILD_TESTS AND PINEFORGE_BUILD_SOURCE_LAYER) else() list(APPEND _pf_genuine_retry_environment "TSAN_OPTIONS=halt_on_error=1:second_deadlock_stack=1") endif() - set_tests_properties(native_live_genuine_retry_e2e PROPERTIES ENVIRONMENT "${_pf_genuine_retry_environment}") + set_tests_properties(native_live_genuine_retry_e2e native_live_equivalence_e2e + PROPERTIES ENVIRONMENT "${_pf_genuine_retry_environment}") endif() endif() diff --git a/scripts/ci_verifier_classes.py b/scripts/ci_verifier_classes.py index 139c24137..05d77cd18 100644 --- a/scripts/ci_verifier_classes.py +++ b/scripts/ci_verifier_classes.py @@ -34,6 +34,7 @@ 'ExamplesAssertLive', 'StaleBinaries', 'ReceiptRecipe', + 'NativeLiveEquivalenceCTest', }) diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py index c5adf91d4..08fa22f57 100644 --- a/scripts/ci_verify.py +++ b/scripts/ci_verify.py @@ -644,7 +644,7 @@ # RELEASE_MIN_TESTS in each, counted the same way. # An excluded run must still discover at least this many rows before -LE. EXCLUDED_REGISTERED_MIN = { - 'debug': 876, 'sanitizers': 867, 'native': 963, + 'debug': 876, 'sanitizers': 867, 'native': 964, # CR-223-ROW adds one native runner row. # D1, run 39221717231 of the PR TSV, REGISTERED counts (Debug/sanitizers # exclude 29 slow rows, so selected + 29): Debug 938 - 876 = 62 and # sanitizers 929 - 867 = 62 below their registry; Release and kernel-only @@ -719,7 +719,7 @@ # 847/838/934). # The verifier supplies no stage or per-test deadline (owner rule, # 2026-10-12); the platform's job envelope is the only hang catcher. -LIVE_SANITIZERS_MIN_TESTS = 12 +LIVE_SANITIZERS_MIN_TESTS = 13 # CR-223-ROW adds the required corpus equivalence row; see docs/ci.md. # Overlay-aware row floors. The private-fixture campaign splits the tree in # two: the private-fixture PR removes the private fixture data, and P1-B # registers each affected ctest row as a SKIP placeholder in @@ -763,7 +763,7 @@ # at ci_verify.py:321 and ci_verify.py:595 do not move; the campaign's own # gate is PRIVATE_FIXTURE_PROFILE_FLOORS below, the one place its numbers # live. The class-fix round added the three remaining split parents (test_coof_close_slippage_tapes, test_cross_mechanism_tapes, test_margin_rules_forward_replay): each reads its fixture data at run time, the private-fixture removals take the whole directory it reads (the removed file beside distributed ones in PR 1, the remaining files in PR 1b), and the row had stayed unconditionally registered -- the fixture-less tree would have run it against absent data. Each now pins whole beside its *_private half on the same EXISTS guard (tests/CMakeLists.txt), exactly as test_source_buying_power_tapes already did at 324; a two-world registration sweep over tests/CMakeLists.txt and runner/CMakeLists.txt closes at `duplicate names: 0` in both worlds. test_callback_lifecycle_tapes joined at 123: its tapes.txt is a distributed list the private-fixture PR leaves in the tree, but 60 of the cases it names are removed, so tests/CMakeLists.txt registers the row as a placeholder while the listed cases are incomplete. One of the 324, test_run_json_outputs, is registered only by the release-shaped profiles (the source layer gates its block); the observed-skip lowering counts it only in the profiles it registers in, so the kernel profile is no longer lowered one row deeper than the rows it skips. Six more joined at 129 (the fifth proof-box round): test_native_oracle_day_key_l2 and test_native_oracle_reversal_same_bar_tx_l2 (their twin source #includes oracle/test_oracle_*.cpp), test_adapter_in_place_reads, test_adapter_runtime_ambient and test_adapter_settlement_carry (tests/pine_d2c_scenarios_fixture.hpp), and test_coof_forced_waypoint_tapes (tests/fixtures/coof_forced_waypoint/cases.inc) -- each target now exists only while its removed include does, so the compile that killed the fifth configure-through run cannot recur. The PR 1b batch moved the remaining 6,242 fixture/probe files (l50/pr1b-inventory.tsv) and took the count to 319: 134 rows in tests/CMakeLists.txt (each a row whose source .cpp, quoted #include or runtime fixture the removal takes, or one of five fixture-fed targets) and 53 in runner/CMakeLists.txt (the strategy-capabilities and confirmed-capabilities e2e rows), and 3 rows whose fixture data the sources read at run time through their fixture-dir defines (test_color_tapes, test_session_transition_clock_tapes, test_source_buying_power_tapes); PR 1's fix round then pinned five rows of its own that PR 1b's 319 does not carry -- test_carried_trail_open_tapes, pinned whole, and the *_private halves of test_coof_close_slippage_tapes, test_cross_mechanism_tapes, test_source_buying_power_tapes and test_margin_rules_forward_replay (each parent row keeps replaying the distributed part, the *_private row beside it runs the same binary with --private-only) -- and the union of the two pin sets, sorted and de-duplicated, is 324; the pinned file header still names the PR 1 manifest sha. The stream-security round took it to 361: tests/CMakeLists.txt registered the fifteen test_stream_security_ targets, from fixtures/stream_security//generated.cpp, inside an unguarded bare-form foreach -- the shape the box's run 1 died on at configure (add_executable could not find the removed source), and the shape the four class sweeps' line walker did not expand -- so each case now sits on its own if(EXISTS ...) and, absent, registers its two rows plus the four test_stream_security_control_* rows as pinned SKIP placeholders: 327 + 34 = 361. The kernel-python round (L6's S-73 two-mode round 4, class 4) took it to 364: three Python rows the kernel profile runs outside the placeholder loop now skip whole when their private input is gone instead of failing on read_text() -- test_native_cpp_abi (scripts/check_native_cpp_abi.py gates on tests/fixtures/native_cpp_abi/), test_pending_order_v1_full_prefix and test_pending_order_v1_compiler_prefix (the two check_pending_order_prefix* scripts gate on tests/fixtures/script_cpp_abi/basev8 and basev9) -- each printing the pinned literal and exiting 77 under SKIP_RETURN_CODE 77, each carrying LABELS private_fixtures so the pinned-vs-labelled self-test still closes at zero. The same round replaced the static lowering (floor minus PRIVATE_FIXTURE_ROW_COUNT, which read -36 for the kernel floor) with an observed-skip lowering, bounded then by a floor-minimum constant of 1; PR 1b's second pass (L5 01:43's ruling) replaced that bound with a per-profile margin. The fix round (R2-F1) replaced that whole machinery with PRIVATE_FIXTURE_PROFILE_FLOORS: the observed-skip lowering moved with the registry it was meant to check (release absent ran 603 against a floor of 602 after two of main's rows had already left the registry, and up to 99 kernel rows could vanish with the floor still met), and no margin repairs a floor that follows the run down. PR 1b's first pass, L5 01:37, took it back down to 347: it had also guarded the 17 legacy-owner base names (L3A_LEGACY_OWNER_TEST_SOURCES intersected with the private rows), which main never compiles -- only their _l4a/_l4b/_l4c/_l4d twins run -- and the fix dropped those guard blocks and moved this count and the pinned file together (364 -> 347), restoring main's registry. PR 1b's second pass (L5 01:43, class 3) took it to 351: the four receipt-gated rows in tests/CMakeLists.txt are now registered on an EXISTS guard over the seven ABI-provider manifests their providers gate on, so absent manifests make each the pinned SKIP placeholder (the literal, LABELS private_fixtures, SKIP_RETURN_CODE 77) and present manifests leave the registration exactly as it was -- check_abi_receipt_skips no longer counts them as failures whose input is simply absent, the preparer's own 77+literal is a skip rather than a failed preparation (ensure_prepared_provider accepts it), and the four skips are pinned accounting like every other private-fixture row's. The class-sweep round (the private-fixtures lane's iteration 2) took it to 354: scripts/check_private_fixture_reads.py found three rows (test_coof_marketable_stop_tapes, test_pooc_close_bar_fills_tapes, test_pooc_pending_stop_tapes) still registered by the TEST_SOURCES loop while every case each replays sits under a removed directory -- they ran on the fixture-less tree and aborted -- and each now pins whole on a removed file it reads, with the kernel-absent branch its source-bound TU needs. -PRIVATE_FIXTURE_ROW_COUNT = 375 +PRIVATE_FIXTURE_ROW_COUNT = 376 PRIVATE_FIXTURE_LABEL = 'private_fixtures' PRIVATE_FIXTURE_ROWS_FILE = 'tests/private_fixture_rows.txt' # The literal every private-fixtures placeholder prints before CTest marks it @@ -969,8 +969,10 @@ def profile_min_tests(profile: Profile, source: Path) -> int | None: def runner_e2e_test_names(source: Path) -> set[str]: - return {path.stem for path in (source / 'tests').glob('native_live*_e2e.py') - if path.stem != 'native_live_equivalence_e2e'} + # Require every driver, including corpus equivalence: native-live.yml + # supplies git-lfs and ensure_corpus_submodule fetches the corpus/feed. + # Missing prerequisites may skip in direct CTest, but fail CI (docs/ci.md). + return {path.stem for path in (source / 'tests').glob('native_live*_e2e.py')} @dataclass @@ -1784,6 +1786,12 @@ def ensure_corpus_submodule(self) -> bool: return False self.pass_stage('corpus-submodule-pin', value, argv=['git', 'submodule', 'status', '--', 'corpus']) + if self.cfg.profile.live_runner and self.cfg.profile.source_layer: + feed = self.invoke('corpus-live-feed', [ + 'git', '-C', str(self.cfg.source / 'corpus'), 'lfs', 'pull', + '--include=data/ohlcv_ETH-USDT-USDT_1m.csv', '--exclude=']) + if feed.returncode != 0: + return False return True def verify_configured_profile(self, cache: dict[str, str]) -> str | None: @@ -2179,6 +2187,9 @@ def run(self) -> int: rows = ctest_rows(ran.stdout + ran.stderr) if rows is None or rows.not_run or rows.disabled: raise RuntimeError(f'{self.cfg.profile.name} cannot accept skipped, disabled or unreadable rows') + required_skips = sorted(set(rows.skipped) & runner_e2e_test_names(self.cfg.source)) + if required_skips: + raise RuntimeError('required runner e2e rows skipped: ' + ', '.join(required_skips)) if rows.skipped and not private_fixtures_overlay(): # The required native-live job's shape (X2): without the # overlay runner/ registers its pinned private-fixture diff --git a/tests/native_live_equivalence_ctest.py b/tests/native_live_equivalence_ctest.py new file mode 100644 index 000000000..b39d3c3ec --- /dev/null +++ b/tests/native_live_equivalence_ctest.py @@ -0,0 +1,105 @@ +"""CTest entry point for the corpus equivalence driver and its prerequisites.""" + +import argparse +import ast +from pathlib import Path +import runpy +import sys +import tempfile + + +SKIP_CODE = 77 +SKIP_REASON = "SKIP native_live_equivalence_e2e prerequisites absent: " + + +def probe_names(script): + # Read the driver's own population without importing its receipt helper + # before the missing-prerequisites check. Never keep a second probe list. + for node in ast.parse(script.read_text()).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "PROBES" + for target in node.targets): + probes = ast.literal_eval(node.value) + if isinstance(probes, dict) and probes and all(isinstance(key, str) for key in probes): + return probes.keys() + raise ValueError("the equivalence driver's PROBES must be a non-empty dictionary") + raise ValueError("the equivalence driver has no PROBES declaration") + + +def missing_prerequisites(root, build, script): + paths = [(str(build / relative), build / relative) for relative in ( + "bin/pineforge-live", "compile_commands.json", "lib/libpineforge.a")] + sources = ["corpus/data/ohlcv_ETH-USDT-USDT_1m.csv", "tests/native_live_receipts.py"] + sources.extend(f"corpus/validation/{probe}/generated.cpp" for probe in probe_names(script)) + paths.extend((relative, root / relative) for relative in sources) + return [name for name, path in paths if not path.is_file()] + + +def run_driver(script, arguments): + previous_argv, previous_path = sys.argv, sys.path[:] + sys.argv = [str(script), *arguments] + sys.path.insert(0, str(script.parent)) + try: + try: + # Execute the real __main__, with its full default probe/scenario + # population. Its output and exceptions remain visible to CTest. + runpy.run_path(str(script), run_name="__main__") + except SystemExit as result: + if result.code is None: + code = 0 + elif isinstance(result.code, int): + code = result.code + else: + print(result.code, file=sys.stderr) + code = 1 + else: + code = 0 + finally: + sys.argv = previous_argv + sys.path[:] = previous_path + if code == SKIP_CODE: + print("equivalence driver exited 77 after prerequisites passed; treating as failure", + file=sys.stderr) + return 1 + # Normalize every other non-zero status too: e.g. SystemExit(256) must + # not wrap to shell exit 0 and turn a failed driver into a CTest pass. + return 0 if code == 0 else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--script", required=True, type=Path) + parser.add_argument("--root", required=True, type=Path) + parser.add_argument("--build-dir", required=True, type=Path) + parser.add_argument("--libraries-dir", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--sanitizer", choices=("asan", "tsan")) + arguments = parser.parse_args() + root, build, script = (path.resolve() for path in ( + arguments.root, arguments.build_dir, arguments.script)) + # Match the existing genuine-retry ctypes witness: Python starts with the + # build's runtime preloaded, native children keep their sanitizer options, + # and LSan checks after native cleanup rather than after CPython teardown. + from native_live_sanitizer import finalize_sanitizer, prepare_sanitizer, run_with_finalizer + prepare_sanitizer(arguments.sanitizer) + + def execute(): + missing = missing_prerequisites(root, build, script) + if missing: + print(SKIP_REASON + "; ".join(missing), flush=True) + return SKIP_CODE + output_root = arguments.out.resolve() + output_root.mkdir(parents=True, exist_ok=True) + retained = Path(tempfile.mkdtemp(prefix="run-", dir=output_root)) + libraries = arguments.libraries_dir.resolve() / retained.name + output = retained / "results" + print(f"Equivalence output: {output}; libraries: {libraries}", flush=True) + return run_driver(script, [ + "--root", str(root), "--build-dir", str(build), + "--libraries-dir", str(libraries), "--out", str(output)]) + + return run_with_finalizer(execute, finalize_sanitizer) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/private_fixture_rows.txt b/tests/private_fixture_rows.txt index dadf4837f..4f6dec42c 100644 --- a/tests/private_fixture_rows.txt +++ b/tests/private_fixture_rows.txt @@ -2,7 +2,7 @@ // when the private fixtures are absent (removed by the private-fixture PRs). The count also // covers the PR 1b batch, whose rows runner/CMakeLists.txt registers the same way. // manifest: l44/pr1-manifest.tsv sha256 9a2727d455c1b470d515665428e2c6e090a22ac07f87255d1ca9c20d5528bd62 -// rows: 375 +// rows: 376 capabilities_equivalence_clock_flags capabilities_equivalence_close checked_settings_e2e @@ -378,3 +378,4 @@ test_tvdef_drops_tapes test_whole_lot_tie_tapes test_zero_offset_trail_rides_l4c test_zero_trail_sibling_stop +native_live_equivalence_e2e From 19350a8f3d5da75f29564347750b6f0a5636dd36 Mon Sep 17 00:00:00 2001 From: Hao-Liang Wen Date: Sun, 25 Oct 2026 17:41:26 +0800 Subject: [PATCH 5/9] ci: release ruleset probe lists every branch ruleset (measurement only, gate unchanged) [replaces #489] (#505) * ci(release): the ruleset probe lists every branch ruleset as visible or hidden, even when the gate verdict is a refusal (CR-245 F1 measurement) The one dispatch of release-ruleset-probe.yml from main (run 40083495967) refused on the first branch ruleset of the inventory, the organization ruleset `default` (13094758): "returned missing or invalid bypass_actors (token cannot see this ruleset's bypass_actors)". The verdict reader stops at the first failure, so the run did not say whether the release App token can see bypass_actors on the repository ruleset that is the real barrier (21476386), nor on any other. The probe now first prints one line per branch ruleset of the includes_parents inventory (id, name, source_type, source, enforcement, updated_at, bypass_actors visible with the actor count or hidden, applies to main, has a pull_request rule), then runs the unchanged verdict: success prints the existing success lines; a refusal prints the same ::error:: text on stderr and exits 1, but stdout now carries the listing. A detail read that fails is reported on its own row and the listing continues. The listing is a measurement: `gate`, `_release_bump` and `_require_release_rulesets` are untouched, every gate test and the existing source mutants are kept, and the probe's verdict replay over every existing ruleset fixture still compares verdict and refusal text byte for byte. The workflow release-ruleset-probe.yml is not changed. The listing reads each branch ruleset detail independently of the verdict read (two reads per detail in the probe), which docs/ci.md now says. Red-first rows: the real situation (org ruleset hidden, repository ruleset visible: both listed, the org one hidden, refusal unchanged), both hidden, one detail read failing, all visible with a valid barrier, the read-only call list, and a source mutant that drops the listing. Rewrites named for the doc-reverts check, all introduced by 9c18d41 (#488, "ci(release): read-only ruleset probe for the release gate (CR-245 F1)"): docs/ci.md "Success prints the barrier id/name and one compact inventory line with enforcement, source and bypass visibility."; docs/ci.md "Refusal exits 1 with empty stdout and the reader's `::error::` diagnostic; API/response failures name the endpoint."; docs/ci.md "Two hidden rulesets need two dispatches: the first refusal names only the first hidden ruleset; make its `bypass_actors` visible before dispatching again to identify the second." (replaced: all hidden rulesets are listed in one dispatch); the probe docstring "Read exactly the bump's ruleset evidence, with the same refusal policy."; two test comments ("The reader report's complete [] inventory fails on main without an endpoint." and "The runtime assertions already hold there; the prose does not."). The module docstring sentence "Each endpoint/page is read at most once; failures refuse without retry or sleep." (ad1ebac, #479) is scoped to the gate's reads. Lane: ci-fu-cr488m (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(release): the probe listing bounds its inventory=ERROR row to 240 characters like the detail-error rows (CR-245 F1 measurement, sol P2) The new stdout listing printed the whole exception text on an `inventory=ERROR` row, while the per-ruleset `detail=ERROR` rows truncate it to 240 characters; a `gh` transport error with 300 characters of stderr was echoed in full. The row now applies the same bound ([:240], still !r-escaped). Four rows assert the exact bounded line (an inventory transport error, a repository transport error, a default-branch rejection with a long repository argument, an invalid summary with a long field) together with the unchanged stderr, exit code 1 and API call list of the verdict; each has a mutation control that removes only the bound and must fail. The verdict functions are untouched and the verdict's own stderr is not bounded. No documentation sentence is changed (none described a length limit). Lane: ci-fu-cr488n (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- docs/ci.md | 22 +- scripts/ci_release_gate.py | 75 ++++++- scripts/test_ci_release_gate.py | 351 ++++++++++++++++++++++++++++---- 3 files changed, 402 insertions(+), 46 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index 361c44c10..e0d167de6 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1812,12 +1812,22 @@ The dispatch must run from `main`; the workflow itself refuses any other ref before minting the App token or checking out repository code. It runs `python3 scripts/ci_release_gate.py rulesets-probe --repo OWNER/REPO`, reading only repository metadata, the parent-inclusive ruleset inventory and -branch details. Success prints the barrier id/name and one compact inventory -line with enforcement, source and bypass visibility. Refusal exits 1 with empty -stdout and the reader's `::error::` diagnostic; API/response failures name the endpoint. -Two hidden rulesets need two dispatches: the first refusal names only the -first hidden ruleset; make its `bypass_actors` visible before dispatching again -to identify the second. +branch details. Before the verdict, stdout lists every branch ruleset from the +complete inventory, one per line; the header states that tag/push targets are omitted. +Each row records id, name, source type/source, enforcement, update time, bypass +visibility (and actor count for a visible list), applicability to main and +whether a `pull_request` rule is present. +Visibility means only that the detail payload's `bypass_actors` is a list. +Unreadable ref conditions print `applies=unreadable`; a failed detail GET prints +`detail=ERROR` on its row and the listing continues with the other branch rulesets. +An unreadable repository or incomplete/invalid inventory prints `inventory=ERROR`. +The listing and the verdict independently read repository metadata, inventory and +branch details, so an endpoint may be read twice. +Success keeps the existing `release ruleset barrier ok` and compact inventory lines. +Refusal exits 1 with the listing on stdout and +the reader's unchanged `::error::` diagnostic on stderr; API/response failures name the endpoint. +One dispatch lists every branch ruleset, including all hidden lists, even when +the gate refuses on the first hidden ruleset. The probe changes no release evidence or gate verdicts. A release `dry_run` on a merged PR does not exercise the bump-only ruleset reader. diff --git a/scripts/ci_release_gate.py b/scripts/ci_release_gate.py index 7624a027c..dde4c8b51 100644 --- a/scripts/ci_release_gate.py +++ b/scripts/ci_release_gate.py @@ -5,7 +5,8 @@ ci_release_gate.py rulesets-probe --repo OWNER/REPO Every GitHub API call is a read (GET), using the release job's own GH_TOKEN. -Each endpoint/page is read at most once; failures refuse without retry or sleep. +Each gate endpoint/page is read at most once; failures refuse without retry or sleep. +The probe measures the inventory before an independent verdict read. Phase 1 checks a unique tree-matching merged PR head or one authenticated, two-file release bump on that gated parent. Phase 2 requires the exact main sha's proof receipt after the fixed watcher cut-over, with no PR/bump fallback. @@ -410,8 +411,78 @@ def _require_release_rulesets(api: API, repo: str) -> tuple[dict, list[dict]]: return barrier, branches +def _ruleset_inventory_report(api: API, repo: str) -> str: + """Measure every branch ruleset without deciding the release verdict.""" + lines = [ + "branch ruleset inventory (includes_parents=true; branch targets only; tag/push omitted):"] + try: + endpoint = f"repos/{repo}" + project = _read_json(api, endpoint) + if not isinstance(project, dict) or project.get("default_branch") != "main": + raise ReleaseGateError(f"API {endpoint} must identify main as the default branch") + + endpoint = f"repos/{repo}/rulesets?includes_parents=true&per_page=100" + listed = _read_json(api, endpoint, complete_rulesets=True, paginate=True) + identities, entries = set(), [] + # Match the verdict reader's complete inventory and summary validation. + for item in listed: + if (not isinstance(item, dict) or type(item.get("id")) is not int or item["id"] <= 0 + or not isinstance(item.get("target"), str) + or item["target"] not in ("branch", "tag", "push") + or not isinstance(item.get("enforcement"), str) + or item["enforcement"] not in ("active", "disabled", "evaluate")): + raise ReleaseGateError( + f"API {endpoint} returned invalid ruleset summary: {item!r}") + if item["id"] in identities: + raise ReleaseGateError( + f"API {endpoint} returned duplicate ruleset id {item['id']}") + identities.add(item["id"]) + entries.append(item) + except ReleaseGateError as exc: + lines.append(f"inventory=ERROR {str(exc)[:240]!r}") + return "\n".join(lines) + + for item in entries: + if item["target"] != "branch": + continue + endpoint = f"repos/{repo}/rulesets/{item['id']}" + display = item + try: + entry = _read_json(api, endpoint) + if (not isinstance(entry, dict) or type(entry.get("id")) is not int + or any(entry.get(key) != item[key] + for key in ("id", "target", "enforcement"))): + raise ReleaseGateError(f"API {endpoint} returned inconsistent ruleset detail") + display = {**item, **entry} + bypass_list = entry.get("bypass_actors") + visibility = (f"visible actors={len(bypass_list)}" + if isinstance(bypass_list, list) else "hidden") + try: + applies = "yes" if _ruleset_applies_to_main(entry, endpoint) else "no" + except ReleaseGateError: + applies = "unreadable" + rule_rows = entry.get("rules") + pull_request = "unreadable" + if isinstance(rule_rows, list): + pull_request = "yes" if any( + isinstance(rule, dict) and rule.get("type") == "pull_request" + for rule in rule_rows) else "no" + evidence = (f"bypass_actors={visibility} applies={applies} " + f"pull_request={pull_request}") + except ReleaseGateError as exc: + evidence = f"detail=ERROR {str(exc)[:240]!r}" + lines.append( + f"ruleset id={item['id']} name={display.get('name', '')!r} " + f"source_type={display.get('source_type', '')!r} " + f"source={display.get('source', '')!r} " + f"enforcement={display['enforcement']!r} " + f"updated_at={display.get('updated_at', '')!r} {evidence}") + return "\n".join(lines) + + def rulesets_probe(repo: str, *, api: API) -> str: - """Read exactly the bump's ruleset evidence, with the same refusal policy.""" + """Print the complete measurement, then use the bump's unchanged verdict.""" + print(_ruleset_inventory_report(api, repo), flush=True) barrier, branches = _require_release_rulesets(api, repo) inventory = "; ".join( f"{row['id']} {row.get('name', '')!r} {row['enforcement']} " diff --git a/scripts/test_ci_release_gate.py b/scripts/test_ci_release_gate.py index 0b8b95571..8de073392 100644 --- a/scripts/test_ci_release_gate.py +++ b/scripts/test_ci_release_gate.py @@ -46,6 +46,8 @@ SINGLE_DOCUMENT_REFUSAL = ( "returned an HTTP response body with trailing data; " "expected exactly one JSON document") +RULESET_INVENTORY_HEADER = ( + "branch ruleset inventory (includes_parents=true; branch targets only; tag/push omitted):\n") def _without_comments(text): @@ -1613,6 +1615,25 @@ def api_for(self, rows=None): api.rulesets(release_rulesets() if rows is None else rows) return api + def measurement_rows(self): + rows = release_rulesets() + rows[0].update(id=13094758, name="default", source_type="Organization", + source="example", updated_at="2026-10-07T16:13:00Z") + rows[1].update(id=21476386, name="PineForge strict CI base", + source_type="Repository", source=REPO, + updated_at="2026-09-26T02:40:00Z") + return rows + + def measurement_listing(self, org_visibility, repo_visibility): + return ( + RULESET_INVENTORY_HEADER + + "ruleset id=13094758 name='default' source_type='Organization' " + "source='example' enforcement='active' updated_at='2026-10-07T16:13:00Z' " + f"bypass_actors={org_visibility} applies=yes pull_request=yes\n" + "ruleset id=21476386 name='PineForge strict CI base' source_type='Repository' " + f"source='{REPO}' enforcement='active' updated_at='2026-09-26T02:40:00Z' " + f"bypass_actors={repo_visibility} applies=yes pull_request=yes\n") + def run_probe(self, api, *, main=None, repo=REPO): stdout, stderr = io.StringIO(), io.StringIO() forbidden = mock.Mock(side_effect=AssertionError("probe must not run subprocesses")) @@ -1629,54 +1650,251 @@ def run_probe(self, api, *, main=None, repo=REPO): return code, stdout.getvalue(), stderr.getvalue() def test_hidden_org_bypass_actors_refuses_with_the_reader_diagnostic(self): - rows = release_rulesets() - rows[0].update(id=13094758, name="default", source_type="Organization", - source="example") + rows = self.measurement_rows() del rows[0]["bypass_actors"] api = self.api_for(rows) self.assertEqual(self.run_probe(api), ( - 1, "", f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " + 1, self.measurement_listing("hidden", "visible actors=1"), + f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " "bypass_actors (token cannot see this ruleset's bypass_actors)\n")) self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None), + (f"repos/{REPO}/rulesets/21476386", False, None), (REPO_API, False, None), (RULESETS_API, True, None), (f"repos/{REPO}/rulesets/13094758", False, None)]) - def test_two_hidden_rulesets_require_two_dispatches_as_documented(self): - docs = (ROOT / "docs/ci.md").read_text() - limit = ( - "Two hidden rulesets need two dispatches: the first refusal names only the\n" - "first hidden ruleset; make its `bypass_actors` visible before dispatching again\n" - "to identify the second.") - message = "docs must explain the first-hidden-ruleset refusal limit" - self.assertIn(limit, docs, message) - changed = docs.replace(limit, "", 1) - self.assertNotEqual(changed, docs) - with self.assertRaisesRegex(AssertionError, message): - self.assertIn(limit, changed, message) - - rows = release_rulesets() - rows[0].update(id=13094758, name="default", source_type="Organization", - source="example") - rows[1].update(source_type="Repository", source=REPO) + def test_two_hidden_rulesets_are_both_listed_in_one_dispatch_as_documented(self): + rows = self.measurement_rows() visible_actors = rows[0].pop("bypass_actors") del rows[1]["bypass_actors"] first = self.api_for(rows) self.assertEqual(self.run_probe(first), ( - 1, "", f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " + 1, self.measurement_listing("hidden", "hidden"), + f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " "bypass_actors (token cannot see this ruleset's bypass_actors)\n")) self.assertEqual(first.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None), + (f"repos/{REPO}/rulesets/21476386", False, None), (REPO_API, False, None), (RULESETS_API, True, None), (f"repos/{REPO}/rulesets/13094758", False, None)]) rows[0]["bypass_actors"] = visible_actors second = self.api_for(rows) self.assertEqual(self.run_probe(second), ( - 1, "", f"::error::API {STRICT_RULESET_API} returned missing or invalid " + 1, self.measurement_listing("visible actors=5", "hidden"), + f"::error::API repos/{REPO}/rulesets/21476386 returned missing or invalid " "bypass_actors (token cannot see this ruleset's bypass_actors)\n")) self.assertEqual(second.calls, [ (REPO_API, False, None), (RULESETS_API, True, None), (f"repos/{REPO}/rulesets/13094758", False, None), - (STRICT_RULESET_API, False, None)]) + (f"repos/{REPO}/rulesets/21476386", False, None), + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None), + (f"repos/{REPO}/rulesets/21476386", False, None)]) + + docs = (ROOT / "docs/ci.md").read_text() + limit = ( + "One dispatch lists every branch ruleset, including all hidden lists, even when\n" + "the gate refuses on the first hidden ruleset.") + message = "docs must explain the complete inventory despite the first refusal" + self.assertIn(limit, docs, message) + changed = docs.replace(limit, "", 1) + self.assertNotEqual(changed, docs) + with self.assertRaisesRegex(AssertionError, message): + self.assertIn(limit, changed, message) + + def test_detail_http_error_is_listed_and_other_branch_details_are_still_read(self): + rows = self.measurement_rows() + api = self.api_for(rows) + summaries = ruleset_summaries(rows) + summaries[0].update(source_type="Organization", source="example", + updated_at="2026-10-07T16:13:00Z") + api.responses[RULESETS_API] = http_response(200, json.dumps(summaries)) + endpoint = f"repos/{REPO}/rulesets/13094758" + api.responses[endpoint] = http_response(503, "unavailable") + self.assertEqual(self.run_probe(api), ( + 1, RULESET_INVENTORY_HEADER + + "ruleset id=13094758 name='default' source_type='Organization' " + "source='example' enforcement='active' updated_at='2026-10-07T16:13:00Z' " + f"detail=ERROR 'API {endpoint} upstream error 503'\n" + "ruleset id=21476386 name='PineForge strict CI base' source_type='Repository' " + f"source='{REPO}' enforcement='active' updated_at='2026-09-26T02:40:00Z' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n", + f"::error::API {endpoint} upstream error 503\n")) + self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (endpoint, False, None), + (f"repos/{REPO}/rulesets/21476386", False, None), + (REPO_API, False, None), (RULESETS_API, True, None), + (endpoint, False, None)]) + + def test_measurement_visibility_is_only_a_list_not_a_valid_bypass_or_rule(self): + rows = self.measurement_rows() + rows[0]["bypass_actors"] = [{}] + rows[0]["conditions"] = {} + rows[1]["bypass_actors"] = None + rows[1]["rules"] = [{"type": "deletion"}] + api = self.api_for(rows) + expected = self.measurement_listing("visible actors=1", "hidden") + expected = expected.replace("applies=yes", "applies=unreadable", 1) + expected = expected.rsplit("pull_request=yes", 1)[0] + "pull_request=no\n" + self.assertEqual(self.run_probe(api), ( + 1, expected, + f"::error::API repos/{REPO}/rulesets/13094758 returned invalid bypass actor: {{}}\n")) + + def test_empty_visible_list_is_measured_without_counting_it_as_a_barrier(self): + rows = self.measurement_rows() + rows[0]["bypass_actors"] = [] + rows[1]["enforcement"] = "disabled" + rows[1]["conditions"]["ref_name"]["exclude"] = ["refs/heads/main"] + api = self.api_for(rows) + listing = self.measurement_listing("visible actors=0", "visible actors=1") + listing = listing.replace( + "enforcement='active' updated_at='2026-09-26T02:40:00Z'", + "enforcement='disabled' updated_at='2026-09-26T02:40:00Z'") + listing = listing.rsplit("applies=yes", 1)[0] + "applies=no pull_request=yes\n" + self.assertEqual(self.run_probe(api), ( + 1, listing, f"::error::{RULESET_BARRIER_REFUSAL}\n")) + + def test_later_inventory_pages_and_disabled_branch_rulesets_are_all_listed(self): + rows = self.measurement_rows() + del rows[0]["bypass_actors"] + rows.append(ruleset(19359220, enforcement="disabled", rules=("deletion",), + include=["refs/heads/release"])) + api = self.api_for(rows) + api.responses[RULESETS_API] = ( + http_response(200, json.dumps(ruleset_summaries(rows[:1])), { + "Link": f'; rel="next"'}) + + http_response(200, json.dumps(ruleset_summaries(rows[1:])))) + self.assertEqual(self.run_probe(api), ( + 1, self.measurement_listing("hidden", "visible actors=1") + + "ruleset id=19359220 name='fixture ruleset 19359220' source_type='' " + "source='' enforcement='disabled' updated_at='' " + "bypass_actors=visible actors=1 applies=no pull_request=no\n", + f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " + "bypass_actors (token cannot see this ruleset's bypass_actors)\n")) + self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None), + (f"repos/{REPO}/rulesets/21476386", False, None), + (f"repos/{REPO}/rulesets/19359220", False, None), + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None)]) + + def test_inventory_failures_are_reported_before_the_unchanged_reader_refusal(self): + invalid = {"id": 707, "target": "branch", "enforcement": "unknown"} + duplicate = ruleset_summaries([ruleset(), ruleset()]) + for endpoint, payload, diagnostic in ( + (REPO_API, json.dumps({"default_branch": "release"}), + f"API {REPO_API} must identify main as the default branch"), + (RULESETS_API, http_response(200, json.dumps([invalid])), + f"API {RULESETS_API} returned invalid ruleset summary: {invalid!r}"), + (RULESETS_API, http_response(200, json.dumps(duplicate)), + f"API {RULESETS_API} returned duplicate ruleset id 202"), + (RULESETS_API, http_response(200, "[]", { + "Link": f'; rel="next"'}), + f"API {RULESETS_API} returned incomplete ruleset pagination: " + "unread or unlinked next page")): + with self.subTest(endpoint=endpoint, diagnostic=diagnostic): + api = self.api_for() + api.responses[endpoint] = payload + self.assertEqual(self.run_probe(api), ( + 1, RULESET_INVENTORY_HEADER + f"inventory=ERROR {diagnostic!r}\n", + f"::error::{diagnostic}\n")) + reads = [(REPO_API, False, None)] + if endpoint == RULESETS_API: + reads.append((RULESETS_API, True, None)) + self.assertEqual(api.calls, reads * 2) + + def test_long_inventory_errors_are_escaped_shortened_and_keep_the_refusal(self): + detail = "gh: Forbidden (HTTP 403)\n" + "x" * 300 + long_repo = "example/" + "r" * 300 + long_repo_api = f"repos/{long_repo}" + invalid = {"id": 707, "target": "branch", "enforcement": "unknown" + "x" * 300} + cases = ( + ("inventory transport", REPO, RULESETS_API, subprocess.CalledProcessError( + 1, ["gh", "api", RULESETS_API], stderr=detail), + f"API {RULESETS_API} failed with HTTP 403: {detail}"), + ("repository transport", REPO, REPO_API, subprocess.CalledProcessError( + 1, ["gh", "api", REPO_API], stderr=detail), + f"API {REPO_API} failed with HTTP 403: {detail}"), + ("default branch", long_repo, long_repo_api, + json.dumps({"default_branch": "release"}), + f"API {long_repo_api} must identify main as the default branch"), + ("summary validation", REPO, RULESETS_API, + http_response(200, json.dumps([invalid])), + f"API {RULESETS_API} returned invalid ruleset summary: {invalid!r}"), + ) + for scenario, repo, endpoint, payload, diagnostic in cases: + with self.subTest(scenario=scenario): + self.assertGreater(len(diagnostic), 240) + api = self.api_for() + api.responses[endpoint] = payload + expected = ( + 1, RULESET_INVENTORY_HEADER + f"inventory=ERROR {diagnostic[:240]!r}\n", + f"::error::{diagnostic}\n") + # RED on 5a866543573c: every row prints the entire diagnostic. + self.assertEqual(self.run_probe(api, repo=repo), expected) + reads = [(f"repos/{repo}", False, None)] + if endpoint == RULESETS_API: + reads.append((RULESETS_API, True, None)) + self.assertEqual(api.calls, reads * 2) + + # Revert only the bound in the full module and require this + # same exact-output assertion to catch every input row. + source = Path(ci_release_gate.__file__).read_text() + before = ' lines.append(f"inventory=ERROR {str(exc)[:240]!r}")\n' + after = ' lines.append(f"inventory=ERROR {str(exc)!r}")\n' + self.assertEqual(source.count(before), 1, + "mutation must have one source target") + mutated = source.replace(before, after, 1) + self.assertNotEqual(mutated, source) + namespace = {"__name__": "ci_release_inventory_error_bound_mutant"} + exec(compile(mutated, ci_release_gate.__file__, "exec"), namespace) + mutant_api = self.api_for() + mutant_api.responses[endpoint] = payload + result = self.run_probe(mutant_api, repo=repo, main=namespace["main"]) + self.assertEqual(result, ( + 1, RULESET_INVENTORY_HEADER + f"inventory=ERROR {diagnostic!r}\n", + f"::error::{diagnostic}\n")) + with self.assertRaises(AssertionError): + self.assertEqual(result, expected) + self.assertEqual(mutant_api.calls, reads * 2) + + def test_detail_transport_error_is_escaped_shortened_and_does_not_hide_next_row(self): + rows = self.measurement_rows() + api = self.api_for(rows) + endpoint = f"repos/{REPO}/rulesets/13094758" + detail = "gh: Forbidden (HTTP 403)\n::notice::" + "x" * 300 + api.responses[endpoint] = subprocess.CalledProcessError( + 1, ["gh", "api", endpoint], stderr=detail) + diagnostic = f"API {endpoint} failed with HTTP 403: {detail}" + repo_line = self.measurement_listing("hidden", "visible actors=1").splitlines()[2] + self.assertEqual(self.run_probe(api), ( + 1, RULESET_INVENTORY_HEADER + + "ruleset id=13094758 name='default' source_type='' " + "source='' enforcement='active' updated_at='' " + f"detail=ERROR {diagnostic[:240]!r}\n{repo_line}\n", + f"::error::{diagnostic}\n")) + self.assertIn((f"repos/{REPO}/rulesets/21476386", False, None), api.calls) + + def test_api_display_metadata_is_escaped_in_each_inventory_field(self): + rows = self.measurement_rows() + rows[0].update(name="org\n::notice::", source_type="Organization\rtest", + source="owner\nname", updated_at="time\tstamp\nnext") + del rows[0]["bypass_actors"] + listing = self.measurement_listing("hidden", "visible actors=1") + listing = listing.replace("name='default'", "name='org\\n::notice::'") + listing = listing.replace("source_type='Organization'", "source_type='Organization\\rtest'") + listing = listing.replace("source='example'", "source='owner\\nname'") + listing = listing.replace( + "updated_at='2026-10-07T16:13:00Z'", "updated_at='time\\tstamp\\nnext'") + self.assertEqual(self.run_probe(self.api_for(rows)), ( + 1, listing, + f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " + "bypass_actors (token cannot see this ruleset's bypass_actors)\n")) def test_probe_dispatch_branch_and_fleets_are_documented(self): docs = (ROOT / "docs/ci.md").read_text() @@ -1712,17 +1930,17 @@ def test_probe_dispatch_branch_and_fleets_are_documented(self): self.assertIn(required, changed, message) def test_empty_inventory_refusal_and_documented_endpoint_scope(self): - # The reader report's complete [] inventory fails on main without an - # endpoint. The runtime assertions already hold there; the prose does not. + # A complete [] inventory has no barrier; its refusal has no endpoint. api = self.api_for([]) self.assertEqual(self.run_probe(api), ( - 1, "", f"::error::{RULESET_BARRIER_REFUSAL}\n")) + 1, RULESET_INVENTORY_HEADER, f"::error::{RULESET_BARRIER_REFUSAL}\n")) self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), (REPO_API, False, None), (RULESETS_API, True, None)]) docs = (ROOT / "docs/ci.md").read_text() - scoped = ("stdout and the reader's `::error::` diagnostic; " + scoped = ("the reader's unchanged `::error::` diagnostic on stderr; " "API/response failures name the endpoint.") - unscoped = ("stdout and the reader's `::error::` diagnostic " + unscoped = ("the reader's unchanged `::error::` diagnostic on stderr " "naming the failing endpoint.") def assert_scope(text): @@ -1741,8 +1959,16 @@ def test_visible_barrier_prints_exact_evidence_and_only_reads_rulesets(self): rows = release_rulesets() rows[0].update(source_type="Organization", source="example") rows[1].update(source_type="Repository", source=REPO) + rows.append(ruleset(404, target="push")) api = self.api_for(rows) self.assertEqual(self.run_probe(api), (0, ( + RULESET_INVENTORY_HEADER + + "ruleset id=101 name='fixture ruleset 101' source_type='Organization' " + "source='example' enforcement='active' updated_at='' " + "bypass_actors=visible actors=5 applies=yes pull_request=yes\n" + "ruleset id=202 name='fixture ruleset 202' source_type='Repository' " + f"source='{REPO}' enforcement='active' updated_at='' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n" "release ruleset barrier ok: ruleset 202 'fixture ruleset 202'\n" "branch rulesets: 101 'fixture ruleset 101' active " "source='Organization':'example' bypass_actors=visible; " @@ -1750,7 +1976,7 @@ def test_visible_barrier_prints_exact_evidence_and_only_reads_rulesets(self): f"source='Repository':'{REPO}' bypass_actors=visible\n"), "")) self.assertEqual(api.calls, [ (REPO_API, False, None), (RULESETS_API, True, None), - (DEFAULT_RULESET_API, False, None), (STRICT_RULESET_API, False, None)]) + (DEFAULT_RULESET_API, False, None), (STRICT_RULESET_API, False, None)] * 2) def test_barrier_identity_and_inventory_are_not_fixture_constants(self): rows = [ruleset(907, enforcement="evaluate"), ruleset(809)] @@ -1760,6 +1986,13 @@ def test_barrier_identity_and_inventory_are_not_fixture_constants(self): repo = "owner/other" api.responses = {key.replace(REPO, repo): value for key, value in api.responses.items()} self.assertEqual(self.run_probe(api, repo=repo), (0, ( + RULESET_INVENTORY_HEADER + + "ruleset id=907 name='other\\nprotection' source_type='Organization' " + "source='owner' enforcement='evaluate' updated_at='' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n" + "ruleset id=809 name='release barrier' source_type='Repository' " + "source='owner/other' enforcement='active' updated_at='' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n" "release ruleset barrier ok: ruleset 809 'release barrier'\n" "branch rulesets: 907 'other\\nprotection' evaluate " "source='Organization':'owner' bypass_actors=visible; " @@ -1769,7 +2002,7 @@ def test_barrier_identity_and_inventory_are_not_fixture_constants(self): (f"repos/{repo}", False, None), (f"repos/{repo}/rulesets?includes_parents=true&per_page=100", True, None), (f"repos/{repo}/rulesets/907", False, None), - (f"repos/{repo}/rulesets/809", False, None)]) + (f"repos/{repo}/rulesets/809", False, None)] * 2) def test_missing_display_metadata_does_not_change_the_gate_verdict(self): rows = [ruleset()] @@ -1780,6 +2013,10 @@ def test_missing_display_metadata_does_not_change_the_gate_verdict(self): detail = {key: value for key, value in rows[0].items() if key != "name"} api.responses[STRICT_RULESET_API] = json.dumps(detail) self.assertEqual(self.run_probe(api), (0, ( + RULESET_INVENTORY_HEADER + + "ruleset id=202 name='' source_type='' " + "source='' enforcement='active' updated_at='' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n" "release ruleset barrier ok: ruleset 202 ''\n" "branch rulesets: 202 '' active " "source='':'' bypass_actors=visible\n"), "")) @@ -1802,17 +2039,22 @@ def compare_then_gate(**kwargs): try: ci_release_gate._require_release_rulesets(direct, REPO) except ci_release_gate.ReleaseGateError as exc: - expected = (1, "", f"::error::{exc}\n") + expected = (1, f"::error::{exc}\n") else: expected = None result = self.run_probe(probe) + self.assertTrue(result[1].startswith(RULESET_INVENTORY_HEADER), result) if expected is None: self.assertEqual(result[0], 0, result) self.assertEqual(result[2], "") - self.assertEqual(len(result[1].splitlines()), 2, result) + success_lines = result[1].splitlines()[-2:] + self.assertTrue(success_lines[0].startswith( + "release ruleset barrier ok: ruleset "), result) + self.assertTrue(success_lines[1].startswith("branch rulesets: "), result) else: - self.assertEqual(result, expected) - self.assertEqual(probe.calls, direct.calls) + self.assertEqual((result[0], result[2]), expected) + self.assertNotIn("release ruleset barrier ok:", result[1]) + self.assertEqual(probe.calls[-len(direct.calls):], direct.calls) checked.append(result[0]) return original(**kwargs) @@ -1827,7 +2069,15 @@ def test_full_source_mutant_skipping_reader_admits_the_hidden_org_fixture(self): rows[0].update(id=13094758, name="default", source_type="Organization", source="example") del rows[0]["bypass_actors"] - refusal = (1, "", f"::error::API repos/{REPO}/rulesets/13094758 returned missing or " + listing = ( + RULESET_INVENTORY_HEADER + + "ruleset id=13094758 name='default' source_type='Organization' " + "source='example' enforcement='active' updated_at='' " + "bypass_actors=hidden applies=yes pull_request=yes\n" + "ruleset id=202 name='fixture ruleset 202' source_type='' " + "source='' enforcement='active' updated_at='' " + "bypass_actors=visible actors=1 applies=yes pull_request=yes\n") + refusal = (1, listing, f"::error::API repos/{REPO}/rulesets/13094758 returned missing or " "invalid bypass_actors (token cannot see this ruleset's bypass_actors)\n") self.assertEqual(self.run_probe(self.api_for(rows)), refusal) source = Path(ci_release_gate.__file__).read_text() @@ -1840,8 +2090,33 @@ def test_full_source_mutant_skipping_reader_admits_the_hidden_org_fixture(self): exec(compile(mutated, ci_release_gate.__file__, "exec"), namespace) api = self.api_for(rows) self.assertEqual(self.run_probe(api, main=namespace["main"]), ( - 0, "release ruleset barrier ok: ruleset 0 'unverified'\nbranch rulesets: \n", "")) - self.assertEqual(api.calls, [], "the mutant must bypass the real readers") + 0, listing + "release ruleset barrier ok: ruleset 0 'unverified'\nbranch rulesets: \n", + "")) + self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None), + (STRICT_RULESET_API, False, None)], + "the mutant must bypass the verdict reader, while retaining measurement") + + def test_full_source_mutant_dropping_listing_loses_only_the_measurement(self): + rows = self.measurement_rows() + del rows[0]["bypass_actors"] + stderr = (f"::error::API repos/{REPO}/rulesets/13094758 returned missing or invalid " + "bypass_actors (token cannot see this ruleset's bypass_actors)\n") + self.assertEqual(self.run_probe(self.api_for(rows)), ( + 1, self.measurement_listing("hidden", "visible actors=1"), stderr)) + source = Path(ci_release_gate.__file__).read_text() + before = " print(_ruleset_inventory_report(api, repo), flush=True)\n" + self.assertEqual(source.count(before), 1, "mutation must have one source target") + mutated = source.replace(before, "", 1) + self.assertNotEqual(mutated, source) + namespace = {"__name__": "ci_release_rulesets_listing_mutant"} + exec(compile(mutated, ci_release_gate.__file__, "exec"), namespace) + api = self.api_for(rows) + self.assertEqual(self.run_probe(api, main=namespace["main"]), (1, "", stderr)) + self.assertEqual(api.calls, [ + (REPO_API, False, None), (RULESETS_API, True, None), + (f"repos/{REPO}/rulesets/13094758", False, None)]) class APICommandTests(unittest.TestCase): From db58170afb4e14b28eb0ca1d242eb7cb84b38fe8 Mon Sep 17 00:00:00 2001 From: Hao-Liang Wen Date: Sun, 25 Oct 2026 19:09:39 +0800 Subject: [PATCH 6/9] ci: fully qualify the two remote-tracking refs the promote-baseline ancestry check and the doc-reverts default base read (ORG-ACTIONPINS-CARD) [replaces #494] (#507) * ci: fully qualify the two remote-tracking refs the promote-baseline ancestry check and the doc-reverts default base read (ORG-ACTIONPINS-CARD, LB-464 P3) promote-baseline.yml tested `git merge-base --is-ancestor "$MERGE_SHA" "origin/$BASE_REF"`, and check_doc_reverts.py default_base() resolved the bare names `origin/main` then `main`. A short name resolves through git's ref-lookup order, so a tag called `origin/main` (or `main`) would win over the remote-tracking branch or the local branch. Both now use the fully qualified names (`refs/remotes/origin/$BASE_REF`; `refs/remotes/origin/main`, then `refs/heads/main`, same order and same fallback to the merged-pull-request walk). The exact-text pin in scripts/ci_preflight.py is updated to the new command and stays as strict; the old short form is now a finding. Red-first tests: the pin refuses the short command and accepts the qualified one; a repository where a TAG named `origin/main` points at an unrelated commit and the real remote-tracking ref points elsewhere, and the same for a tag named `main` with no remote-tracking ref, each returns the merge base of the real ref and not the tag's. Five controls keep the order, the environment override, the local fallback, the merged-pull-request fallback and the no-base result. The first-parent walk is untouched (the lab-host defect is a separate card). The default_base docstring in scripts/check_doc_reverts.py is re-wrapped around the two renamed refs; the sentences it carries come from 1e360bd31 and d0935605c and keep their wording, only the ref names change. Lane: ci-fu-shortrefs (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci: release ruleset probe lists every branch ruleset (measurement only, gate unchanged) [replaces #489] (#505) * ci(release): the ruleset probe lists every branch ruleset as visible or hidden, even when the gate verdict is a refusal (CR-245 F1 measurement) The one dispatch of release-ruleset-probe.yml from main (run 40083495967) refused on the first branch ruleset of the inventory, the organization ruleset `default` (13094758): "returned missing or invalid bypass_actors (token cannot see this ruleset's bypass_actors)". The verdict reader stops at the first failure, so the run did not say whether the release App token can see bypass_actors on the repository ruleset that is the real barrier (21476386), nor on any other. The probe now first prints one line per branch ruleset of the includes_parents inventory (id, name, source_type, source, enforcement, updated_at, bypass_actors visible with the actor count or hidden, applies to main, has a pull_request rule), then runs the unchanged verdict: success prints the existing success lines; a refusal prints the same ::error:: text on stderr and exits 1, but stdout now carries the listing. A detail read that fails is reported on its own row and the listing continues. The listing is a measurement: `gate`, `_release_bump` and `_require_release_rulesets` are untouched, every gate test and the existing source mutants are kept, and the probe's verdict replay over every existing ruleset fixture still compares verdict and refusal text byte for byte. The workflow release-ruleset-probe.yml is not changed. The listing reads each branch ruleset detail independently of the verdict read (two reads per detail in the probe), which docs/ci.md now says. Red-first rows: the real situation (org ruleset hidden, repository ruleset visible: both listed, the org one hidden, refusal unchanged), both hidden, one detail read failing, all visible with a valid barrier, the read-only call list, and a source mutant that drops the listing. Rewrites named for the doc-reverts check, all introduced by 9c18d41 (#488, "ci(release): read-only ruleset probe for the release gate (CR-245 F1)"): docs/ci.md "Success prints the barrier id/name and one compact inventory line with enforcement, source and bypass visibility."; docs/ci.md "Refusal exits 1 with empty stdout and the reader's `::error::` diagnostic; API/response failures name the endpoint."; docs/ci.md "Two hidden rulesets need two dispatches: the first refusal names only the first hidden ruleset; make its `bypass_actors` visible before dispatching again to identify the second." (replaced: all hidden rulesets are listed in one dispatch); the probe docstring "Read exactly the bump's ruleset evidence, with the same refusal policy."; two test comments ("The reader report's complete [] inventory fails on main without an endpoint." and "The runtime assertions already hold there; the prose does not."). The module docstring sentence "Each endpoint/page is read at most once; failures refuse without retry or sleep." (ad1ebac, #479) is scoped to the gate's reads. Lane: ci-fu-cr488m (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(release): the probe listing bounds its inventory=ERROR row to 240 characters like the detail-error rows (CR-245 F1 measurement, sol P2) The new stdout listing printed the whole exception text on an `inventory=ERROR` row, while the per-ruleset `detail=ERROR` rows truncate it to 240 characters; a `gh` transport error with 300 characters of stderr was echoed in full. The row now applies the same bound ([:240], still !r-escaped). Four rows assert the exact bounded line (an inventory transport error, a repository transport error, a default-branch rejection with a long repository argument, an invalid summary with a long field) together with the unchanged stderr, exit code 1 and API call list of the verdict; each has a mutation control that removes only the bound and must fail. The verdict functions are untouched and the verdict's own stderr is not bounded. No documentation sentence is changed (none described a length limit). Lane: ci-fu-cr488n (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --------- Co-authored-by: ci-cx Co-authored-by: Claude Sonnet 5.5 --- .github/workflows/promote-baseline.yml | 4 +- scripts/check_doc_reverts.py | 22 ++++---- scripts/ci_preflight.py | 2 +- scripts/test_check_doc_reverts.py | 69 ++++++++++++++++++++++++++ scripts/test_ci_preflight.py | 21 ++++++++ 5 files changed, 104 insertions(+), 14 deletions(-) diff --git a/.github/workflows/promote-baseline.yml b/.github/workflows/promote-baseline.yml index 476d81f4e..06e229149 100644 --- a/.github/workflows/promote-baseline.yml +++ b/.github/workflows/promote-baseline.yml @@ -340,8 +340,8 @@ jobs: set -euo pipefail # The PR head's objects, to read its tree: fetched, never checked out or run. git fetch --quiet origin "$BASE_REF" "+refs/pull/$PR_NUMBER/head:refs/remotes/origin/pr-head" - if ! git merge-base --is-ancestor "$MERGE_SHA" "origin/$BASE_REF"; then - echo "::notice::merge commit $MERGE_SHA is not on origin/$BASE_REF; skipping" + if ! git merge-base --is-ancestor "$MERGE_SHA" "refs/remotes/origin/$BASE_REF"; then + echo "::notice::merge commit $MERGE_SHA is not on refs/remotes/origin/$BASE_REF; skipping" echo "ok=false" >> "$GITHUB_OUTPUT"; exit 0 fi merge_tree=$(git rev-parse "$MERGE_SHA^{tree}") diff --git a/scripts/check_doc_reverts.py b/scripts/check_doc_reverts.py index c8886ef51..0b13a7cd7 100644 --- a/scripts/check_doc_reverts.py +++ b/scripts/check_doc_reverts.py @@ -52,16 +52,16 @@ Without ``--head`` the gate walks the non-merge commits of ``base..HEAD`` one by one, each against its parent and with its own message. ``base`` is ``--base``, else ``$PINEFORGE_DOC_REVERT_BASE``, else the merge base with -``origin/main`` or ``main``, else -- in a checkout with no branch refs, as the -lab's remote hosts have -- the newest first-parent ancestor of ``HEAD`` whose -subject ends in a pull request number, `` (#N)``: the squash or merge commit -GitHub wrote on ``main``, which is the merge base CI would find (its branch's -commits were judged one by one, with their own messages, before it merged; a -squash commit's message is the pull request's title and cannot name them -again). Where none resolves, the walk starts at ``HEAD`` and stops at the first -commit that predates this gate. A commit whose own tree -has no ``scripts/check_doc_reverts.py`` is never judged: the rule binds from -the commit that introduced it on. At most ``MAX_COMMITS`` commits are walked. +``refs/remotes/origin/main`` or ``refs/heads/main``, else -- in a checkout with +no branch refs, as the lab's remote hosts have -- the newest first-parent +ancestor of ``HEAD`` whose subject ends in a pull request number, `` (#N)``: +the squash or merge commit GitHub wrote on ``main``, which is the merge base +CI would find (its branch's commits were judged one by one, with their own +messages, before it merged; a squash commit's message is the pull request's +title and cannot name them again). Where none resolves, the walk starts at +``HEAD`` and stops at the first commit that predates this gate. A commit whose +own tree has no ``scripts/check_doc_reverts.py`` is never judged: the rule binds +from the commit that introduced it on. At most ``MAX_COMMITS`` commits are walked. Exit 0 when every revert and deletion is named, 1 when one is not, 2 when git or a named ref cannot be read. @@ -352,7 +352,7 @@ def default_base(root: Path) -> str | None: env = os.environ.get('PINEFORGE_DOC_REVERT_BASE') if env: return resolve(env, root) - for ref in ('origin/main', 'main'): + for ref in ('refs/remotes/origin/main', 'refs/heads/main'): tip = git('rev-parse', '--verify', '--quiet', ref + '^{commit}', root=root, check=False).strip() if tip: diff --git a/scripts/ci_preflight.py b/scripts/ci_preflight.py index 4d78f37f9..7c4d70670 100644 --- a/scripts/ci_preflight.py +++ b/scripts/ci_preflight.py @@ -1884,7 +1884,7 @@ def ci_workflow_findings(ci: str, native: str, promote: str, cmake: str, or 'BASE_REF: main' not in exacttree or 'git fetch --quiet origin "$BASE_REF" "+refs/pull/$PR_NUMBER/head:refs/remotes/origin/pr-head"' not in exacttree - or 'git merge-base --is-ancestor "$MERGE_SHA" "origin/$BASE_REF"' not in exacttree + or 'git merge-base --is-ancestor "$MERGE_SHA" "refs/remotes/origin/$BASE_REF"' not in exacttree or '--is-ancestor "$HEAD_SHA"' in promote or 'merge_tree=$(git rev-parse "$MERGE_SHA^{tree}")' not in exacttree or 'head_tree=$(git rev-parse "$HEAD_SHA^{tree}")' not in exacttree diff --git a/scripts/test_check_doc_reverts.py b/scripts/test_check_doc_reverts.py index f6536c727..3b2adc1f6 100644 --- a/scripts/test_check_doc_reverts.py +++ b/scripts/test_check_doc_reverts.py @@ -18,11 +18,13 @@ import contextlib import io +import os from pathlib import Path import shutil import subprocess import tempfile import unittest +from unittest import mock import check_doc_reverts as guard @@ -91,6 +93,73 @@ def with_keep_handle(page: str = BASE_PAGE) -> str: return page + '\n' + KEEP_HANDLE + '\n' +class DefaultBase(unittest.TestCase): + def setUp(self) -> None: + env = mock.patch.dict(os.environ, {'PINEFORGE_DOC_REVERT_BASE': ''}) + env.start() + self.addCleanup(env.stop) + + def test_tags_cannot_shadow_branch_refs(self) -> None: + for ref, tag in (('refs/remotes/origin/main', 'origin/main'), + ('refs/heads/main', 'main')): + with self.subTest(ref=ref), repo() as r: + common = r.commit(BASE_PAGE, 'docs: common ancestor') + r.git('checkout', '-q', '-b', 'tag-history') + tag_tip = r.commit(with_keep_handle(), 'docs: unrelated tag history') + r.git('tag', tag, tag_tip) + r.git('checkout', '-q', 'lane') + base = r.commit(BASE_PAGE + '\nThe main branch adds its own sentence.\n', + 'docs: main branch') + r.git('update-ref', ref, base) + r.commit(with_keep_handle(), 'docs: lane after main') + self.assertEqual(r.git('merge-base', 'HEAD', f'refs/tags/{tag}'), common) + self.assertEqual(r.git('merge-base', 'HEAD', ref), base) + self.assertNotEqual(common, base) + self.assertEqual(guard.default_base(r.root), base) + + def test_remote_tracking_main_precedes_local_main(self) -> None: + with repo() as r: + remote = r.commit(BASE_PAGE, 'docs: remote main') + r.git('update-ref', 'refs/remotes/origin/main', remote) + local = r.commit(with_keep_handle(), 'docs: local main') + r.git('update-ref', 'refs/heads/main', local) + r.commit(BASE_PAGE + '\nThe lane adds a new sentence after local main.\n', + 'docs: lane') + self.assertEqual(guard.default_base(r.root), remote) + + def test_local_main_is_used_without_remote_tracking_main(self) -> None: + with repo() as r: + base = r.commit(BASE_PAGE, 'docs: local main') + r.git('update-ref', 'refs/heads/main', base) + r.commit(with_keep_handle(), 'docs: lane') + self.assertEqual(guard.default_base(r.root), base) + + def test_environment_base_precedes_branch_refs(self) -> None: + with repo() as r: + override = r.commit(BASE_PAGE, 'docs: explicit base') + remote = r.commit(with_keep_handle(), 'docs: remote main') + r.git('update-ref', 'refs/remotes/origin/main', remote) + local = r.commit(BASE_PAGE + '\nLocal main adds another sentence.\n', + 'docs: local main') + r.git('update-ref', 'refs/heads/main', local) + r.commit(with_keep_handle(), 'docs: lane') + with mock.patch.dict(os.environ, {'PINEFORGE_DOC_REVERT_BASE': override}): + self.assertEqual(guard.default_base(r.root), override) + + def test_merged_pull_request_is_used_without_branch_refs(self) -> None: + with repo() as r: + r.commit(BASE_PAGE, 'docs: initial guide') + base = r.commit(with_keep_handle(), 'docs: merged guide (#12)') + r.commit(BASE_PAGE + '\nThe lane adds a new sentence after the merge.\n', + 'docs: lane') + self.assertEqual(guard.default_base(r.root), base) + + def test_no_branch_ref_or_merged_pull_request_has_no_base(self) -> None: + with repo() as r: + r.commit(BASE_PAGE, 'docs: initial guide') + self.assertIsNone(guard.default_base(r.root)) + + class MustFail(unittest.TestCase): def setup_history(self, r: Repo) -> tuple[str, str]: base = r.commit(BASE_PAGE, 'docs: the guide') diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index dab69bcbe..1fcb30bd1 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -868,6 +868,27 @@ def test_ci_contract_pins_parallel_pr_light_and_full_events(self): self.assertNotEqual(changed[index], original[index]) self.assertNotEqual(ci_workflow_findings(*changed), []) + def test_baseline_promotion_accepts_qualified_base_ref(self): + sources = self.ci_sources() + short = 'git merge-base --is-ancestor "$MERGE_SHA" "origin/$BASE_REF"' + qualified = ('git merge-base --is-ancestor "$MERGE_SHA" ' + '"refs/remotes/origin/$BASE_REF"') + # Supply the same qualified input before and after the workflow fix. + sources[2] = sources[2].replace(short, qualified) + self.assertEqual(sources[2].count(qualified), 1) + self.assertEqual(ci_workflow_findings(*sources, others=self.other_sources()), []) + + def test_baseline_promotion_rejects_short_base_ref(self): + sources = self.ci_sources() + short = 'git merge-base --is-ancestor "$MERGE_SHA" "origin/$BASE_REF"' + qualified = ('git merge-base --is-ancestor "$MERGE_SHA" ' + '"refs/remotes/origin/$BASE_REF"') + # Supply the old command even when the workflow already uses the fix. + sources[2] = sources[2].replace(qualified, short) + self.assertEqual(sources[2].count(short), 1) + self.assertIn('baseline promotion must admit a squash merge by tree equality only', + ci_workflow_findings(*sources, others=self.other_sources())) + def test_baseline_promotion_rejects_later_pr_head_checkout(self): finding = ("baseline promotion must run main's copy and require " "the latest two head statuses") From f8ec601282400568673c6e19213380a12d78915c Mon Sep 17 00:00:00 2001 From: Hao-Liang Wen Date: Sun, 25 Oct 2026 19:10:40 +0800 Subject: [PATCH 7/9] ci(release): the permission-claim text guard also catches backslash-wrapped, bytes-decoded and fullwidth respellings (PROBE-FU) [replaces #493] (#506) * ci(release): the permission-claim text guard also catches backslash-wrapped, bytes-decoded and fullwidth respellings (PROBE-FU, sol P2 1/2/4) The guard in WorkflowPinTests.assert_ruleset_visibility_documentation fails when the gate source, docs/ci.md or release.yml state that the App needs "administration: read". A read of the probe found three respellings it accepted: a Markdown hard break written as a backslash before the newline, a bytes constant (b"administr\x61tion: re\x61d", with or without .decode()), and fullwidth compatibility letters. The collapsed view now treats backslash-newline like whitespace, bytes constants are decoded into the scanned strings, and every view is NFKC-normalised before case folding. Three new tests (12 rows) each read the real artifacts, assert the unmodified baseline passes, mutate a copy and require the specific permission-claim AssertionError; removing only the matching handling turns its rows red. Not closed, by design: YAML double-quoted scalars with escapes in release.yml (sol P2 3) and runtime string construction (sol P3); a static text guard cannot decide them, and the exact-output pins on the refusal text remain the decisive protection. No existing assertion, control or forbidden pattern is changed. The rewritten `view` tuple is code, not a documentation or comment sentence. Lane: ci-fu-pfuguard (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(release): the permission-claim guard scans the casefold-only views as well as the NFKC views (PROBE-FU round 2, sol R237 P2) The first PROBE-FU commit normalised every scanned view with NFKC and replaced the casefold-only views. For an input such as `Needs administration: read` followed by the trade mark sign, the old guard matched (the sign is a non-word character after `read`), but after NFKC the text is `readtm` and the word boundary no longer matches, so the claim was accepted: the change made the guard weaker for that input. Each text is now checked in four views (the two original views casefolded, and both NFKC+casefolded), so normalisation can only add matches. A restoration control appends the trade mark case to the gate source, docs/ci.md and the workflow text (direct and line-wrapped, six rows), and a removal mutant drops the casefold-only scan and must break all six rows. No existing assertion or control changes; no documentation or comment sentence is deleted or rewritten. Lane: ci-fu-pfuguard2 (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci: release ruleset probe lists every branch ruleset (measurement only, gate unchanged) [replaces #489] (#505) * ci(release): the ruleset probe lists every branch ruleset as visible or hidden, even when the gate verdict is a refusal (CR-245 F1 measurement) The one dispatch of release-ruleset-probe.yml from main (run 40083495967) refused on the first branch ruleset of the inventory, the organization ruleset `default` (13094758): "returned missing or invalid bypass_actors (token cannot see this ruleset's bypass_actors)". The verdict reader stops at the first failure, so the run did not say whether the release App token can see bypass_actors on the repository ruleset that is the real barrier (21476386), nor on any other. The probe now first prints one line per branch ruleset of the includes_parents inventory (id, name, source_type, source, enforcement, updated_at, bypass_actors visible with the actor count or hidden, applies to main, has a pull_request rule), then runs the unchanged verdict: success prints the existing success lines; a refusal prints the same ::error:: text on stderr and exits 1, but stdout now carries the listing. A detail read that fails is reported on its own row and the listing continues. The listing is a measurement: `gate`, `_release_bump` and `_require_release_rulesets` are untouched, every gate test and the existing source mutants are kept, and the probe's verdict replay over every existing ruleset fixture still compares verdict and refusal text byte for byte. The workflow release-ruleset-probe.yml is not changed. The listing reads each branch ruleset detail independently of the verdict read (two reads per detail in the probe), which docs/ci.md now says. Red-first rows: the real situation (org ruleset hidden, repository ruleset visible: both listed, the org one hidden, refusal unchanged), both hidden, one detail read failing, all visible with a valid barrier, the read-only call list, and a source mutant that drops the listing. Rewrites named for the doc-reverts check, all introduced by 9c18d41 (#488, "ci(release): read-only ruleset probe for the release gate (CR-245 F1)"): docs/ci.md "Success prints the barrier id/name and one compact inventory line with enforcement, source and bypass visibility."; docs/ci.md "Refusal exits 1 with empty stdout and the reader's `::error::` diagnostic; API/response failures name the endpoint."; docs/ci.md "Two hidden rulesets need two dispatches: the first refusal names only the first hidden ruleset; make its `bypass_actors` visible before dispatching again to identify the second." (replaced: all hidden rulesets are listed in one dispatch); the probe docstring "Read exactly the bump's ruleset evidence, with the same refusal policy."; two test comments ("The reader report's complete [] inventory fails on main without an endpoint." and "The runtime assertions already hold there; the prose does not."). The module docstring sentence "Each endpoint/page is read at most once; failures refuse without retry or sleep." (ad1ebac, #479) is scoped to the gate's reads. Lane: ci-fu-cr488m (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(release): the probe listing bounds its inventory=ERROR row to 240 characters like the detail-error rows (CR-245 F1 measurement, sol P2) The new stdout listing printed the whole exception text on an `inventory=ERROR` row, while the per-ruleset `detail=ERROR` rows truncate it to 240 characters; a `gh` transport error with 300 characters of stderr was echoed in full. The row now applies the same bound ([:240], still !r-escaped). Four rows assert the exact bounded line (an inventory transport error, a repository transport error, a default-branch rejection with a long repository argument, an invalid summary with a long field) together with the unchanged stderr, exit code 1 and API call list of the verdict; each has a mutation control that removes only the bound and must fail. The verdict functions are untouched and the verdict's own stderr is not bounded. No documentation sentence is changed (none described a length limit). Lane: ci-fu-cr488n (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --------- Co-authored-by: ci-cx Co-authored-by: Claude Sonnet 5.5 --- scripts/test_ci_release_gate.py | 109 +++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/scripts/test_ci_release_gate.py b/scripts/test_ci_release_gate.py index 8de073392..7ab6cf71a 100644 --- a/scripts/test_ci_release_gate.py +++ b/scripts/test_ci_release_gate.py @@ -11,6 +11,7 @@ import re import subprocess import tempfile +import unicodedata import unittest from unittest import mock @@ -2262,6 +2263,8 @@ def assert_ruleset_visibility_documentation(self, source, docs, workflow): def constant_string(node): if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value + if isinstance(node, ast.Constant) and isinstance(node.value, bytes): + return node.value.decode("utf-8", errors="replace") if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): left = constant_string(node.left) right = constant_string(node.right) @@ -2276,8 +2279,13 @@ def constant_string(node): if value is not None: texts.append(value) for text in texts: - for view in (text.replace("`", "").casefold(), - re.sub(r"""(?:#|//|['"`]|\s)+""", " ", text).casefold()): + views = (text.replace("`", ""), + re.sub(r"""(?:\\\r?\n|#|//|['"`]|\s)+""", " ", text)) + # Compatibility normalization must only add matches to the original views. + casefold_views = [view.casefold() for view in views] + normalized_views = [unicodedata.normalize("NFKC", view).casefold() + for view in views] + for view in casefold_views + normalized_views: self.assertNotRegex(view, r"\badministration\s*:\s*read\b|\bapp\s+requires\b", "unverified App permission claim must stay absent") @@ -2386,6 +2394,103 @@ def test_ruleset_visibility_guard_rejects_wrapped_and_quoted_claims(self): AssertionError, "unverified App permission claim must stay absent"): self.assert_ruleset_visibility_documentation(*changed) + def test_ruleset_visibility_guard_rejects_backslash_wrapped_claims(self): + original = [Path(ci_release_gate.__file__).read_text(), + (ROOT / "docs/ci.md").read_text(), self.workflow()] + self.assert_ruleset_visibility_documentation(*original) + for index, claim in ( + (1, "\nNeeds administration:\\\nread\n"), + (2, "\n# Needs administration:\\\n# read\n")): + with self.subTest(place=index, claim=claim): + changed = original.copy() + changed[index] += claim + self.assertNotEqual(changed[index], original[index]) + with self.assertRaisesRegex( + AssertionError, "unverified App permission claim must stay absent"): + self.assert_ruleset_visibility_documentation(*changed) + + def test_ruleset_visibility_guard_rejects_decoded_bytes_claims(self): + original = [Path(ci_release_gate.__file__).read_text(), + (ROOT / "docs/ci.md").read_text(), self.workflow()] + self.assert_ruleset_visibility_documentation(*original) + for expression in ( + 'b"administr\\x61tion: re\\x61d"', + 'b"administr\\x61tion: re\\x61d".decode()', + 'b"administr\\x61tion:" + b" re\\x61d"', + 'b"\\xffadministr\\x61tion: re\\x61d".decode("utf-8", errors="replace")'): + with self.subTest(expression=expression): + changed = original.copy() + changed[0] += f"\nPERMISSION_CLAIM = {expression}\n" + self.assertNotEqual(changed[0], original[0]) + ast.parse(changed[0]) + with self.assertRaisesRegex( + AssertionError, "unverified App permission claim must stay absent"): + self.assert_ruleset_visibility_documentation(*changed) + + def test_ruleset_visibility_guard_rejects_fullwidth_claims(self): + original = [Path(ci_release_gate.__file__).read_text(), + (ROOT / "docs/ci.md").read_text(), self.workflow()] + self.assert_ruleset_visibility_documentation(*original) + for index in range(len(original)): + for claim in ("\n# Ａｐｐ requires\n", "\n# Ａｐｐ\n# requires\n"): + with self.subTest(place=index, claim=claim): + changed = original.copy() + changed[index] += claim + self.assertNotEqual(changed[index], original[index]) + with self.assertRaisesRegex( + AssertionError, "unverified App permission claim must stay absent"): + self.assert_ruleset_visibility_documentation(*changed) + + def test_ruleset_visibility_guard_rejects_compatibility_suffix_claims(self): + original = [Path(ci_release_gate.__file__).read_text(), + (ROOT / "docs/ci.md").read_text(), self.workflow()] + self.assert_ruleset_visibility_documentation(*original) + for index, claim in ( + (0, "\n# Needs administration: read\u2122\n"), + (1, "\nNeeds administration: read\u2122\n"), + (2, "\n# Needs administration: read\u2122\n"), + (0, "\n# Needs administration:\n# read\u2122\n"), + (1, "\nNeeds administration:\n# read\u2122\n"), + (2, "\n# Needs administration:\n# read\u2122\n")): + with self.subTest(place=index, claim=claim): + changed = original.copy() + changed[index] += claim + self.assertNotEqual(changed[index], original[index]) + with self.assertRaisesRegex( + AssertionError, "unverified App permission claim must stay absent"): + self.assert_ruleset_visibility_documentation(*changed) + + def test_ruleset_visibility_guard_casefold_scan_removal_breaks_restoration_controls(self): + regression = "test_ruleset_visibility_guard_rejects_compatibility_suffix_claims" + original_result = unittest.TestResult() + WorkflowPinTests(regression).run(original_result) + self.assertEqual(original_result.testsRun, 1) + self.assertEqual(original_result.errors, []) + self.assertEqual(original_result.failures, [], + "restoration controls must pass before mutation") + + source = Path(__file__).read_text() + before = " for view in casefold_views + normalized_views:\n" + self.assertEqual(source.count(before), 1, "mutation must have one source target") + mutated = source.replace(before, " for view in normalized_views:\n", 1) + self.assertNotEqual(mutated, source) + guard = next(node for node in ast.walk(ast.parse(mutated)) + if isinstance(node, ast.FunctionDef) + and node.name == "assert_ruleset_visibility_documentation") + namespace = {"ast": ast, "re": re, "unicodedata": unicodedata} + exec(compile(ast.Module(body=[guard], type_ignores=[]), __file__, "exec"), namespace) + case = WorkflowPinTests(regression) + case.assert_ruleset_visibility_documentation = namespace[ + "assert_ruleset_visibility_documentation"].__get__(case) + result = unittest.TestResult() + case.run(result) + self.assertEqual(result.testsRun, 1) + self.assertEqual(result.errors, []) + self.assertEqual(len(result.failures), 6, + "removing the casefold-only scan must break every restoration row") + for _, traceback in result.failures: + self.assertIn("AssertionError not raised", traceback) + def test_prebuilt_builds_the_emitted_sha_and_asserts_tag_identity(self): jobs = _jobs(self.workflow()) self.assertRegex(jobs["release"], From 0a43c2cd8424e74af301da57a9e92762659d0297 Mon Sep 17 00:00:00 2001 From: Hao-Liang Wen Date: Sun, 25 Oct 2026 19:11:30 +0800 Subject: [PATCH 8/9] tests: drop the TIMEOUT 180 ctest property (owner's no-time-limit rule, P2-CM-F4) (#504) * tests: drop the TIMEOUT 180 property from the six ownership manifest tests (P2-CM-F4) One set_tests_properties call carried TIMEOUT 180 for test_selected_window_ownership_manifest, the four test_selected_window_ownership_manifest_controls_shard_{1..4}of4 and test_selected_window_ownership_d4. The owner's no-time-limit rule forbids a ctest ceiling: under pool contention shard 1of4 (119 s normally) hit the 180 s ceiling on the b250ceba merge-commit ci-post (#283). WORKING_DIRECTORY is kept; no test, label or line count changes. Co-Authored-By: Claude Sonnet 5.5 * ci: release ruleset probe lists every branch ruleset (measurement only, gate unchanged) [replaces #489] (#505) * ci(release): the ruleset probe lists every branch ruleset as visible or hidden, even when the gate verdict is a refusal (CR-245 F1 measurement) The one dispatch of release-ruleset-probe.yml from main (run 40083495967) refused on the first branch ruleset of the inventory, the organization ruleset `default` (13094758): "returned missing or invalid bypass_actors (token cannot see this ruleset's bypass_actors)". The verdict reader stops at the first failure, so the run did not say whether the release App token can see bypass_actors on the repository ruleset that is the real barrier (21476386), nor on any other. The probe now first prints one line per branch ruleset of the includes_parents inventory (id, name, source_type, source, enforcement, updated_at, bypass_actors visible with the actor count or hidden, applies to main, has a pull_request rule), then runs the unchanged verdict: success prints the existing success lines; a refusal prints the same ::error:: text on stderr and exits 1, but stdout now carries the listing. A detail read that fails is reported on its own row and the listing continues. The listing is a measurement: `gate`, `_release_bump` and `_require_release_rulesets` are untouched, every gate test and the existing source mutants are kept, and the probe's verdict replay over every existing ruleset fixture still compares verdict and refusal text byte for byte. The workflow release-ruleset-probe.yml is not changed. The listing reads each branch ruleset detail independently of the verdict read (two reads per detail in the probe), which docs/ci.md now says. Red-first rows: the real situation (org ruleset hidden, repository ruleset visible: both listed, the org one hidden, refusal unchanged), both hidden, one detail read failing, all visible with a valid barrier, the read-only call list, and a source mutant that drops the listing. Rewrites named for the doc-reverts check, all introduced by 9c18d41 (#488, "ci(release): read-only ruleset probe for the release gate (CR-245 F1)"): docs/ci.md "Success prints the barrier id/name and one compact inventory line with enforcement, source and bypass visibility."; docs/ci.md "Refusal exits 1 with empty stdout and the reader's `::error::` diagnostic; API/response failures name the endpoint."; docs/ci.md "Two hidden rulesets need two dispatches: the first refusal names only the first hidden ruleset; make its `bypass_actors` visible before dispatching again to identify the second." (replaced: all hidden rulesets are listed in one dispatch); the probe docstring "Read exactly the bump's ruleset evidence, with the same refusal policy."; two test comments ("The reader report's complete [] inventory fails on main without an endpoint." and "The runtime assertions already hold there; the prose does not."). The module docstring sentence "Each endpoint/page is read at most once; failures refuse without retry or sleep." (ad1ebac, #479) is scoped to the gate's reads. Lane: ci-fu-cr488m (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(release): the probe listing bounds its inventory=ERROR row to 240 characters like the detail-error rows (CR-245 F1 measurement, sol P2) The new stdout listing printed the whole exception text on an `inventory=ERROR` row, while the per-ruleset `detail=ERROR` rows truncate it to 240 characters; a `gh` transport error with 300 characters of stderr was echoed in full. The row now applies the same bound ([:240], still !r-escaped). Four rows assert the exact bounded line (an inventory transport error, a repository transport error, a default-branch rejection with a long repository argument, an invalid summary with a long field) together with the unchanged stderr, exit code 1 and API call list of the verdict; each has a mutation control that removes only the bound and must fail. The verdict functions are untouched and the verdict's own stderr is not bounded. No documentation sentence is changed (none described a length limit). Lane: ci-fu-cr488n (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 30c04079c..f54fc71b4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3825,7 +3825,7 @@ add_test(NAME test_selected_window_ownership_d4 ManifestCheckerTests.test_d4_independent_reconstruction_matches_manifest) set_tests_properties(test_selected_window_ownership_manifest ${_pf_ownership_control_shards} test_selected_window_ownership_d4 - PROPERTIES WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} TIMEOUT 180) + PROPERTIES WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) # The C bridge test also links the shared window-plan helper, whose bridge # (src/selected_window_plan_c.cpp) is not in the kernel archive. The loop above From be9f20d0087b66ab6a85138a90998ed444d62dd4 Mon Sep 17 00:00:00 2001 From: Hao-Liang Wen Date: Sun, 25 Oct 2026 19:31:52 +0800 Subject: [PATCH 9/9] ci: pin every third-party action to a full commit SHA with a version comment, and lint it in preflight (ORG-ACTIONPINS) [replaces #495] (#508) * ci: pin every third-party action in the engine workflows to a full commit SHA with a version comment (ORG-ACTIONPINS) 66 `uses:` lines changed in .github/workflows/*.yml and .github/actions/setup-live-curl/action.yml: 65 mutable tags rewritten to `@<40-hex sha> # v` and one bare SHA that lacked its comment. Each pin is the same major the file used before; no step, job, permission, environment, secret or token mint changes (a before/after comparison of the diff with the refs stripped is identical line for line). Resolved on 2026-10-24 22:48 with `git ls-remote --tags` on a spot box; every tag is lightweight, so the tag's sha is the commit: actions/checkout v4 11d5960a326750d5838078e36cf38b85af677262 v4.4.0 actions/upload-artifact v4 ea165f8d65b6e75b540449e92b4886f43607fa02 v4.6.2 actions/setup-python v5 a26af69be951a213d495a4c3e4e4022e16d87065 v5.6.0 actions/cache, cache/save, cache/restore v4 0057852bfaa89a56745cba8c7296529d2fc39830 v4.3.0 actions/create-github-app-token v1 d72941d797fd3113feb6b93fd0dec494b13a2547 v1.12.0 cloudflare/wrangler-action v3 9acf94ace14e7dc412b076f2c5c20b8ce93c79cd v3.15.0 The byte-pinned copies of the workflow text that scripts/ci_preflight.py, scripts/test_ci_preflight.py and scripts/test_ci_release_gate.py compare against (the embedded probe workflow, the release and cache expectations, the Wrangler deployment, test mutations and fixtures) are moved to the same pinned text so each still equals its workflow. Two tests read the real workflows and composite action and require a 40-hex sha and a version comment on every third-party reference. No documentation or comment sentence is deleted or rewritten. Lane: ci-fu-pins (Codex gpt-6.1-sol). Co-Authored-By: Claude Sonnet 5.5 * ci(preflight): refuse any third-party `uses:` that is not a 40-hex commit SHA with a same-line version comment; the lint accepts plain block-style YAML only (ORG-ACTIONPINS) scripts/ci_preflight.py gains `action_pin_findings`: every `uses:` in .github/workflows/*.yml and every composite .github/actions/**/action.yml (or .yaml) that is not a local `./` path or a `docker://` image must be `owner/action[/path]@<40 lowercase hex> # v` on the same line; quoted values, step and job-level `uses:`, remote reusable workflows and a ref-less `uses:` are judged; the finding names file, line and the offending reference. The lint is a regression guard for our own reviewed workflow files, so it does not parse arbitrary YAML: it accepts plain block-style YAML and refuses, with a finding naming file, line, construct and source text, what it cannot follow: anchors, aliases, merge keys, explicit keys, tags, non-empty flow mappings, quoted keys, backslash escapes in quoted values, quoted values that do not end on their line, tab indentation, document markers, `uses` without a same-line scalar, unrecognised block lines, non-ASCII whitespace in structural positions, and any non-LF line break (bare CR, NEL, LS, PS, VT, FF) or C0 control character in the whole file. Allowed (surveyed on the real tree, twelve YAML files): plain block mappings and sequences, plain and quoted scalar values, empty `{}` and `[]` as a whole plain value, non-empty flow sequences, block-scalar bodies including `- |`. Quotes inside flow sequences count only at element start; delimiters of a refused flow collection are counted without quote masking and the refusal region ends at the first outdent. The `${{ ... }}` scan is linear. The loaders read the bytes and decode strict UTF-8 (a file that is not UTF-8 is a finding), so a bare CR is no longer erased before the refusal runs. The documented limit for exotic YAML is in docs/ci.md. The lint was read by sol and argon over five rounds (every finding closed or classified); the loader and the three promotion-guard tests that now also see the lint's quoted-key finding were fixed from the lab legs. Tests: 15 first-round and the later red-first tests (one per refused construct, each break character in a block-scalar body and in a comment, the C0 controls, flow-quote variants, outdent resumption, loader tests on real files in a temp layout, a real-tree zero-findings test, mutation controls for the refusal branch and the integration call). docs/ci.md describes the rule. This commit is the squash of the lint rounds of the series; the pins commit before it rewrites the workflow files. Lanes: ci-fu-pinlint, ci-fu-pins2/3/4/5 (Codex gpt-6.1-sol), ci-fu-pins6 and pins7 (Claude Sonnet 5.5 via claudock). Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: ci-cx Co-authored-by: Claude Sonnet 5.5 --- .github/actions/setup-live-curl/action.yml | 4 +- .github/workflows/ci-post.yml | 48 +- .github/workflows/ci.yml | 16 +- .github/workflows/corpus-parity-subset.yml | 10 +- .github/workflows/corpus-parity.yml | 18 +- .github/workflows/docs-post.yml | 6 +- .github/workflows/docs.yml | 4 +- .github/workflows/native-live.yml | 10 +- .github/workflows/release-ruleset-probe.yml | 4 +- .github/workflows/release.yml | 12 +- docs/ci.md | 53 + scripts/ci_preflight.py | 439 +++++++- scripts/test_ci_preflight.py | 1055 ++++++++++++++++++- scripts/test_ci_release_gate.py | 2 +- 14 files changed, 1547 insertions(+), 134 deletions(-) diff --git a/.github/actions/setup-live-curl/action.yml b/.github/actions/setup-live-curl/action.yml index 2ebe7b6b4..f09f708fe 100644 --- a/.github/actions/setup-live-curl/action.yml +++ b/.github/actions/setup-live-curl/action.yml @@ -34,7 +34,7 @@ runs: - name: Restore pinned WebSocket-enabled libcurl id: curl-cache if: ${{ inputs.cold-cache != 'true' }} - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: build-native-deps/curl-install key: curl-${{ inputs.cache-namespace }}-${{ steps.dependencies.outputs.version }}-${{ steps.dependencies.outputs.sha256 }}-${{ steps.dependencies.outputs.identity }}-${{ hashFiles('.github/actions/setup-live-curl/action.yml', 'scripts/build_live_curl.sh') }}-${{ runner.os }}-${{ runner.arch }} @@ -48,7 +48,7 @@ runs: # job timeouts must not prevent a valid pinned installation from surviving. - name: Save pinned WebSocket-enabled libcurl if: ${{ inputs.cold-cache != 'true' && steps.curl-build.outcome == 'success' && steps.curl-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: build-native-deps/curl-install key: ${{ steps.curl-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/ci-post.yml b/.github/workflows/ci-post.yml index 1d2352d53..a04227c7c 100644 --- a/.github/workflows/ci-post.yml +++ b/.github/workflows/ci-post.yml @@ -121,7 +121,7 @@ jobs: steps: # The tested commit and its parents: a pull request's merge and the base # it merged into, a push's commit and its parent. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 2 persist-credentials: false @@ -171,7 +171,7 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false @@ -203,7 +203,7 @@ jobs: - name: Retain preflight diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-preflight path: build-ci-preflight/ @@ -220,13 +220,13 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' - name: Install the harness test runner @@ -239,7 +239,7 @@ jobs: run: python3 scripts/ci_preflight.py --output-dir build-ci-preflight-slow --tier slow - name: Retain preflight diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-preflight-slow path: build-ci-preflight-slow/ @@ -286,14 +286,14 @@ jobs: ubuntu-24.04:Linux|macos-26:macOS) ;; *) echo "::error::post fleet runner OS $ACTUAL_OS cannot verify $REQUESTED_PLATFORM"; exit 1 ;; esac - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -317,7 +317,7 @@ jobs: # fallback. Only main writes this namespace, never a PR merge ref. - name: Restore ccache id: ccache-restore - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-main-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-${{ github.sha }} @@ -346,7 +346,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-${{ matrix.platform.os }}-${{ matrix.build_type }} path: ci-diagnostics/ @@ -354,7 +354,7 @@ jobs: - name: Save main ccache if: ${{ success() && github.ref == 'refs/heads/main' && steps.ccache-restore.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ${{ steps.ccache-restore.outputs.cache-primary-key }} @@ -388,14 +388,14 @@ jobs: ubuntu-24.04:Linux|macos-26:macOS) ;; *) echo "::error::post fleet runner OS $ACTUAL_OS cannot verify $REQUESTED_PLATFORM"; exit 1 ;; esac - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -416,7 +416,7 @@ jobs: # Keep the main cache namespace shared with PR restores. Only main saves. - name: Restore ccache id: ccache-restore - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-main-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-${{ github.sha }} @@ -440,7 +440,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-${{ matrix.os }}-${{ matrix.build_type }} path: ci-diagnostics/ @@ -448,7 +448,7 @@ jobs: - name: Save main ccache if: ${{ success() && github.ref == 'refs/heads/main' && steps.ccache-restore.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ${{ steps.ccache-restore.outputs.cache-primary-key }} @@ -477,14 +477,14 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -504,7 +504,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev tzdata-legacy ccache - name: Restore ccache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers-${{ github.sha }} @@ -521,7 +521,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-sanitizers path: ci-diagnostics/ @@ -547,14 +547,14 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -574,7 +574,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev tzdata-legacy libsqlite3-dev libssl-dev zlib1g-dev ccache - name: Restore ccache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-kernel-${{ github.sha }} @@ -596,7 +596,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-kernel path: ci-diagnostics/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011e6e016..b585e4a30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: # The tested commit and its parents: a pull request's merge and the base # it merged into, a push's commit and its parent. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 2 persist-credentials: false @@ -239,7 +239,7 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false @@ -271,7 +271,7 @@ jobs: - name: Retain preflight diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-preflight path: build-ci-preflight/ @@ -308,14 +308,14 @@ jobs: run: | echo "::error::authz did not pass; nothing of this run's code is checked out or run" exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: ${{ needs.authz.result == 'success' }} with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -332,7 +332,7 @@ jobs: # fallback. Only main writes this namespace, never a PR merge ref. - name: Restore ccache id: ccache-restore - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-main-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-${{ github.sha }} @@ -370,7 +370,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-${{ matrix.platform.os }}-${{ matrix.build_type }} path: ci-diagnostics/ @@ -378,7 +378,7 @@ jobs: - name: Save main ccache if: ${{ success() && github.ref == 'refs/heads/main' && steps.ccache-restore.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ${{ steps.ccache-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/corpus-parity-subset.yml b/.github/workflows/corpus-parity-subset.yml index de92cac5c..8a228765a 100644 --- a/.github/workflows/corpus-parity-subset.yml +++ b/.github/workflows/corpus-parity-subset.yml @@ -107,12 +107,12 @@ jobs: # Credentials stay persisted, exactly as the nightly job above: the # submodule clone and `git lfs pull` below go out over https and the # nightly is the configuration that is known to work for them. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -147,7 +147,7 @@ jobs: - name: Restore the corpus chart feed id: feed - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: corpus/data key: corpus-feed-${{ steps.pin.outputs.gitlink }} @@ -175,7 +175,7 @@ jobs: done - name: Restore ccache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-corpus-parity-subset-${{ github.sha }} @@ -207,7 +207,7 @@ jobs: - name: Retain parity diagnostics if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: corpus-parity-subset-diagnostics path: corpus-parity-diagnostics/ diff --git a/.github/workflows/corpus-parity.yml b/.github/workflows/corpus-parity.yml index e2366c4a5..857e14823 100644 --- a/.github/workflows/corpus-parity.yml +++ b/.github/workflows/corpus-parity.yml @@ -147,12 +147,12 @@ jobs: # the only hang catcher (owner rule, # 2026-10-12: no bound of our own may fail a run for slowness). steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -177,7 +177,7 @@ jobs: git -C corpus status --porcelain -- validation - name: Restore ccache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-corpus-parity-${{ github.sha }} @@ -208,7 +208,7 @@ jobs: - name: Retain parity diagnostics if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: corpus-parity-diagnostics path: corpus-parity-diagnostics/ @@ -228,12 +228,12 @@ jobs: # Credentials stay persisted, exactly as the nightly job above: the # submodule clone and `git lfs pull` below go out over https and the # nightly is the configuration that is known to work for them. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -268,7 +268,7 @@ jobs: - name: Restore the corpus chart feed id: feed - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: corpus/data key: corpus-feed-${{ steps.pin.outputs.gitlink }} @@ -296,7 +296,7 @@ jobs: done - name: Restore ccache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-corpus-parity-subset-${{ github.sha }} @@ -328,7 +328,7 @@ jobs: - name: Retain parity diagnostics if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: corpus-parity-subset-diagnostics path: corpus-parity-diagnostics/ diff --git a/.github/workflows/docs-post.yml b/.github/workflows/docs-post.yml index 4f7989571..7ff21f073 100644 --- a/.github/workflows/docs-post.yml +++ b/.github/workflows/docs-post.yml @@ -93,7 +93,7 @@ jobs: CF_PROJECT: ${{ vars.CF_PAGES_PROJECT || 'pineforge-docs' }} DEPLOY_BRANCH: ${{ github.ref_name }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 # need tags for git describe in build.sh @@ -122,7 +122,7 @@ jobs: bash docs/build.sh - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: docs-html path: | @@ -138,7 +138,7 @@ jobs: echo "branch=$SAFE" >> "$GITHUB_OUTPUT" - name: Deploy to Cloudflare Pages - uses: cloudflare/wrangler-action@v3 + uses: cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd # v3.15.0 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 30bac62db..965fb4741 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -80,7 +80,7 @@ jobs: needs: [authz] runs-on: ["self-hosted", "pf-gcp-pr"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 # need tags for git describe in build.sh @@ -109,7 +109,7 @@ jobs: bash docs/build.sh - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: docs-html path: | diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index e51cda630..44ae91a22 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -124,13 +124,13 @@ jobs: PF_CI_LABELS: '["self-hosted","pf-ci","pf-ci-x64-16"]' CCACHE_DIR: ${{ github.workspace }}/.ccache/${{ matrix.profile }} steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -150,7 +150,7 @@ jobs: - name: Restore ccache id: ccache-restore if: ${{ !inputs.cold_cache }} - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ runner.arch }}-live-v2-${{ matrix.profile }}-${{ github.sha }} @@ -190,7 +190,7 @@ jobs: - name: Retain CI diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-diagnostics-native-live-${{ matrix.profile }} path: ci-diagnostics/ @@ -200,7 +200,7 @@ jobs: # after diagnostics, even on normal failure; cancellation is no cache plan. - name: Save ccache after verification if: ${{ !cancelled() && !inputs.cold_cache && steps.verify.outcome != 'skipped' && steps.ccache-restore.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} key: ${{ steps.ccache-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/release-ruleset-probe.yml b/.github/workflows/release-ruleset-probe.yml index 3a7860f6e..ae34bcff5 100644 --- a/.github/workflows/release-ruleset-probe.yml +++ b/.github/workflows/release-ruleset-probe.yml @@ -86,7 +86,7 @@ jobs: - name: Mint org App token id: app - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.PINEFORGE_APP_ID }} private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }} @@ -95,7 +95,7 @@ jobs: - name: Checkout if: ${{ needs.authz.result == 'success' }} - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ba230255..b0abd7674 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,7 +94,7 @@ jobs: # ci_release_gate.py pins that login for the one-bump phase-1 exemption. - name: Mint org App token id: app - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.PINEFORGE_APP_ID }} private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }} @@ -102,7 +102,7 @@ jobs: repositories: pineforge-engine - name: Checkout (full history) - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 # App token bypasses the org ruleset (PR-required) for the bump push. @@ -340,7 +340,7 @@ jobs: - name: Keep the dry run's run-failure catalog assets as a workflow artifact if: ${{ inputs.dry_run }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: run-failure-catalog-${{ steps.bump.outputs.tag }} path: ${{ steps.catalog.outputs.assets }} @@ -381,7 +381,7 @@ jobs: steps: # A dry run has no tag yet: it builds the dispatched commit. - name: Checkout release commit - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ inputs.dry_run && github.sha || needs.release.outputs.sha }} fetch-depth: 0 # the version helper still observes the git sha @@ -511,7 +511,7 @@ jobs: - name: Keep the dry run's tarball as a workflow artifact if: ${{ inputs.dry_run }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ steps.pack.outputs.tarball }} path: | @@ -531,7 +531,7 @@ jobs: steps: - name: Mint org App token (hub dispatch) id: app - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.PINEFORGE_APP_ID }} private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }} diff --git a/docs/ci.md b/docs/ci.md index e0d167de6..e088cd11e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -101,6 +101,59 @@ and fails, never skips, without it. Actionlint only accepts declared custom self-hosted runner labels; preflight checks the declarations in `.github/actionlint.yaml`. +ORG-ACTIONPINS is enforced by preflight's `ci-workflow-contract`: every +third-party `uses:` in `.github/workflows/*.yml` (also `.yaml`) and nested +`.github/actions/**/action.yml` or `action.yaml` must have the shape +`uses: owner/action@<40 lowercase hex commit SHA> # v4.2.2`, with the version +comment on the same line; prerelease and build suffixes are allowed. Local +`./` actions and reusable workflows, and `docker://` images, are exempt. +The lint accepts plain block-style YAML only, with scalar flow sequences +(for example `runs-on: ["self-hosted", "pf-gcp-pr"]` and `branches: [main]`) +and empty `{}` / `[]` on one line as a whole plain `key:` value as exceptions +(ASCII spaces and tabs inside an empty collection are allowed). In flow +sequences, quotes may start a quoted scalar only at the start of an element, +after `[` or `,` and optional ASCII spaces; embedded quotes and mapping +elements are refused. +Keys must be unquoted identifiers (`[A-Za-z_][A-Za-z0-9_-]*`); scalar values +may be plain or single/double quoted on one line, or use `|` / `>` blocks +with chomping and indentation indicators. GitHub `${{ ... }}` expressions, +comments and LF/CRLF line endings are supported. Before scanning any YAML, +the lint refuses a whole file containing a bare CR (U+000D), NEL (U+0085), +LS (U+2028), PS (U+2029), VT (U+000B) or FF (U+000C) as +`unsupported line break`, naming the first offending line and code point. +NUL and every other C0 control except TAB, LF and CR within CRLF are likewise +refused as `unsupported control character`. These file checks include +comments and block-scalar bodies; ordinary non-ASCII text remains supported. +Indented block-scalar bodies are otherwise excluded, including `run:`, +github-script's `with: script:`, and +multiline action inputs. A sequence item such as `- |`, `- >-` or `- |2+` +owns its body from the dash column; `- run: |` uses the key's column. +Whitespace outside block-scalar bodies must be ASCII-only: indentation and +key separators use ASCII space (U+0020). Non-ASCII whitespace and a BOM +(including at the start of the file) are refused as `unsupported whitespace`; +tabs in indentation remain refused. +Unsupported `uses` key shapes are refused: every `uses:` value must be a +scalar on the same line. The lint also refuses anchors, aliases, merge +keys, explicit keys, tags, nonempty flow mappings (including multiline and +implicit mappings inside sequences), all quoted keys, backslashes in YAML +quoted values, tabs in indentation, and `---` / `...` document markers. +Empty collections outside the whole plain-key value exception, extra +inline mapping keys, unrecognised block lines and incomplete quoted/flow +values are refused too. Each refusal names the file, line, construct and +source text and asks for a rewrite in plain block style. +This scanner checks a reviewed YAML subset, not arbitrary YAML syntax. +After refusing a flow collection, it counts braces and brackets on +continuation lines with quotes treated as plain text, ending that region +when depth reaches zero or a nonblank, noncomment line outdents to the +opening line's indentation or less. An aligned closing tail remains part +of the original refusal; other outdented lines resume block scanning. +Delimiter characters inside quoted data or comments in an already-refused +flow can shorten or extend the region until that outdent. Findings inside +such regions are not exhaustive; exotic YAML outside the documented subset +must be rewritten in plain block style. +To bump an action, resolve the desired upstream version to its full commit +SHA and update both the SHA and version comment together. + This checks the CI/native workflow syntax, expressions and shell commands, source ABI/hash/schema guards, and the verifier's failure-handling tests. A failure fails the advisory aggregate `build` job and retains logs in diff --git a/scripts/ci_preflight.py b/scripts/ci_preflight.py index 7c4d70670..94520870d 100644 --- a/scripts/ci_preflight.py +++ b/scripts/ci_preflight.py @@ -20,6 +20,7 @@ from __future__ import annotations import argparse +from collections.abc import Iterator from datetime import datetime, timezone import hashlib import json @@ -30,6 +31,7 @@ import subprocess import sys import time +import unicodedata from ci_verify import (ROOT, PRIVATE_FIXTURE_SKIP_LITERAL, private_fixtures_overlay, source_guard_commands) @@ -736,6 +738,309 @@ def _key_value(line: str, key: str) -> str | None: return '' if value.startswith('#') else value +ACTION_PIN_SHA = re.compile(r'\A[^@\s]+@[0-9a-f]{40}\Z') +ACTION_PIN_COMMENT = re.compile( + r'\A#[ \t]+v[0-9]+(?:\.[0-9]+)*' + r'(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?' + r'(?:\+[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?[ \t]*\Z') +ACTION_PIN_EXEMPT_PREFIXES = ('./', 'docker://') +# This is a whitelist for our reviewed workflow files, not a YAML parser. +# Only plain block keys, scalar values, block scalars and scalar sequences +# are supported; literal {} / [] are allowed as a whole plain-key value. +_ACTION_YAML_KEY = re.compile(r'([A-Za-z_][A-Za-z0-9_-]*) *:(?: +(.*)|$)') +_ACTION_YAML_QUOTED = re.compile(r'"(?:\\.|[^"\\])*"' r"|'(?:''|[^'])*'") +_ACTION_BLOCK_SCALAR = re.compile(r'[|>](?:[+-][1-9]?|[1-9][+-]?)?') +_ACTION_EMPTY_COLLECTION = re.compile(r'\{[ \t]*\}|\[[ \t]*\]') +_ACTION_YAML_FORBIDDEN_CHARACTER = re.compile( + r'\r(?!\n)|[\x00-\x08\x0b\x0c\x0e-\x1f\x85\u2028\u2029]') +_ACTION_YAML_PROPERTIES = {'&': 'anchors', '*': 'aliases', '!': 'tags', + '?': 'explicit keys', ':': 'explicit keys'} + + +def _action_yaml_visible(value: str, *, + sequence: list[str] | None = None) -> tuple[str, str, list[str]]: + """Return code, masked code, quoted tokens; expressions/comments are opaque.""" + visible, quoted, index = [], [], 0 + depth = len(sequence) if sequence else 0 + flow = bool(depth) or value.startswith('[') + element_start = bool(sequence and sequence[-1] in ('empty', 'comma')) + while index < len(value): + char = value[index] + if char == '#' and (index == 0 or value[index - 1] == ' '): + return value[:index].rstrip(' '), ''.join(visible), quoted + if value.startswith('${{', index): + end, in_string = index + 3, False + # Every iteration advances; doubled quotes cannot be repartitioned + # into adjacent strings when an expression has no closing }}. + while end < len(value): + if value[end] == "'": + if in_string and value.startswith("''", end): + end += 2 + continue + in_string = not in_string + elif not in_string and value.startswith('}}', end): + end += 2 + break + end += 1 + else: + visible.append(value[index:]) + break + token = value[index:end] + elif char in ('"', "'") and (not flow or depth and element_start): + end = index + 1 + while end < len(value): + if char == '"' and value[end] == '\\': + end += 2 + continue + if value[end] == char: + if char == "'" and value.startswith("''", end): + end += 2 + continue + end += 1 + break + end += 1 + else: + # Keep the opening quote visible so the grammar refuses it. + visible.append(value[index:]) + break + token = value[index:end] + quoted.append(token) + else: + visible.append(char) + if flow: + if char == '[': + depth += 1 + element_start = True + elif char == ']': + depth = max(0, depth - 1) + element_start = False + elif char == ',': + element_start = True + elif char != ' ': + # An embedded quote remains data, never a masking token. + element_start = False + index += 1 + continue + # Retain positions and a nonblank scalar placeholder, including for "". + visible.append('v' + ' ' * (len(token) - 1)) + element_start = False + index = end + return value.rstrip(' '), ''.join(visible), quoted + + +def _action_yaml_value(value: str, *, whole_key: bool, sequence: list[str]) -> str | None: + """Accept a scalar or scalar flow sequence; return a construct to refuse.""" + code, visible, quoted = _action_yaml_visible(value, sequence=sequence) + if not code: + return None + if not sequence: + if code[0] in _ACTION_YAML_PROPERTIES: + return _ACTION_YAML_PROPERTIES[code[0]] + if whole_key and _ACTION_EMPTY_COLLECTION.fullmatch(code): + return None + if code.startswith('{'): + return 'flow mappings' + if code.startswith(('"', "'")): + if not _ACTION_YAML_QUOTED.fullmatch(code): + return 'quoted scalars' + return 'quoted escapes' if '\\' in code else None + if _ACTION_BLOCK_SCALAR.fullmatch(code): + return None + if not code.startswith('['): + # Quotes embedded in a plain scalar are data (e.g. shell printf), + # while a colon separator would introduce another mapping key. + if re.search(r':(?:[ \t]|$)', visible): + return 'inline mapping keys' + if code[0] in ']}|>%@`,' or re.match(r'-(?:[ \t]|$)', code): + return 'block structure' + return 'expressions' if '${{' in visible else None + # Flow sequences may span lines, but their items must remain scalars or + # nonempty sequences: even an implicit mapping without braces is refused. + if re.search(r'[{}]|:(?=[ \t,\]}]|$)', visible): + return 'flow mappings' + property_token = re.search(r'(?:^|[ \t,\[])([&*!?:])', visible) + if property_token: + return _ACTION_YAML_PROPERTIES[property_token.group(1)] + if any('\\' in token for token in quoted): + return 'quoted escapes' + if re.search(r'''["']''', visible): + return 'quoted scalars' + if '${{' in visible: + return 'expressions' + closed = False + for char in visible: + if char in ' \t': + continue + if closed: + return 'flow sequences' + if char == '[': + if sequence: + if sequence[-1] in ('value', 'closed'): + return 'flow sequences' + sequence[-1] = 'value' + sequence.append('empty') + elif char == ']': + if not sequence: + return 'flow sequences' + if sequence.pop() == 'empty': + return 'empty collections' + if sequence: + sequence[-1] = 'closed' + else: + closed = True + elif char == ',': + if not sequence or sequence[-1] not in ('value', 'closed'): + return 'flow sequences' + sequence[-1] = 'comma' + else: + if not sequence or sequence[-1] == 'closed': + return 'flow sequences' + if sequence[-1] != 'value' and char in '|>%@`': + return 'flow sequences' + sequence[-1] = 'value' + return None + + +def action_uses(sources: dict[str, str], *, + findings: list[str]) -> Iterator[tuple[str, int, str, str]]: + """Whitelist workflow YAML, then yield plain block uses scalars for pinning.""" + for name, text in sources.items(): + # Refuse the raw file before any scalar-body, comment or flow skip. + # LF and CRLF are the only supported line endings, including in data. + offense = _ACTION_YAML_FORBIDDEN_CHARACTER.search(text) + if offense is not None: + char = offense.group() + number = text.count('\n', 0, offense.start()) + 1 + source_line = text.replace('\r\n', '\n').split('\n')[number - 1].strip(' \t') + construct = ('unsupported line break' if char in '\r\x85\u2028\u2029\v\f' + else 'unsupported control character') + findings.append( + f'{name}:{number}: unsupported YAML construct in a workflow ' + f'({construct} U+{ord(char):04X}): ' + f'rewrite it in plain block style; {source_line!r}') + continue + scalar_depth = None + sequence = [] + sequence_start = None + refused_flow_depth = 0 + refused_flow_indent = 0 + for number, line in enumerate(text.replace('\r\n', '\n').split('\n'), 1): + if scalar_depth is not None: + if not line.strip(' ') or _indent(line) > scalar_depth: + continue + scalar_depth = None + unsupported_whitespace = any( + char == '\ufeff' or char not in ' \t' and + unicodedata.category(char) in {'Zs', 'Zl', 'Zp', 'Cc'} + for char in line) + if not unsupported_whitespace and ( + not line.strip(' ') or line.lstrip(' ').startswith('#')): + continue + if refused_flow_depth: + if _indent(line) <= refused_flow_indent: + refused_flow_depth = 0 + # An aligned closing tail belongs to the refused flow; + # any other outdent is scanned again as a block line. + if line.lstrip(' ').startswith(('}', ']')): + continue + else: + # This collection is already refused. Quotes are plain + # text here: they must never mask a closing delimiter. + for char in line: + if char in '{[': + refused_flow_depth += 1 + elif char in '}]': + refused_flow_depth -= 1 + if not refused_flow_depth: + break + continue + stripped = line.strip(' ') + step = re.match(r'-[ \t]+', stripped) + key_line = stripped[step.end():] if step and not sequence else stripped + key = _ACTION_YAML_KEY.fullmatch(key_line) if not sequence else None + first = (key.group(2) or '').strip(' ') if key else None + node = first if first is not None else key_line + code, visible, _ = _action_yaml_visible(node, sequence=sequence) + sequence_depth = len(sequence) + # Recover only the extent of a refused property-prefixed block, + # so an anchored script produces its anchor finding, not script + # uses findings. This never authorizes the property itself. + recovery = re.sub(r'^(?:[&*!][^ \t]+[ \t]*)+', '', code) + if not sequence and _ACTION_BLOCK_SCALAR.fullmatch(recovery): + # A bare "- |" owns its body from the dash column; "- run: |" + # owns it from the mapping key's column after the dash. + scalar_depth = _indent(line) + (step.end() if step and key else 0) + quoted_key = _ACTION_YAML_QUOTED.match(key_line) + if unsupported_whitespace: + unsupported = 'unsupported whitespace' + elif '\t' in re.match(r'[ \t]*', line).group() or ( + step and '\t' in step.group()): + unsupported = 'tab indentation' + elif re.match(r'(?:---|\.\.\.)(?:[ \t]|$)', stripped): + unsupported = 'document markers' + elif not sequence and quoted_key and key_line[quoted_key.end():].lstrip(' ').startswith(':'): + unsupported = 'quoted keys' + elif not sequence and re.match(r'<<[ \t]*:', key_line): + unsupported = 'merge keys' + elif not sequence and not key and not step: + unsupported = (_ACTION_YAML_PROPERTIES.get(key_line[0]) + or 'block structure') + else: + unsupported = _action_yaml_value(node, whole_key=key is not None, + sequence=sequence) + if unsupported is None and key and key.group(1) == 'uses' and ( + not code or _ACTION_BLOCK_SCALAR.fullmatch(code) + or code.startswith(('{', '['))): + unsupported = 'uses keys' + if sequence and not sequence_depth: + sequence_start = number, line + if unsupported is not None: + # The finding refuses the entire unsupported collection; + # its inner mapping lines cannot become accepted block keys. + if sequence_depth or recovery.startswith(('{', '[')): + refused_flow_depth = max(0, sequence_depth + + sum(1 if char in '{[' else -1 + for char in visible if char in '{}[]')) + refused_flow_indent = _indent(sequence_start[1] if sequence_depth else line) + sequence.clear() + source_line = line.strip(' \t') + findings.append( + f'{name}:{number}: unsupported YAML construct in a workflow ({unsupported}): ' + f'rewrite it in plain block style; {source_line!r}') + continue + if not key or key.group(1) != 'uses': + continue + # Keep quotes separate from the trailing comment: a # inside a + # quoted scalar is part of the reference, not a version comment. + scalar = re.fullmatch( + r"""(?:"([^"]*)"|'([^']*)'|([^\s'"#]+))(?:[ \t]+(#[^\r\n]*))?[ \t]*""", + first) + reference = (next(group for group in scalar.groups()[:3] if group is not None) + if scalar else first) + comment = (scalar.group(4) or '') if scalar else '' + yield name, number, reference, comment + if sequence: + number, line = sequence_start + source_line = line.strip(' \t') + findings.append( + f'{name}:{number}: unsupported YAML construct in a workflow (flow sequences): ' + f'rewrite it in plain block style; {source_line!r}') + + +def action_pin_findings(sources: dict[str, str]) -> list[str]: + """Refuse YAML outside the supported subset and check every uses scalar.""" + findings = [] + for name, number, reference, comment in action_uses(sources, findings=findings): + if reference.startswith(ACTION_PIN_EXEMPT_PREFIXES): + continue + if ACTION_PIN_SHA.fullmatch(reference) and ACTION_PIN_COMMENT.fullmatch(comment): + continue + findings.append( + f'{name}:{number}: third-party uses {reference!r} must be pinned to a ' + '40-character lowercase hex SHA with a trailing # v comment') + return findings + + def _job_values(body: str, key: str, *, unquote: bool = True) -> list[str]: """Every value of a job-level key, a block value's lines folded onto one.""" lines = body.split('\n') @@ -1261,7 +1566,7 @@ def _step_block(step: str, key: str) -> list[str]: - name: Mint org App token id: app - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.PINEFORGE_APP_ID }} private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }} @@ -1270,7 +1575,7 @@ def _step_block(step: str, key: str) -> list[str]: - name: Checkout if: ${{ needs.authz.result == 'success' }} - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false @@ -1314,14 +1619,14 @@ def release_bump_workflow_findings(workflow: str) -> list[str]: app = _step_named(steps, ' - name: Mint org App token') checkout = _step_named(steps, ' - name: Checkout (full history)') if (_step_values(app, 'id') != ['app'] - or _step_values(app, 'uses') != ['actions/create-github-app-token@v1'] + or _step_values(app, 'uses') != ['actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547'] or _step_block(app, 'with') != [ ' with:', ' app-id: ${{ secrets.PINEFORGE_APP_ID }}', ' private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }}', ' owner: ${{ github.repository_owner }}', ' repositories: pineforge-engine'] - or _step_values(checkout, 'uses') != ['actions/checkout@v4'] + or _step_values(checkout, 'uses') != ['actions/checkout@11d5960a326750d5838078e36cf38b85af677262'] or _step_block(checkout, 'with') != [ ' with:', ' fetch-depth: 0', ' token: ${{ steps.app.outputs.token }}']): @@ -1532,7 +1837,7 @@ def native_workflow_findings(native: str, curl_action: str | None = None) -> lis upload = _step_named(steps, ' - name: Retain CI diagnostics') cache_key = 'ccache-${{ runner.os }}-${{ runner.arch }}-live-v2-${{ matrix.profile }}-' if (_step_values(restore, 'if') != ['${{ !inputs.cold_cache }}'] - or _step_values(restore, 'uses') != ['actions/cache/restore@v4'] + or _step_values(restore, 'uses') != ['actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830'] or f' key: {cache_key}${{{{ github.sha }}}}' not in _code(restore) or f' {cache_key}' not in _code(restore) or ' CCACHE_DIR: ${{ github.workspace }}/.ccache/${{ matrix.profile }}' not in _code(job) @@ -1555,7 +1860,7 @@ def native_workflow_findings(native: str, curl_action: str | None = None) -> lis findings.append('native-live must verify each full profile with no step deadline or timeout flag and ' 'native-only PR exclusion') if (_step_values(save, 'if') != [NATIVE_CACHE_SAVE] - or _step_values(save, 'uses') != ['actions/cache/save@v4'] + or _step_values(save, 'uses') != ['actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830'] or ' key: ${{ steps.ccache-restore.outputs.cache-primary-key }}' not in _code(save) or ' path: ${{ env.CCACHE_DIR }}' not in _code(save) or (save and upload and job.index(save) < job.index(upload))): @@ -1574,7 +1879,7 @@ def native_workflow_findings(native: str, curl_action: str | None = None) -> lis built = _step_named(action_steps, ' id: curl-build') saved = _step_named(action_steps, ' - name: Save pinned WebSocket-enabled libcurl') if (_step_values(restored, 'if') != ["${{ inputs.cold-cache != 'true' }}"] - or _step_values(restored, 'uses') != ['actions/cache/restore@v4'] + or _step_values(restored, 'uses') != ['actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830'] or 'key: curl-${{ inputs.cache-namespace }}-' not in restored or 'restore-keys:' in restored or any(token not in restored for token in ( @@ -1583,11 +1888,11 @@ def native_workflow_findings(native: str, curl_action: str | None = None) -> lis or ' cold-cache:\n' not in curl_action or " default: 'false'" not in curl_action): findings.append('pinned curl restore must include dependency identity and bypass every cold-cache restore') if (_step_values(saved, 'if') != [CURL_CACHE_SAVE] - or _step_values(saved, 'uses') != ['actions/cache/save@v4'] + or _step_values(saved, 'uses') != ['actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830'] or _step_values(built, 'run') != ['bash scripts/build_live_curl.sh build-native-deps 4'] or ' key: ${{ steps.curl-cache.outputs.cache-primary-key }}' not in _code(saved) or not built or not saved or curl_action.index(saved) < curl_action.index(built) - or 'uses: actions/cache@v4' in curl_action): + or 'uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830' in curl_action): findings.append('pinned curl must save immediately after its successful build, independently of later tests') return findings @@ -1639,7 +1944,7 @@ def pipeline_workflow_findings(workflow: str, name: str, *, pr: bool) -> list[st findings.append('preflight must invoke only its fast stage plan') linters = _step_named(steps, ' - name: Install workflow linters') if (len(steps) != 5 or [_step_values(step, 'uses') for step in steps] != [ - [], ['actions/checkout@v4'], [], [], ['actions/upload-artifact@v4']] + [], ['actions/checkout@11d5960a326750d5838078e36cf38b85af677262'], [], [], ['actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02']] or _step_values(linters, 'run') != [PREFLIGHT_LINTER_RUN] or any(_step_values(step, 'run') for step in steps if _step_values(step, 'uses'))): findings.append('preflight must keep only checkout, linters, fast checks and diagnostics') @@ -1694,13 +1999,13 @@ def pipeline_workflow_findings(workflow: str, name: str, *, pr: bool) -> list[st findings.append(f'{job} must run the fast Release smoke only on PRs and full profiles otherwise') cache = _step_named(_steps(build), ' - name: Restore ccache') save = _step_named(_steps(build), ' - name: Save main ccache') - if (_step_values(cache, 'uses') != ['actions/cache/restore@v4'] + if (_step_values(cache, 'uses') != ['actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830'] or _step_values(cache, 'id') != ['ccache-restore'] or _step_block(cache, 'with') != [ ' with:', ' path: ${{ env.CCACHE_DIR }}', f' key: {BUILD_CACHE_KEY}${{{{ github.sha }}}}', ' restore-keys: |', f' {BUILD_CACHE_KEY}'] - or _step_values(save, 'uses') != ['actions/cache/save@v4'] + or _step_values(save, 'uses') != ['actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830'] or _step_values(save, 'if') != [BUILD_CACHE_SAVE] or _step_block(save, 'with') != [ ' with:', ' path: ${{ env.CCACHE_DIR }}', @@ -1736,16 +2041,60 @@ def pipeline_workflow_findings(workflow: str, name: str, *, pr: bool) -> list[st return findings +def read_ci_text(path: Path, findings: list[str], root: Path | None = None) -> str: + """A tracked CI file's text, every line break byte exactly as stored. + + Path.read_text() decodes in universal-newline mode: a bare CR reached every + checker as LF, so the action-pin lint's refusal of it could never fire on a + real file. This reads the bytes and decodes them as strict UTF-8, never the + locale's codec. A file that is not UTF-8 is a finding (never a crash) and its + text, bad bytes replaced, still goes on so the other findings print too. The + finding names the file relative to root (the repository when None). + """ + data = path.read_bytes() + try: + return data.decode('utf-8') + except UnicodeDecodeError as error: + name = path.relative_to(ROOT if root is None else root).as_posix() + line = data.count(b'\n', 0, error.start) + 1 + findings.append(f'{name}:{line}: not valid UTF-8 ({error.reason} at byte offset ' + f'{error.start}): save the file as UTF-8') + return data.decode('utf-8', 'replace') + + +def _universal_newlines(text: str) -> str: + """The text as Path.read_text() always gave it: CRLF and a bare CR are LF. + + Every check except the action-pin lint reads this view, so a CRLF file is + judged exactly as it was before the loaders kept the bytes. + """ + return text.replace('\r\n', '\n').replace('\r', '\n') + + def ci_workflow_findings(ci: str, native: str, promote: str, cmake: str, parity: str, docs: str, post: str, subset: str, docs_post: str, others: dict[str, str] | None = None, - *, curl_action: str | None = None) -> list[str]: - """Check actual PR/post files together, including every reusable call edge.""" + *, curl_action: str | None = None, + actions: dict[str, str] | None = None) -> list[str]: + """Check actual PR/post files together, including every reusable call edge. + + The texts are the files as stored (read_ci_text): the action-pin lint judges + them so, and every other check gets the universal-newline view it always had. + """ workflows = {'ci.yml': ci, 'ci-post.yml': post, 'native-live.yml': native, 'corpus-parity.yml': parity, 'corpus-parity-subset.yml': subset, 'docs.yml': docs, 'docs-post.yml': docs_post, 'promote-baseline.yml': promote, **(others or {})} + action_sources = {f'.github/workflows/{name}': text for name, text in workflows.items()} + action_sources.update(actions or {}) + if curl_action is not None: + action_sources.setdefault('.github/actions/setup-live-curl/action.yml', curl_action) + workflows = {name: _universal_newlines(text) for name, text in workflows.items()} + ci, native, promote, cmake, parity, docs, post, subset, docs_post = map( + _universal_newlines, (ci, native, promote, cmake, parity, docs, post, subset, docs_post)) + curl_action = None if curl_action is None else _universal_newlines(curl_action) findings = runner_findings(workflows) + findings += action_pin_findings(action_sources) findings += pipeline_workflow_findings(ci, 'ci.yml', pr=True) findings += pipeline_workflow_findings(post, 'ci-post.yml', pr=False) findings += native_workflow_findings(native, curl_action) @@ -1946,7 +2295,7 @@ def docs_workflow_findings(workflow: str, post: str | None = None) -> list[str]: findings.append('docs-post.yml must retain only main pushes and manual dispatch') deploy = _step_named(_steps(_jobs(post).get('build', '')), ' - name: Deploy to Cloudflare Pages') - if (_step_values(deploy, 'uses') != ['cloudflare/wrangler-action@v3'] + if (_step_values(deploy, 'uses') != ['cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd'] or _step_values(deploy, 'if') or 'apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}' not in deploy or 'accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}' not in deploy @@ -1961,8 +2310,8 @@ def docs_workflow_self_test(workflow: str, post: str) -> int: return 1 no_pr = workflow.replace(DOCS_PR_TRIGGER, '', 1) deploy_on_pr = workflow.replace(' - name: Upload artifact\n', - ' - uses: cloudflare/wrangler-action@v3\n - name: Upload artifact\n', 1) - no_deploy = post.replace('uses: cloudflare/wrangler-action@v3', 'run: "true"', 1) + ' - uses: cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd # v3.15.0\n - name: Upload artifact\n', 1) + no_deploy = post.replace('uses: cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd # v3.15.0', 'run: "true"', 1) if no_pr == workflow or deploy_on_pr == workflow or no_deploy == post: print('docs workflow contract: could not form all three mutations') return 1 @@ -1985,23 +2334,24 @@ def actionlint_runner_label_findings(source: Path) -> list[str]: parses this same required file with its full YAML/schema parser. """ config_name = '.github/actionlint.yaml' + findings = [] try: - config = (source / config_name).read_text() - except (OSError, UnicodeError) as error: + config = _universal_newlines(read_ci_text(source / config_name, findings, source)) + except OSError as error: return [f'{config_name}: cannot read config: {error}'] lines = [re.sub(r'\s+#.*$', '', line).rstrip() for line in _code(config)] entries = [re.fullmatch(r' - (' + _LITERAL_RUNNER_LABEL + r')', line) for line in lines[2:]] if (lines[:2] != ['self-hosted-runner:', ' labels:'] or not entries or not all(entries)): - return [f'{config_name}: cannot parse config: expected self-hosted-runner.labels ' + return [*findings, + f'{config_name}: cannot parse config: expected self-hosted-runner.labels ' 'as a block list of literal labels'] declared = {entry.group(1).strip('\'"') for entry in entries} - findings = [] for path in sorted((source / '.github/workflows').glob('*')): if path.suffix not in ('.yml', '.yaml'): continue - workflow = path.read_text() + workflow = _universal_newlines(read_ci_text(path, findings, source)) if _unparsed(workflow): findings.append(f'{path.name}: cannot parse jobs for literal runner labels') continue @@ -2273,19 +2623,28 @@ def main() -> int: workflows = ROOT / '.github/workflows' named = ('ci.yml', 'native-live.yml', 'promote-baseline.yml', 'corpus-parity.yml', 'docs.yml', 'ci-post.yml', 'corpus-parity-subset.yml', 'docs-post.yml') - findings = ci_workflow_findings( - (workflows / 'ci.yml').read_text(), - (workflows / 'native-live.yml').read_text(), - (workflows / 'promote-baseline.yml').read_text(), - (ROOT / 'tests/CMakeLists.txt').read_text(), - (workflows / 'corpus-parity.yml').read_text(), - (workflows / 'docs.yml').read_text(), - (workflows / 'ci-post.yml').read_text(), - (workflows / 'corpus-parity-subset.yml').read_text(), - (workflows / 'docs-post.yml').read_text(), - {path.name: path.read_text() for path in sorted(workflows.iterdir()) + # Every file is read as stored: the lint must see a bare CR on disk. + # A file that is not UTF-8 is a finding here, and the curl action is + # read twice (as an action and as the native job's input): list it once. + findings = [] + actions = {path.relative_to(ROOT).as_posix(): read_ci_text(path, findings) + for path in sorted((ROOT / '.github/actions').rglob('*')) + if path.name in ('action.yml', 'action.yaml') and path.is_file()} + checked = ci_workflow_findings( + read_ci_text(workflows / 'ci.yml', findings), + read_ci_text(workflows / 'native-live.yml', findings), + read_ci_text(workflows / 'promote-baseline.yml', findings), + read_ci_text(ROOT / 'tests/CMakeLists.txt', findings), + read_ci_text(workflows / 'corpus-parity.yml', findings), + read_ci_text(workflows / 'docs.yml', findings), + read_ci_text(workflows / 'ci-post.yml', findings), + read_ci_text(workflows / 'corpus-parity-subset.yml', findings), + read_ci_text(workflows / 'docs-post.yml', findings), + {path.name: read_ci_text(path, findings) for path in sorted(workflows.iterdir()) if path.suffix in ('.yml', '.yaml') and path.name not in named}, - curl_action=(ROOT / '.github/actions/setup-live-curl/action.yml').read_text()) + curl_action=read_ci_text(ROOT / '.github/actions/setup-live-curl/action.yml', findings), + actions=actions) + findings = list(dict.fromkeys(findings + checked)) for finding in findings: print(finding) if not findings: @@ -2293,11 +2652,15 @@ def main() -> int: 'fast/slow rows, runners and time limits OK') return 1 if findings else 0 if args.check_docs_workflow or args.self_test_docs_workflow: - workflow = (ROOT / '.github/workflows/docs.yml').read_text() - post = (ROOT / '.github/workflows/docs-post.yml').read_text() + findings = [] + workflow = _universal_newlines(read_ci_text(ROOT / '.github/workflows/docs.yml', findings)) + post = _universal_newlines(read_ci_text(ROOT / '.github/workflows/docs-post.yml', findings)) if args.self_test_docs_workflow: - return docs_workflow_self_test(workflow, post) - findings = docs_workflow_findings(workflow, post) + code = docs_workflow_self_test(workflow, post) + for finding in findings: + print(finding) + return 1 if findings else code + findings += docs_workflow_findings(workflow, post) for finding in findings: print(finding) if not findings: diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index 1fcb30bd1..03a4c2927 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -2,6 +2,7 @@ """Exercise preflight failure propagation with real child processes.""" import contextlib import io +import inspect import json import os from pathlib import Path @@ -155,7 +156,7 @@ - name: Mint org App token id: app - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.PINEFORGE_APP_ID }} private-key: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }} @@ -164,7 +165,7 @@ - name: Checkout if: ${{ needs.authz.result == 'success' }} - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false @@ -224,6 +225,100 @@ def routed_line(before, value): return before.split(': ', 1)[0] + ': ' + value + '\n' +class WorkflowActionPins(unittest.TestCase): + """Check the actual workflow and composite-action references.""" + + def action_sources(self): + paths = sorted(path for path in (ROOT / '.github/workflows').iterdir() + if path.suffix in ('.yml', '.yaml')) + for name in ('action.yml', 'action.yaml'): + paths.extend(sorted((ROOT / '.github/actions').rglob(name))) + # The bytes as stored, decoded as preflight decodes them: read_text() + # would turn a bare CR into LF before the lint ever saw it. + return {str(path.relative_to(ROOT)): path.read_bytes().decode('utf-8') + for path in paths} + + def third_party_uses(self, sources=None): + if sources is None: + sources = self.action_sources() + findings = [] + references = [(Path(name), number, reference, comment) + for name, number, reference, comment in + ci_preflight.action_uses(sources, findings=findings) + if not reference.startswith(ci_preflight.ACTION_PIN_EXEMPT_PREFIXES)] + self.assertEqual(findings, [], '\n'.join(findings)) + self.assertTrue(references, 'no third-party action references were checked') + return references + + def assert_full_commit_sha(self, reference): + self.assertRegex(reference, ci_preflight.ACTION_PIN_SHA) + + def assert_version_comment(self, comment): + self.assertRegex(comment, ci_preflight.ACTION_PIN_COMMENT) + + def test_third_party_actions_use_full_commit_shas(self): + for path, number, reference, _ in self.third_party_uses(): + with self.subTest(path=str(path), line=number): + self.assert_full_commit_sha(reference) + + def test_third_party_actions_have_version_comments(self): + for path, number, _, comment in self.third_party_uses(): + with self.subTest(path=str(path), line=number): + self.assert_version_comment(comment) + + def test_real_tree_helper_accepts_lint_scalar_grammar(self): + reference = 'actions/checkout@11d5960a326750d5838078e36cf38b85af677262' + for value in (f'"{reference}" # v4.4.0', f"'{reference}' # v4.4.0", + 'docker://alpine:3.20', f'{reference} # v4.4.0-rc.1'): + with self.subTest(value=value): + # Retain a third-party row when the second row is exempt. + sources = {'.github/workflows/grammar.yml': + f'- uses: {reference} # v4.4.0\n- uses: {value}\n'} + self.assertEqual(ci_preflight.action_pin_findings(sources), []) + rows = self.third_party_uses(sources) + expected = [reference] if value.startswith('docker://') else [reference, reference] + self.assertEqual([row[2] for row in rows], expected) + for _, _, parsed, comment in rows: + self.assert_full_commit_sha(parsed) + self.assert_version_comment(comment) + # The shared real-tree assertions must still reject these mutations. + path = '.github/workflows/grammar.yml' + for _, _, reference, _ in self.third_party_uses( + {path: '- uses: actions/checkout@main # v4.4.0\n'}): + with self.assertRaises(AssertionError): + self.assert_full_commit_sha(reference) + for _, _, _, comment in self.third_party_uses( + {path: '- uses: actions/checkout@0123456789abcdef0123456789abcdef01234567\n'}): + with self.assertRaises(AssertionError): + self.assert_version_comment(comment) + + def test_real_tree_helper_refuses_unsupported_uses_shapes(self): + path = '.github/workflows/grammar.yml' + text = ('- uses: actions/checkout@0123456789abcdef0123456789abcdef01234567 # v4.4.0\n' + '- {uses: actions/checkout@main}\n') + expected = (f'{path}:2: unsupported YAML construct in a workflow (flow mappings): ' + "rewrite it in plain block style; '- {uses: actions/checkout@main}'") + with self.assertRaises(AssertionError) as failure: + self.third_party_uses({path: text}) + self.assertIn(expected, str(failure.exception)) + + def test_every_real_action_source_has_zero_findings(self): + sources = self.action_sources() + self.assertTrue(any(name.startswith('.github/workflows/') for name in sources)) + self.assertTrue(any(name.startswith('.github/actions/') for name in sources)) + self.assertEqual(ci_preflight.action_pin_findings(sources), []) + # Keep the complete population and a valid pin while introducing a + # hidden key: zero findings must mean the grammar was checked too. + path = '.github/workflows/grammar-control.yml' + line = r'- "\u0075ses": actions/checkout@main' + changed = {**sources, path: ( + '- uses: actions/checkout@0123456789abcdef0123456789abcdef01234567 # v4.4.0\n' + + line + '\n')} + self.assertEqual(ci_preflight.action_pin_findings(changed), [ + f'{path}:2: unsupported YAML construct in a workflow (quoted keys): ' + f'rewrite it in plain block style; {line!r}']) + + class WorkflowEventFleets(unittest.TestCase): """Fleet expectations come from events and call edges, never file names.""" @@ -490,6 +585,874 @@ def test_post_hosted_exception_never_applies_to_pull_request(self): {'pr.yml': caller, 'ci-post.yml': callee})) +class ActionPins(unittest.TestCase): + SHA = '0123456789abcdef0123456789abcdef01234567' + WORKFLOW = '.github/workflows/pinlint.yml' + ACTION = '.github/actions/setup-live-curl/action.yml' + + @staticmethod + def workflow(value, *, named=False): + steps = (' - name: External action\n' + f' uses: {value}\n' if named else f' - uses: {value}\n') + return ('on: workflow_dispatch\njobs:\n inspect:\n' + ' runs-on: ["self-hosted","pf-gcp-post"]\n' + ' steps:\n' + steps) + + @staticmethod + def finding(path, line, reference): + return (f'{path}:{line}: third-party uses {reference!r} must be pinned to a ' + '40-character lowercase hex SHA with a trailing # v comment') + + def pin_findings(self, text, *, composite=False, checker=None): + # Use the existing contract entrypoint, so main fails for a missing + # finding, not an import error for a newly introduced helper. + sources = [(ROOT / path).read_text() for path in CI_SOURCES] + checker = ci_workflow_findings if checker is None else checker + if composite: + findings = checker(*sources, curl_action=text) + path = self.ACTION + else: + findings = checker(*sources, others={Path(self.WORKFLOW).name: text}) + path = self.WORKFLOW + return [finding for finding in findings if finding.startswith(path + ':')] + + def assert_refused(self, value, reference, *, named=False, checker=None): + self.assertEqual( + self.pin_findings(self.workflow(value, named=named), checker=checker), + [self.finding(self.WORKFLOW, 7 if named else 6, reference)]) + + def test_mutable_tag_is_refused(self): + self.assert_refused('actions/checkout@v4 # v4.2.2', 'actions/checkout@v4') + + def test_branch_ref_is_refused(self): + self.assert_refused('actions/checkout@main # v4.2.2', 'actions/checkout@main', + named=True) + + def test_short_sha_is_refused(self): + self.assert_refused('actions/checkout@0123456 # v4.2.2', 'actions/checkout@0123456') + + def test_sha_without_same_line_version_is_refused(self): + reference = 'actions/checkout@' + self.SHA + for value in (reference, reference + '\n # v4.2.2'): + with self.subTest(value=value): + self.assert_refused(value, reference) + + def test_malformed_version_comments_are_refused(self): + reference = 'actions/checkout@' + self.SHA + for comment in ('# 4.2.2', '# vfour', '# v4..2', '# v4.2.', + '# v4.2.2 extra', '# v4.2.2-', '# v4.2.2+'): + with self.subTest(comment=comment): + self.assert_refused(reference + ' ' + comment, reference) + + def test_uppercase_sha_is_refused(self): + reference = 'actions/checkout@' + self.SHA.upper() + self.assert_refused(reference + ' # v4.2.2', reference) + + def test_quoted_mutable_values_are_refused(self): + for quote in ('"', "'"): + for named in (False, True): + with self.subTest(quote=quote, named=named): + self.assert_refused(f'{quote}actions/checkout@v4{quote} # v4.2.2', + 'actions/checkout@v4', named=named) + + def test_reference_without_ref_is_refused(self): + self.assert_refused('actions/checkout # v4.2.2', 'actions/checkout') + + def test_remote_reusable_workflow_is_refused(self): + reference = 'example/shared/.github/workflows/build.yml@main' + text = ('on: workflow_dispatch\njobs:\n call:\n' + f' uses: {reference} # v1.2.3\n') + self.assertEqual(self.pin_findings(text), + [self.finding(self.WORKFLOW, 4, reference)]) + + def test_composite_step_is_refused(self): + reference = 'actions/cache/restore@v4' + text = ('name: Example\nruns:\n using: composite\n steps:\n' + f' - uses: {reference} # v4.2.2\n') + self.assertEqual(self.pin_findings(text, composite=True), + [self.finding(self.ACTION, 5, reference)]) + + @staticmethod + def unsupported_finding(path, line, text, construct='flow mappings'): + return (f'{path}:{line}: unsupported YAML construct in a workflow ({construct}): ' + f'rewrite it in plain block style; {text!r}') + + def assert_unsupported_step(self, step, *, composite=False, checker=None, + construct='flow mappings'): + if composite: + original = (ROOT / self.ACTION).read_text() + self.assertTrue(original.endswith('\n')) + number = len(original.splitlines()) + 1 + text = original + ' ' + step + '\n' + path = self.ACTION + else: + original = self.workflow('actions/checkout@main') + text = original.replace('- uses: actions/checkout@main', step, 1) + self.assertNotEqual(text, original) + number, path = 6, self.WORKFLOW + self.assertEqual(self.pin_findings(text, composite=composite, checker=checker), + [self.unsupported_finding(path, number, step, construct)]) + + def assert_yaml_refusals(self, text, rows): + # The public checker already exists on f4a03dd712: a red result + # is a behavioral mismatch, never a missing-helper/import failure. + for path in (self.WORKFLOW, self.ACTION): + with self.subTest(path=path): + expected = [self.unsupported_finding(path, number, line, construct) + for number, construct, line in rows] + self.assertEqual(ci_preflight.action_pin_findings({path: text}), expected) + + def test_escaped_quoted_keys_are_refused(self): + for line in (r'- "\u0075ses": actions/checkout@main', + r'- "\x75ses": actions/checkout@main', + r'- "\U00000075ses": actions/checkout@main'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'quoted keys', line)]) + + def test_all_quoted_keys_are_refused(self): + for key in ('"uses"', "'uses'", '"steps"', "'permissions'"): + line = f'- {key}: actions/checkout@{self.SHA} # v4.4.0' + with self.subTest(key=key): + self.assert_yaml_refusals(line + '\n', [(1, 'quoted keys', line)]) + + def test_explicit_keys_are_refused(self): + text = '- ? uses\n : actions/checkout@main\n' + self.assert_yaml_refusals(text, [ + (1, 'explicit keys', '- ? uses'), + (2, 'explicit keys', ': actions/checkout@main')]) + + def test_anchors_are_refused_without_a_uses_key(self): + line = 'steps: &checkout' + self.assert_yaml_refusals(line + '\n', [(1, 'anchors', line)]) + + def test_aliases_are_refused_without_a_uses_key(self): + line = '- *checkout' + self.assert_yaml_refusals(line + '\n', [(1, 'aliases', line)]) + + def test_merge_keys_are_refused(self): + line = '<<: *checkout' + self.assert_yaml_refusals(line + '\n', [(1, 'merge keys', line)]) + + def test_tags_are_refused(self): + for tag in ('!!map', '!custom', '!'): + line = 'steps: ' + tag + with self.subTest(tag=tag): + self.assert_yaml_refusals(line + '\n', [(1, 'tags', line)]) + + def test_anchored_multiline_mapping_is_refused(self): + text = ('steps:\n' + ' - &checkout {\n' + f' uses: actions/checkout@{self.SHA} # v4.4.0\n' + ' }\n' + ' - *checkout\n') + self.assert_yaml_refusals(text, [ + (2, 'anchors', '- &checkout {'), + (5, 'aliases', '- *checkout')]) + + def test_anchored_run_scalar_is_refused_without_scanning_its_body(self): + text = ("steps:\n - run: &script |\n" + " printf '%s\\n' '\n" + " uses: actions/checkout@main\n" + " '\n") + self.assert_yaml_refusals(text, [(2, 'anchors', '- run: &script |')]) + + def test_flow_mappings_without_uses_are_refused(self): + for line in ('env: {CHECKOUT: main}', '- {name: Checkout}', '- {}', + 'env: {} trailing', '- { }', 'labels: [{}]', + 'labels: [name: Checkout]'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'flow mappings', line)]) + self.assert_yaml_refusals('env: {\n CHECKOUT: main\n}\n', + [(1, 'flow mappings', 'env: {')]) + + def test_reported_flow_continuation_resumes_before_mutable_action(self): + path = '.github/workflows/ci.yml' + text = ('env: {\n' + ' A: b", } # "\n' + 'steps:\n' + ' - uses: actions/checkout@main\n') + self.assertEqual(ci_preflight.action_pin_findings({path: text}), [ + self.unsupported_finding(path, 1, 'env: {'), + self.finding(path, 4, 'actions/checkout@main')]) + + def test_refused_flow_continuations_count_delimiters_despite_plain_quotes(self): + cases = ( + ('env: {', 'A: b", } # "'), + ('env: {', "A: b', } # '"), + ('env: {', 'A: [b", linux]} # "'), + ('env: {', "A: [b', {B: value}]} # '"), + ('options: [{', 'A: b", }] # "'), + ('options: [[{', "A: b', }]] # '"), + ) + for opening, continuation in cases: + text = ('steps:\n - ' + opening + '\n' + ' ' + continuation + '\n' + ' uses: actions/checkout@main\n') + for path in (self.WORKFLOW, self.ACTION): + with self.subTest(opening=opening, continuation=continuation, path=path): + # The uses key is deeper than the opening line: outdent + # recovery alone cannot make this assertion pass. + self.assertEqual(ci_preflight.action_pin_findings({path: text}), [ + self.unsupported_finding(path, 2, '- ' + opening), + self.finding(path, 4, 'actions/checkout@main')]) + + def test_refused_flow_continuations_end_at_outdent_even_with_unbalanced_depth(self): + for prefix, indent in (('', ''), ('options:\n', ' ')): + opening = indent + 'env: {' + # The quoted { adds depth in the conservative scan; the embedded + # quote after b masks the real closer on the old implementation. + continuation = indent + ' A: "{", B: b", } # "' + for following, uses_line in ( + (indent + 'uses: actions/checkout@main\n', 3), + (indent + 'uses: actions/checkout@main # }\n', 3), + ('steps:\n - uses: actions/checkout@main\n', 4)): + text = prefix + opening + '\n' + continuation + '\n' + following + offset = 1 if prefix else 0 + for path in (self.WORKFLOW, self.ACTION): + with self.subTest(prefix=prefix, following=following, path=path): + self.assertEqual(ci_preflight.action_pin_findings({path: text}), [ + self.unsupported_finding(path, 1 + offset, opening.strip(' ')), + self.finding(path, uses_line + offset, 'actions/checkout@main')]) + + def test_empty_sequences_require_a_whole_plain_key_value(self): + for line in ('- []', 'labels: [[]]', 'labels: [] trailing'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'empty collections', line)]) + + def test_properties_inside_flow_sequences_are_refused(self): + for value, construct in (('&runner linux', 'anchors'), ('*runner', 'aliases'), + ('!!str linux', 'tags'), ('? uses', 'explicit keys')): + line = 'runs-on: [' + value + ']' + with self.subTest(value=value): + self.assert_yaml_refusals(line + '\n', [(1, construct, line)]) + + def test_flow_sequence_quotes_cannot_mask_mappings_or_comments(self): + for quote in ('"', "'"): + line = (f'options: [a{quote}, {{uses: actions/checkout@main}}] ' + f'# {quote}]') + with self.subTest(quote=quote): + self.assert_yaml_refusals(line + '\n', [(1, 'flow mappings', line)]) + code, visible, _ = ci_preflight._action_yaml_visible( + line.split(': ', 1)[1]) + self.assertEqual(code, line.split(': ', 1)[1].split(' # ', 1)[0]) + self.assertIn('{uses: actions/checkout@main}', visible) + + def test_quotes_in_flow_sequences_must_start_an_element(self): + for value in ('[a"b"]', "[a'b']", '["a" "b"]', "['a' 'b']", + '[linux, [a"b"]]'): + line = 'options: ' + value + with self.subTest(value=value): + self.assert_yaml_refusals(line + '\n', [(1, 'quoted scalars', line)]) + for first in ('options: [', 'options: [linux,', 'options: [linux'): + line = 'a"b"]' + with self.subTest(first=first): + self.assert_yaml_refusals(first + '\n ' + line + '\n', + [(2, 'quoted scalars', line)]) + + def test_flow_quotes_at_element_start_preserve_data_and_scan_resumption(self): + text = ('runs-on: ["self-hosted", "pf-gcp-pr"]\n' + 'branches: [main]\n' + 'needs: [a, b]\n' + 'options: ["a # {uses: actions/checkout@main}",\n' + " ['it''s # data', linux]] # actual comment\n") + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), []) + text += '- uses: actions/upload-artifact@main\n' + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), [ + self.finding(self.WORKFLOW, 6, 'actions/upload-artifact@main')]) + + def test_unicode_whitespace_at_line_start_is_refused(self): + spaces = ('\u0085', '\u00a0', '\u1680', '\u2000', '\u2001', '\u2002', + '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', + '\u2009', '\u200a', '\u2028', '\u2029', '\u202f', '\u205f', + '\u3000', '\ufeff') + for space in spaces: + line = space + '#hidden: {uses: actions/checkout@main}' + construct = (f'unsupported line break U+{ord(space):04X}' + if space in '\u0085\u2028\u2029' else 'unsupported whitespace') + for prefix in ('', 'name: Example\n'): + with self.subTest(space=repr(space), prefix=prefix): + self.assert_yaml_refusals( + prefix + line + '\n', + [(1 if not prefix else 2, construct, line)]) + + def test_unicode_whitespace_around_separators_is_refused(self): + for space in ('\u0085', '\u00a0', '\u2028', '\u2029', '\ufeff'): + construct = (f'unsupported line break U+{ord(space):04X}' + if space in '\u0085\u2028\u2029' else 'unsupported whitespace') + for line in (f'name{space}: Example', f'name:{space}Example', + f'name: Example{space}#hidden: {{uses: actions/checkout@main}}', + f'name: Example #{space}hidden: {{uses: actions/checkout@main}}', + f'name: Example{space}: hidden', f'name: Example:{space}hidden'): + with self.subTest(space=repr(space), line=line): + self.assert_yaml_refusals(line + '\n', [(1, construct, line)]) + + def test_unsupported_line_breaks_in_block_scalars_refuse_the_whole_file(self): + for separator in ('\r', '\u0085', '\u2028', '\u2029', '\v', '\f'): + for prefix, body, following, number in ( + ('steps:\n - run: |\n', ' echo ok', + ' - uses: actions/checkout@main', 3), + ('steps:\n - run: |\n', ' ', + ' - uses: actions/checkout@main', 3), + ('name: Example\nruns:\n using: composite\n steps:\n' + ' - shell: bash\n run: |\n', ' echo ok', + ' - uses: actions/checkout@main', 7)): + line = body + separator + following + with self.subTest(separator=repr(separator), prefix=prefix, body=body): + self.assert_yaml_refusals(prefix + line + '\n', [ + (number, f'unsupported line break U+{ord(separator):04X}', + line.strip(' \t'))]) + + def test_unsupported_line_breaks_in_comments_refuse_the_whole_file(self): + for separator in ('\r', '\u0085', '\u2028', '\u2029', '\v', '\f'): + line = ' # comment' + separator + ' - uses: actions/checkout@main' + with self.subTest(separator=repr(separator)): + self.assert_yaml_refusals('steps:\n' + line + '\n', [ + (2, f'unsupported line break U+{ord(separator):04X}', + line.strip(' \t'))]) + + def test_c0_controls_in_block_scalars_and_comments_refuse_the_whole_file(self): + for codepoint in range(0x20): + if codepoint in (0x09, 0x0a, 0x0b, 0x0c, 0x0d): + continue # TAB/LF/CRLF controls and the six line breaks have separate rows. + control = chr(codepoint) + for prefix, line, number in ( + ('run: |\n', ' echo ' + control + 'ok', 2), + ('name: Example\n', '# comment ' + control + 'text', 2)): + with self.subTest(codepoint=codepoint, prefix=prefix): + self.assert_yaml_refusals(prefix + line + '\n', [ + (number, f'unsupported control character U+{codepoint:04X}', + line.strip(' \t'))]) + + def test_file_character_refusal_reports_only_the_first_offense_before_scanning(self): + first = ' echo \x01ok' + text = ('uses: actions/checkout@main\r\n' + 'run: |\r\n' + first + '\r\n' + '# later \u2028\n') + for path, other in ((self.WORKFLOW, self.ACTION), (self.ACTION, self.WORKFLOW)): + with self.subTest(path=path): + self.assertEqual(ci_preflight.action_pin_findings({ + path: text, other: 'uses: actions/upload-artifact@main\n'}), [ + self.unsupported_finding( + path, 3, first.strip(' '), 'unsupported control character U+0001'), + self.finding(other, 1, 'actions/upload-artifact@main')]) + + def test_crlf_block_scalar_resumption_and_non_ascii_text_remain_supported(self): + text = ('steps:\n' + ' - run: |\n' + ' echo café — →\tdata\n' + ' uses: actions/checkout@main\n' + ' # café — →\n' + f' - uses: actions/upload-artifact@{self.SHA} # v4.4.0\n') + for newline in ('\n', '\r\n'): + for path in (self.WORKFLOW, self.ACTION): + with self.subTest(newline=newline, path=path): + sources = {path: text.replace('\n', newline)} + self.assertEqual(ci_preflight.action_pin_findings(sources), []) + changed = sources[path].replace( + f'actions/upload-artifact@{self.SHA}', 'actions/upload-artifact@main', 1) + self.assertNotEqual(changed, sources[path]) + self.assertEqual(ci_preflight.action_pin_findings({path: changed}), [ + self.finding(path, 6, 'actions/upload-artifact@main')]) + + def test_unterminated_expressions_with_many_quotes_are_refused(self): + # Thread-free, with no timing threshold: the large input cannot finish + # in normal test flow with the old exponentially backtracking regex. + for count in (100, 5000, 200000): + line = 'name: ${{' + "'" * count + with self.subTest(count=count): + self.assert_yaml_refusals(line + '\n', [(1, 'expressions', line)]) + + def test_expression_strings_keep_delimiters_and_doubled_quotes_opaque(self): + text = ("name: ${{ format('it''s # }} uses: {0}', 'ignored') }}\n" + "options: [${{ format('}} # uses: {0}', 'ignored') }}, linux]\n") + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), []) + text += '- uses: actions/checkout@main\n' + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), [ + self.finding(self.WORKFLOW, 3, 'actions/checkout@main')]) + + def test_backslashes_in_yaml_quoted_values_are_refused(self): + for line in (r'name: "line\nbreak"', r"name: 'line\nbreak'", + r'runs-on: ["self\u002dhosted", linux]'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'quoted escapes', line)]) + + def test_quoted_values_must_end_on_their_line(self): + for line in ('name: "unfinished', "name: 'finished' trailing", 'labels: ["unfinished'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'quoted scalars', line)]) + + def test_flow_sequence_boundaries_are_refused(self): + for line in ('labels: [linux][windows]', 'labels: [linux] [windows]', + 'labels: [linux] trailing', 'labels: [[linux] windows]', + 'labels: [linux,'): + with self.subTest(line=line): + self.assert_yaml_refusals(line + '\n', [(1, 'flow sequences', line)]) + + def test_tab_indentation_is_refused(self): + for prefix in ('\t', ' \t', '\t '): + line = prefix + f'- uses: actions/checkout@{self.SHA} # v4.4.0' + with self.subTest(prefix=prefix): + self.assert_yaml_refusals(line + '\n', + [(1, 'tab indentation', line.strip())]) + line = f'-\tuses: actions/checkout@{self.SHA} # v4.4.0' + self.assert_yaml_refusals(line + '\n', [(1, 'tab indentation', line)]) + self.assert_yaml_refusals('\t\n', [(1, 'tab indentation', '')]) + + def test_workflow_document_markers_are_refused(self): + for marker in ('---', '...', '--- # another document', '... # end'): + text = 'name: Example\n' + marker + '\n' + with self.subTest(marker=marker): + self.assert_yaml_refusals(text, [(2, 'document markers', marker)]) + + def test_uses_requires_a_same_line_scalar(self): + for value in ('', '|', '>-', '{}', '[]', '[./local]'): + line = '- uses:' + (' ' + value if value else '') + with self.subTest(value=value): + self.assert_yaml_refusals(line + '\n', [(1, 'uses keys', line)]) + + def test_unrecognised_block_lines_are_refused(self): + line = '%YAML 1.2' + self.assert_yaml_refusals(line + '\n', [(1, 'block structure', line)]) + + def test_supported_yaml_values_and_collections_remain_accepted(self): + for value in ('{}', '{ }', '{ \t}', '[]', '[ ]', '[\t ]', + '[linux, [windows, macos]]'): + with self.subTest(value=value): + self.assertEqual(ci_preflight.action_pin_findings( + {self.WORKFLOW: 'options: ' + value + ' # supported value\n'}), []) + text = ('permissions: {} # no permissions\n' + 'options: [] # no options\n' + 'branches: [main]\n' + 'runs-on: ["self-hosted", "pf-gcp-pr"]\n' + "cron: '30 1 * * *'\n" + "paths-ignore: ['.github/**']\n" + 'shell: python3 {0}\n' + "run: printf '%s\\n' \"${NEXT}\" > VERSION\n" + 'group: prefix-${{ github.workflow }}-${{ github.run_id }}\n' + "if: ${{ !cancelled() && contains('uses: & ! {', 'x') }}\n" + 'labels: [linux,\n' + ' "self-hosted"]\n' + 'steps:\n' + f' - uses: "actions/checkout@{self.SHA}" # v4.4.0\n') + for newline in ('\n', '\r\n'): + with self.subTest(newline=newline): + sources = {self.WORKFLOW: text.replace('\n', newline)} + self.assertEqual(ci_preflight.action_pin_findings(sources), []) + changed = sources[self.WORKFLOW].replace( + f'actions/checkout@{self.SHA}', 'actions/checkout@main', 1) + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: changed}), [ + self.finding(self.WORKFLOW, 14, 'actions/checkout@main')]) + + def test_script_and_other_block_scalar_bodies_are_excluded(self): + for key in ('run', 'script', 'path', 'restore-keys'): + for indicator in ('|+', '|-', '>2-', '|2+', '>+2'): + with self.subTest(key=key, indicator=indicator): + text = (f'with:\n {key}: {indicator}\n' + ' - "\\u0075ses": actions/checkout@main\n' + ' ? uses\n' + ' <<: *checkout\n' + ' steps: &checkout {uses: actions/checkout@main}\n' + ' ---\n' + ' \t!!str uses: actions/checkout@main\n' + ' next: value\n') + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), []) + text += '- uses: actions/upload-artifact@main\n' + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), [ + self.finding(self.WORKFLOW, 10, 'actions/upload-artifact@main')]) + + def test_sequence_item_block_scalar_bodies_are_excluded(self): + for indicator in ('|', '>-', '|2+', '|+2', '>2-'): + text = (f'args:\n - {indicator}\n' + ' uses: actions/checkout@main\n') + with self.subTest(indicator=indicator): + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), []) + text += ' - uses: actions/upload-artifact@main\n' + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), [ + self.finding(self.WORKFLOW, 4, 'actions/upload-artifact@main')]) + + def test_sequence_mapping_block_scalar_resumes_at_key_indent(self): + for key in ('run', 'script'): + for indicator in ('|', '>-', '|2+'): + text = (f'args:\n - {key}: {indicator}\n' + ' uses: actions/checkout@main\n' + ' uses: actions/upload-artifact@main\n') + with self.subTest(key=key, indicator=indicator): + self.assertEqual(ci_preflight.action_pin_findings({self.WORKFLOW: text}), [ + self.finding(self.WORKFLOW, 4, 'actions/upload-artifact@main')]) + + def test_flow_mapping_mutable_steps_are_refused(self): + steps = ( + '- {uses: actions/checkout@main}', + '- { uses : actions/checkout@main }', + '- {"uses": "actions/checkout@main"}', + "- { 'uses' : 'actions/checkout@main' }", + '- {"uses":"actions/checkout@main"}', + '- {name: External action, uses: actions/checkout@main}', + '- {name: External action, "uses" : actions/checkout@main, with: {fetch-depth: 0}}', + "- {with: {fetch-depth: 0}, 'uses': actions/checkout@main}", + '- name: External action, uses: actions/checkout@main', + "- name: External action, 'uses' : actions/checkout@main", + '- uses: ./.github/actions/local, uses: actions/checkout@main', + '- uses: docker://alpine:3.20, "uses": actions/checkout@main', + '- {&key "uses": actions/checkout@main}', + '- {!!str uses: actions/checkout@main}', + ) + for composite in (False, True): + for step in steps: + with self.subTest(composite=composite, step=step): + construct = ('flow mappings' if step.startswith('- {') + else 'inline mapping keys') + self.assert_unsupported_step(step, composite=composite, construct=construct) + + def test_pinned_flow_mapping_is_refused_as_unsupported(self): + step = f'- {{uses: actions/checkout@{self.SHA}}} # v4.4.0' + for composite in (False, True): + with self.subTest(composite=composite): + self.assert_unsupported_step(step, composite=composite) + + def test_multiline_flow_mapping_is_refused_as_unsupported(self): + line = f'uses: actions/checkout@{self.SHA} # v4.4.0' + original = self.workflow('actions/checkout@main') + text = original.replace('- uses: actions/checkout@main', + '- {\n ' + line + '\n }', 1) + self.assertNotEqual(text, original) + self.assertEqual(self.pin_findings(text), + [self.unsupported_finding(self.WORKFLOW, 6, '- {')]) + + def test_flow_mapping_reusable_workflow_is_refused(self): + line = 'call: {uses: example/shared/.github/workflows/build.yml@main}' + text = 'on: workflow_dispatch\njobs:\n ' + line + '\n' + self.assertEqual(self.pin_findings(text), + [self.unsupported_finding(self.WORKFLOW, 3, line)]) + + def test_run_script_uses_text_is_not_a_yaml_key(self): + for indicator in ('|', '|-', '|+', '|2', '>', '>-', '>+', '>2-'): + with self.subTest(indicator=indicator): + script = (f' - run: {indicator}\n' + " printf '%s\\n' '\n" + ' uses: actions/checkout@main\n' + " '\n" + ' # {uses: actions/checkout@main}\n' + ' echo "uses: actions/checkout@main"\n') + text = self.workflow('actions/checkout@main').replace( + ' - uses: actions/checkout@main\n', script, 1) + self.assertEqual(self.pin_findings(text), []) + number = len(text.splitlines()) + 1 + text += ' - uses: actions/upload-artifact@main\n' + self.assertEqual(self.pin_findings(text), [ + self.finding(self.WORKFLOW, number, 'actions/upload-artifact@main')]) + # Quoted YAML values and comments must not expose apparent flow keys. + text = ('on: workflow_dispatch\njobs:\n inspect:\n' + ' runs-on: ["self-hosted","pf-gcp-post"]\n steps:\n' + ' - name: "Example, uses: actions/checkout@main"\n' + ' run: \'echo "{uses: actions/checkout@main}"\'\n' + ' - name: A plain scalar mentioning uses:literal\n' + ' run: "true"\n' + ' # - {"uses": actions/checkout@main}\n') + self.assertEqual(self.pin_findings(text), []) + + def test_removing_unsupported_shape_branch_is_detected(self): + step = '- {uses: actions/checkout@main}' + sources = {self.WORKFLOW: self.workflow('actions/checkout@main').replace( + '- uses: actions/checkout@main', step, 1)} + expected = [self.unsupported_finding(self.WORKFLOW, 6, step)] + self.assertEqual(ci_preflight.action_pin_findings(sources), expected) + source = inspect.getsource(ci_preflight.action_uses) + start = source.index(' if unsupported is not None:\n') + end = source.index(" if not key or key.group(1) != 'uses':\n", start) + guard = source[start:end] + self.assertEqual(source.count(guard), 1) + mutant = source.replace(guard, '', 1) + namespace = dict(ci_preflight.action_pin_findings.__globals__) + exec(compile(mutant, '', 'exec'), namespace) + exec(compile(inspect.getsource(ci_preflight.action_pin_findings), + '', 'exec'), namespace) + # Exercise real extraction and validation with the refusal branch removed. + with self.assertRaises(AssertionError): + self.assertEqual(namespace['action_pin_findings'](sources), expected) + + def test_every_offending_line_is_reported(self): + text = self.workflow('actions/checkout@main # v4.2.2') + text += ' - uses: actions/upload-artifact@v4 # v4.6.2\n' + self.assertEqual(self.pin_findings(text), [ + self.finding(self.WORKFLOW, 6, 'actions/checkout@main'), + self.finding(self.WORKFLOW, 7, 'actions/upload-artifact@v4'), + ]) + + def test_pinned_quoted_and_local_shapes_are_accepted(self): + for named in (False, True): + for quote in ('', '"', "'"): + for version in ('v4', 'v4.2.2', 'v1.12.0', 'v4.2.2-rc.1', + 'v4.2.2+build.7', 'v4.2.2-rc.1+build.7'): + value = f'{quote}actions/checkout@{self.SHA}{quote} # {version}' + with self.subTest(value=value, named=named): + self.assertEqual(self.pin_findings(self.workflow(value, named=named)), []) + for reference in ('./.github/actions/x', './.github/workflows/y.yml', + 'docker://alpine:3.20'): + with self.subTest(reference=reference, quote=quote, named=named): + value = quote + reference + quote + text = self.workflow(value, named=named) + text += ' # - uses: actions/checkout@main\n' + self.assertEqual(self.pin_findings(text), []) + + @staticmethod + def run_contract(tree): + output = io.StringIO() + with mock.patch.object(ci_preflight, 'ROOT', tree), \ + mock.patch.object(sys, 'argv', ['ci_preflight.py', '--check-ci-workflow']), \ + contextlib.redirect_stdout(output): + code = ci_preflight.main() + return code, output.getvalue() + + def test_nested_action_yml_and_yaml_are_discovered(self): + with tempfile.TemporaryDirectory() as directory: + tree = Path(directory) + # Copy data inputs only; the real, already imported checker runs. + shutil.copytree(ROOT / '.github/workflows', tree / '.github/workflows') + shutil.copytree(ROOT / '.github/actions', tree / '.github/actions') + (tree / 'tests').mkdir() + shutil.copyfile(ROOT / 'tests/CMakeLists.txt', tree / 'tests/CMakeLists.txt') + paths = ('.github/actions/nested/alpha/action.yml', + '.github/actions/nested/beta/action.yaml') + reference = 'actions/cache/save@main' + for name in paths: + path = tree / name + path.parent.mkdir(parents=True) + path.write_text('name: Example\nruns:\n using: composite\n steps:\n' + f' - uses: {reference} # v4.2.2\n') + code, output = self.run_contract(tree) + self.assertEqual( + [line for line in output.splitlines() + if any(line.startswith(name + ':') for name in paths)], + [self.finding(name, 5, reference) for name in paths]) + self.assertEqual(code, 1, output) + + def test_removing_pin_check_is_detected(self): + value, reference = 'actions/checkout@main # v4.2.2', 'actions/checkout@main' + self.assert_refused(value, reference) + source = inspect.getsource(ci_workflow_findings) + guard = ' findings += action_pin_findings(action_sources)\n' + self.assertEqual(source.count(guard), 1) + mutant = source.replace(guard, '', 1) + namespace = dict(ci_workflow_findings.__globals__) + exec(compile(mutant, '', 'exec'), namespace) + # Delete the actual integration call, then apply the same regression + # assertion to that function. No checker result is stubbed. + with self.assertRaises(AssertionError): + self.assert_refused(value, reference, checker=namespace['ci_workflow_findings']) + + # Expected to fail with the new check on base c82fd00e until the separate + # action-pins lane lands. Keep this a normal test, never an xfail or skip. + def test_real_tree_has_only_versioned_action_pins(self): + self.assert_refused('actions/checkout@main # v4.2.2', 'actions/checkout@main') + for tier in ('all', 'fast', 'github-only'): + plan = {entry[0]: entry for entry in check_commands(ROOT, tier=tier)} + self.assertEqual(plan['ci-workflow-contract'], ( + 'ci-workflow-contract', + [sys.executable, str(ROOT / 'scripts/ci_preflight.py'), '--check-ci-workflow'])) + code, output = self.run_contract(ROOT) + self.assertEqual(code, 0, output) + + +class CiFileLoading(unittest.TestCase): + """The stages read tracked CI files as stored: bytes, strict UTF-8, no newline translation. + + Every test writes real files into a temporary tree laid out like the + repository and runs main(), the entry point CI runs, over it. A checker + handed a string never passes through the loaders, and the loaders are where + a bare CR used to vanish: Path.read_text() decodes in universal-newline mode. + """ + DOCS = '.github/workflows/docs.yml' + DOCS_POST = '.github/workflows/docs-post.yml' + CONFIG = '.github/actionlint.yaml' + CMAKE = 'tests/CMakeLists.txt' + + @staticmethod + def action_sources(): + """Every file the action-pin lint reads, relative to the repository root.""" + paths = sorted(path for path in (ROOT / '.github/workflows').iterdir() + if path.suffix in ('.yml', '.yaml')) + for name in ('action.yml', 'action.yaml'): + paths.extend(sorted((ROOT / '.github/actions').rglob(name))) + return [path.relative_to(ROOT).as_posix() for path in paths] + + def workflow_sources(self): + return [name for name in self.action_sources() if name.startswith('.github/workflows/')] + + def make_tree(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + tree = Path(directory.name) + shutil.copytree(ROOT / '.github/workflows', tree / '.github/workflows') + shutil.copytree(ROOT / '.github/actions', tree / '.github/actions') + shutil.copyfile(ROOT / self.CONFIG, tree / self.CONFIG) + (tree / 'tests').mkdir() + shutil.copyfile(ROOT / self.CMAKE, tree / self.CMAKE) + return tree + + def crlf_tree(self, names): + tree = self.make_tree() + for name in names: + original = (ROOT / name).read_bytes() + self.assertNotIn(b'\r', original, name) + (tree / name).write_bytes(original.replace(b'\n', b'\r\n')) + return tree + + @staticmethod + def run_stage(tree, flag): + output = io.StringIO() + with mock.patch.object(ci_preflight, 'ROOT', tree), \ + mock.patch.object(sys, 'argv', ['ci_preflight.py', flag]), \ + contextlib.redirect_stdout(output): + code = ci_preflight.main() + return code, output.getvalue() + + @staticmethod + def about(output, name): + """The stage's lines about one file.""" + return [line for line in output.splitlines() if line.startswith(name + ':')] + + @staticmethod + def not_utf8(name, line, offset): + return (f'{name}:{line}: not valid UTF-8 (invalid start byte at byte offset {offset}): ' + 'save the file as UTF-8') + + @staticmethod + def bare_cr_finding(name, changed): + """The finding for the first bare CR of `changed`: its LF-delimited line and text.""" + index = changed.index(b'\r') + number = changed.count(b'\n', 0, index) + 1 + source = changed.decode('utf-8').split('\n')[number - 1].strip(' \t') + return ActionPins.unsupported_finding( + name, number, source, 'unsupported line break U+000D') + + def assert_bare_cr_refused(self, name, changed): + """main() over a tree holding `changed` as `name` must name its first bare CR.""" + tree = self.make_tree() + (tree / name).write_bytes(changed) + code, output = self.run_stage(tree, '--check-ci-workflow') + self.assertEqual(self.about(output, name), [self.bare_cr_finding(name, changed)], output) + self.assertEqual(code, 1, output) + + def assert_undecodable_is_a_finding(self, name, flag): + """An invalid byte appended to a real file is a finding that names it, never a crash.""" + original = (ROOT / name).read_bytes() + self.assertTrue(original.endswith(b'\n'), name) + tree = self.make_tree() + (tree / name).write_bytes(original + b'# \xff\n') + try: + code, output = self.run_stage(tree, flag) + except UnicodeError as error: + self.fail(f'{flag} crashed on an undecodable {name}: {error}') + self.assertEqual(self.about(output, name), [ + self.not_utf8(name, original.count(b'\n') + 1, len(original) + 2)], output) + self.assertEqual(code, 1, output) + + def test_a_bare_cr_in_place_of_the_final_lf_of_any_real_source_is_refused(self): + sources = self.action_sources() + self.assertTrue(any(name.startswith('.github/workflows/') for name in sources)) + self.assertTrue(any(name.startswith('.github/actions/') for name in sources)) + for name in sources: + with self.subTest(name=name): + original = (ROOT / name).read_bytes() + self.assertTrue(original.endswith(b'\n') and b'\r' not in original) + self.assert_bare_cr_refused(name, original[:-1] + b'\r') + + def test_a_bare_cr_between_two_lines_of_a_real_workflow_is_refused(self): + original = (ROOT / self.DOCS).read_bytes() + self.assertEqual(original.count(b'\non:\n'), 1) + self.assert_bare_cr_refused(self.DOCS, original.replace(b'\non:\n', b'\non:\r', 1)) + + def test_the_stage_argv_ci_runs_refuses_the_reported_input(self): + # No mock at all: the stage's own argv in a child process, over a copy + # of the tree whose docs.yml has its final LF replaced by a CR. + changed = (ROOT / self.DOCS).read_bytes()[:-1] + b'\r' + tree = self.make_tree() + shutil.copytree(ROOT / 'scripts', tree / 'scripts', + ignore=shutil.ignore_patterns('__pycache__', '*.pyc')) + (tree / self.DOCS).write_bytes(changed) + argv = {entry[0]: entry[1] for entry in check_commands(tree)}['ci-workflow-contract'] + result = subprocess.run(argv, cwd=tree, capture_output=True, text=True) + output = result.stdout + result.stderr + self.assertEqual(self.about(output, self.DOCS), [self.bare_cr_finding(self.DOCS, changed)], + output) + self.assertEqual(result.returncode, 1, output) + + def test_crlf_copies_of_every_stage_input_are_accepted_and_still_scanned(self): + tree = self.crlf_tree([*self.action_sources(), self.CMAKE]) + code, output = self.run_stage(tree, '--check-ci-workflow') + self.assertEqual(code, 0, output) + self.assertIn('runners and time limits OK', output) + # Zero findings must mean the grammar was checked: a mutable action + # after the last block scalar of a CRLF file is found on its own line. + original = (ROOT / self.DOCS).read_bytes() + (tree / self.DOCS).write_bytes(original.replace(b'\n', b'\r\n') + + b' - uses: actions/checkout@main\r\n') + code, output = self.run_stage(tree, '--check-ci-workflow') + self.assertEqual(self.about(output, self.DOCS), [ActionPins.finding( + self.DOCS, original.count(b'\n') + 1, 'actions/checkout@main')], output) + self.assertEqual(code, 1, output) + + def test_the_combined_checker_takes_the_file_texts_as_stored(self): + sources = [(ROOT / path).read_bytes().decode('utf-8') for path in CI_SOURCES] + self.assertEqual(ci_workflow_findings(*sources), []) + # The loaders now hand it CRLF files unchanged: every check but the + # action-pin lint still has to read them as the LF text they always did. + self.assertEqual(ci_workflow_findings(*(text.replace('\n', '\r\n') for text in sources)), []) + + def test_a_file_that_is_not_utf8_is_a_finding_not_a_crash(self): + for name in (self.DOCS, '.github/workflows/release.yml', + '.github/actions/setup-live-curl/action.yml', self.CMAKE): + with self.subTest(name=name): + self.assert_undecodable_is_a_finding(name, '--check-ci-workflow') + + def test_the_docs_stages_read_the_same_way(self): + for flag in ('--check-docs-workflow', '--self-test-docs-workflow'): + with self.subTest(flag=flag, kind='crlf'): + code, output = self.run_stage(self.crlf_tree((self.DOCS, self.DOCS_POST)), flag) + self.assertEqual(code, 0, output) + for name in (self.DOCS, self.DOCS_POST): + with self.subTest(flag=flag, name=name): + self.assert_undecodable_is_a_finding(name, flag) + + def test_the_runner_label_stage_reads_the_same_way(self): + flag = '--check-actionlint-runner-labels' + tree = self.crlf_tree([self.CONFIG, *self.workflow_sources()]) + code, output = self.run_stage(tree, flag) + self.assertEqual(code, 0, output) + self.assertIn('all literal self-hosted labels declared', output) + for name in ('.github/workflows/ci.yml', self.CONFIG): + with self.subTest(name=name): + self.assert_undecodable_is_a_finding(name, flag) + + def test_reverting_the_loader_to_read_text_is_detected(self): + changed = (ROOT / self.DOCS).read_bytes()[:-1] + b'\r' + self.assert_bare_cr_refused(self.DOCS, changed) + source = inspect.getsource(ci_preflight.read_ci_text) + guard = 'data = path.read_bytes()\n' + self.assertEqual(source.count(guard), 1) + mutant = source.replace( + guard, "data = path.read_text(encoding='utf-8').encode('utf-8')\n", 1) + namespace = {} + exec(compile(mutant, '', 'exec'), + vars(ci_preflight), namespace) + with mock.patch.object(ci_preflight, 'read_ci_text', namespace['read_ci_text']): + with self.assertRaises(AssertionError): + self.assert_bare_cr_refused(self.DOCS, changed) + + def test_dropping_the_lf_view_of_the_other_checks_is_detected(self): + for flag, names in (('--check-ci-workflow', (self.CMAKE,)), + ('--check-docs-workflow', (self.DOCS, self.DOCS_POST)), + ('--check-actionlint-runner-labels', + (self.CONFIG, *self.workflow_sources()))): + with self.subTest(flag=flag): + code, output = self.run_stage(self.crlf_tree(names), flag) + self.assertEqual(code, 0, output) + with mock.patch.object(ci_preflight, '_universal_newlines', lambda text: text): + code, output = self.run_stage(self.crlf_tree(names), flag) + self.assertEqual(code, 1, output) + + class PreflightFailures(unittest.TestCase): def ci_sources(self): return [(ROOT / path).read_text() for path in CI_SOURCES] @@ -721,13 +1684,13 @@ def test_aggregates_refuse_first_even_when_authz_is_skipped_from_mutated_files(s name = Path(CI_SOURCES[index]).name for after in ('', AUTHZ_REFUSE_STEP.replace('exit 1', 'true'), AUTHZ_REFUSE_STEP.replace("!= 'success'", "== 'failure'"), - AUTHZ_REFUSE_STEP + ' - uses: actions/checkout@v4\n'): + AUTHZ_REFUSE_STEP + ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n'): with self.subTest(workflow=name, mutation=after): findings = self.mutated_file_findings(index, job, AUTHZ_REFUSE_STEP, after) self.assertIn(f'{name} {job} must always run and refuse authz first', findings) # Keeping the refusal after checkout cannot satisfy refuse-first. findings = self.mutated_file_findings(index, job, AUTHZ_REFUSE_STEP, - ' - uses: actions/checkout@v4\n' + AUTHZ_REFUSE_STEP) + ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n' + AUTHZ_REFUSE_STEP) self.assertIn(f'{name} {job} must always run and refuse authz first', findings) for condition in ('${{ !cancelled() }}', '${{ success() }}'): findings = self.mutated_file_findings(index, job, ' if: always()\n', @@ -739,7 +1702,7 @@ def test_status_function_jobs_refuse_before_checkout_from_mutated_files(self): for index, job in ((0, 'preflight'), (0, 'build'), (6, 'build'), (6, 'build-macos'), (6, 'sanitizers'), (6, 'kernel-only')): for after in ('', AUTHZ_REFUSE_STEP.replace('exit 1', 'true'), - ' - uses: actions/checkout@v4\n' + AUTHZ_REFUSE_STEP): + ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n' + AUTHZ_REFUSE_STEP): with self.subTest(job=job, mutation=after): findings = self.mutated_file_findings(index, job, AUTHZ_REFUSE_STEP, after) self.assertIn(f'{Path(CI_SOURCES[index]).name} job {job} with a status function must refuse authz first', @@ -750,7 +1713,7 @@ def test_all_status_functions_require_refusal_and_repository_guards_from_mutated self.assertEqual(ci_workflow_findings(*original, others=self.other_sources()), []) body = _jobs(original[0])['preflight'] steps = ci_preflight._steps(body) - checkout = ci_preflight._step_named(steps, ' - uses: actions/checkout@v4') + checkout = ci_preflight._step_named(steps, ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0') verify = ci_preflight._step_named(steps, ' - name: Verify workflow wiring and source guards') guard = f' if: {ci_preflight.AUTHZ_SUCCESS}\n' unnamed_run = ' - run: python3 scripts/ci_preflight.py --output-dir build-ci-preflight\n' @@ -981,7 +1944,7 @@ def test_promotion_guard_rejects_mutations_of_the_real_workflow(self): ' env:\n KEY: ${{ secrets.PINEFORGE_APP_PRIVATE_KEY }}\n' ' steps:\n', guard_finding), ('promote-guard', ' steps:\n', - ' steps:\n - uses: actions/checkout@v4\n', guard_finding), + ' steps:\n - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n', guard_finding), ('promote-guard', ' steps:\n', ' if: false\n steps:\n', guard_finding), ('promote-guard', ' steps:\n', @@ -1082,6 +2045,20 @@ def test_promotion_guard_rejects_mutations_of_the_real_workflow(self): self.assertIn("baseline promotion must run main's copy and require the latest two head statuses", ci_workflow_findings(*changed)) + @staticmethod + def quoted_key_lint_findings(workflow, setting): + """The action-pin lint's refusal of an injected setting whose key is quoted. + + The promotion guard reads a quoted key like the plain one, so its own + findings do not change; the YAML whitelist refuses the quoted key too, + first, at the line the setting starts on in the changed workflow. + """ + if not setting.lstrip(' ').startswith(('"', "'")): + return [] + number = workflow.count('\n', 0, workflow.index(setting)) + 1 + return [ActionPins.unsupported_finding( + CI_SOURCES[2], number, setting.strip(), 'quoted keys')] + def test_promotion_guard_rejects_workflow_defaults_and_env(self): original = self.ci_sources() self.assertEqual(ci_workflow_findings(*original), []) @@ -1100,9 +2077,10 @@ def test_promotion_guard_rejects_workflow_defaults_and_env(self): changed[2] = changed[2].replace('\njobs:\n', '\n' + setting + 'jobs:\n', 1) self.assertNotEqual(changed[2], original[2]) self.assertEqual(_jobs(changed[2]), _jobs(original[2])) - self.assertEqual(ci_workflow_findings(*changed), [ + expected = self.quoted_key_lint_findings(changed[2], setting) + [ f'baseline promotion must not define workflow-level {key}', - ]) + ] + self.assertEqual(ci_workflow_findings(*changed), expected) def test_promotion_guard_rejects_job_defaults(self): original = self.ci_sources() @@ -1116,10 +2094,11 @@ def test_promotion_guard_rejects_job_defaults(self): changed[2] = in_job(changed[2], 'promote-guard', ' steps:\n', setting + ' steps:\n') self.assertNotEqual(changed[2], original[2]) - self.assertEqual(ci_workflow_findings(*changed), [ + expected = self.quoted_key_lint_findings(changed[2], setting) + [ 'baseline promotion promote-guard must not define job-level defaults', 'baseline promotion must start with the pinned read-only promote-guard', - ]) + ] + self.assertEqual(ci_workflow_findings(*changed), expected) def promotion_guard_result(self, *, workflow=None, **overrides): """Run the real workflow shell; only the external gh CLI is stubbed.""" @@ -1680,7 +2659,7 @@ def test_preflight_split_refuses_slow_stages_on_pr_and_missing_deferred_stages(s self.assertIn('preflight must keep only checkout, linters, fast checks and diagnostics', findings) for step in ( ' - run: cmake --build build\n', - ' - uses: actions/setup-python@v5\n', + ' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0\n', ' - run: python3 scripts/ci_verify.py release\n'): findings = self.mutated_file_findings(0, 'preflight', ' steps:\n', ' steps:\n' + step) @@ -1733,12 +2712,12 @@ def test_verification_steps_cannot_skip_their_home_events(self): def test_ci_smoke_cache_refuses_losing_main_fallback_or_pr_writes(self): mutations = [ - ('uses: actions/cache/restore@v4', 'uses: actions/cache@v4'), + ('uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830', 'uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830'), ('key: ccache-main-', 'key: ccache-pr-'), (' ccache-main-', ' ccache-'), (' restore-keys: |\n', ' restore-keys: |\n ccache-pr-\n'), (ci_preflight.BUILD_CACHE_SAVE, '${{ success() }}'), - ('uses: actions/cache/save@v4', 'uses: actions/cache/restore@v4'), + ('uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830', 'uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830'), ('key: ${{ steps.ccache-restore.outputs.cache-primary-key }}', 'key: ccache-pr'), ] for index, job in ((0, 'build'), (6, 'build'), (6, 'build-macos')): @@ -2451,8 +3430,8 @@ def test_legacy_metadata_cannot_replace_runner_pins(self): # A step-level declaration also cannot stand in for the actual runner. before = f' runs-on: {PR_RUNNER}\n' changed = in_job(workflows['ci.yml'], 'preflight', before, routed_line(before, RELEASE_RUNNER)) - changed = in_job(changed, 'preflight', ' - uses: actions/checkout@v4\n', - ' - uses: actions/checkout@v4\n env:\n' + changed = in_job(changed, 'preflight', ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n', + ' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0\n env:\n' f" PF_CI_LABELS: '{PR_RUNNER}'\n") findings = ci_preflight.runner_findings(dict(workflows, **{'ci.yml': changed})) self.assertIn(f'ci.yml job preflight must run on {PR_RUNNER}', findings) @@ -3017,6 +3996,19 @@ def classify(name, base_files, change): def test_harmless_spellings_are_not_findings(self): original, others = self.ci_sources(), self.other_sources() + + def quoted_key_findings(sources): + # These two existing spellings remain harmless to the runner and + # sizing policies, but the YAML whitelist now explicitly refuses + # quoted keys. Keep both inputs and require exactly that refusal. + findings = [] + for index in (0, 5): + for number, line in enumerate(sources[index].splitlines(), 1): + if re.match(r' *(?:"PF_CI_LABELS"|"runs-on"):', line): + findings.append(ActionPins.unsupported_finding( + CI_SOURCES[index], number, line.strip(), 'quoted keys')) + return findings + verify = KERNEL_VERIFY edits = ( (6, 'kernel-only', ' steps:\n', @@ -3048,7 +4040,10 @@ def test_harmless_spellings_are_not_findings(self): (6, 'kernel-only', ' - name: Verify (kernel)\n', ' - name: Verify (kernel), no longer --jobs 4\n'), (6, 'kernel-only', KERNEL_VERIFY, - 'uses: nick-fields/retry@v3\n with:\n command: python3 scripts/ci_verify.py ' + # A wrapper action spelled the way the ORG-ACTIONPINS lint accepts: a (synthetic) 40-hex sha + # and a version comment, so only the harmless spelling of the command is under test here. + 'uses: nick-fields/retry@0123456789abcdef0123456789abcdef01234567 # v3.2.0\n with:\n' + ' command: python3 scripts/ci_verify.py ' f'kernel --build-dir build-kernel --jobs {CORES} --ccache'), (2, 'promote', f' runs-on: {RELEASE_RUNNER}\n', f' runs-on: {RELEASE_RUNNER} # isolated release\n'), @@ -3059,12 +4054,14 @@ def test_harmless_spellings_are_not_findings(self): with self.subTest(file=CI_SOURCES[index], job=job, after=after): changed = original.copy() changed[index] = in_job(changed[index], job, before, after) - self.assertEqual(ci_workflow_findings(*changed, others=others), []) + self.assertEqual(ci_workflow_findings(*changed, others=others), + quoted_key_findings(changed)) changed = original.copy() for index, job, before, after in edits: if before in (changed[index] if job is None else _jobs(changed[index]).get(job, '')): changed[index] = in_job(changed[index], job, before, after) - self.assertEqual(ci_workflow_findings(*changed, others=others), []) + self.assertEqual(ci_workflow_findings(*changed, others=others), + quoted_key_findings(changed)) def test_a_finding_is_reported_once(self): changed = self.ci_sources() @@ -3942,7 +4939,7 @@ def test_release_ruleset_probe_workflow_mutations_refuse(self): ('job permissions', ' probe:\n', ' probe:\n permissions:\n contents: write\n'), ('runner', POST_RUNNER, RELEASE_RUNNER), ('PR runner', POST_RUNNER, PR_RUNNER), - ('app action', 'actions/create-github-app-token@v1', + ('app action', 'actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547', 'actions/create-github-app-token@v2'), ('app output', ' id: app', ' id: other'), ('app id', 'secrets.PINEFORGE_APP_ID', 'secrets.OTHER_APP_ID'), @@ -3987,7 +4984,7 @@ def test_release_ruleset_probe_refuses_missing_workflow_or_different_release_tok if name != 'release-ruleset-probe.yml'} self.assertIn(RULESETS_PROBE_FINDING, ci_workflow_findings(*original, others=missing)) for before, after in ( - ('actions/create-github-app-token@v1', 'actions/create-github-app-token@v2'), + ('actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547', 'actions/create-github-app-token@v2'), ('secrets.PINEFORGE_APP_ID', 'secrets.OTHER_APP_ID'), ('secrets.PINEFORGE_APP_PRIVATE_KEY', 'secrets.OTHER_PRIVATE_KEY'), (' repositories: pineforge-engine', @@ -4814,8 +5811,8 @@ def test_all_live_profiles_use_the_shared_pinned_dependency(self): self.assertIn('--curl-dir "${{ steps.curl-deps.outputs.curl-dir }}"', line) action = (ROOT / '.github/actions/setup-live-curl/action.yml').read_text() self.assertIn('bash scripts/build_live_curl.sh --metadata', action) - self.assertIn('uses: actions/cache/restore@v4', action) - self.assertIn('uses: actions/cache/save@v4', action) + self.assertIn('uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830', action) + self.assertIn('uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830', action) self.assertIn('bash scripts/build_live_curl.sh build-native-deps 4', action) def test_shared_curl_pin_and_websocket_build_flags(self): @@ -4853,7 +5850,7 @@ def test_native_contract_rejects_lost_profiles_budgets_caches_and_diagnostics(se ('live-v2-${{ matrix.profile }}-', 'live-v2-shared-'), ('ccache --zero-stats', 'ccache --show-stats'), (NATIVE_CACHE_SAVE, "${{ success() }}"), - ('uses: actions/cache/save@v4', 'uses: actions/cache@v4'), + ('uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830', 'uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830'), ('name: ci-diagnostics-native-live-${{ matrix.profile }}', 'name: ci-diagnostics-native-live'), ('--ccache ', '--ccache --test-timeout 900 '), @@ -4897,8 +5894,8 @@ def test_curl_contract_rejects_post_only_save_and_unguarded_cold_restore(self): for before, after in ( ("if: ${{ inputs.cold-cache != 'true' }}", 'if: always()'), (CURL_CACHE_SAVE, '${{ success() }}'), - ('uses: actions/cache/restore@v4', 'uses: actions/cache@v4'), - ('uses: actions/cache/save@v4', 'uses: actions/cache@v4'), + ('uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830', 'uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830'), + ('uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830', 'uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830'), ('curl-${{ inputs.cache-namespace }}-', 'curl-shared-'), ('${{ steps.dependencies.outputs.sha256 }}', 'unpinned'), ('${{ steps.dependencies.outputs.identity }}', 'shared-toolchain'), @@ -4944,9 +5941,9 @@ def test_cold_mode_bypasses_both_restores_and_dependencies_save_early(self): self.assertIn('if: ${{ !inputs.cold_cache }}', native) self.assertIn('cold-cache: ${{ inputs.cold_cache }}', native) self.assertIn("if: ${{ inputs.cold-cache != 'true' }}", action) - self.assertIn('uses: actions/cache/restore@v4', action) - self.assertIn('uses: actions/cache/save@v4', action) - self.assertNotIn('uses: actions/cache@v4', action) + self.assertIn('uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830', action) + self.assertIn('uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830', action) + self.assertNotIn('uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830', action) if __name__ == '__main__': diff --git a/scripts/test_ci_release_gate.py b/scripts/test_ci_release_gate.py index 7ab6cf71a..1752424ce 100644 --- a/scripts/test_ci_release_gate.py +++ b/scripts/test_ci_release_gate.py @@ -2502,7 +2502,7 @@ def test_prebuilt_builds_the_emitted_sha_and_asserts_tag_identity(self): ' echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"', push) steps = _steps(jobs["prebuilt"]) - checkout = next(step for step in steps if "uses: actions/checkout@v4" in step) + checkout = next(step for step in steps if "uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0" in step) self.assertIn(" ref: ${{ inputs.dry_run && github.sha || needs.release.outputs.sha }}\n", checkout) self.assertIn(" fetch-depth: 0", checkout)