From c14916322079eb808a0dd10aa469b96e4cf1ae2e Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:31:01 +0000 Subject: [PATCH] Add bounded integer and call-layout repairs before random permutation --- README.md | 8 + ROADMAP.md | 5 + src/decompnet/app/branch_layout_repairs.py | 203 +++++++++++++++++ src/decompnet/app/callee_prototype_repairs.py | 60 +++++ src/decompnet/app/draft_repairs.py | 11 + src/decompnet/app/guide/workflow.md | 7 + src/decompnet/app/integer_lifetime_repairs.py | 96 ++++++++ src/decompnet/app/integer_repair_support.py | 89 ++++++++ src/decompnet/app/local_experiments.py | 13 +- src/decompnet/app/local_widening_repairs.py | 62 +++++ src/decompnet/app/repair_ranking.py | 10 +- src/decompnet/app/tools.py | 2 +- tests/test_integer_call_repairs.py | 211 ++++++++++++++++++ 13 files changed, 771 insertions(+), 6 deletions(-) create mode 100644 src/decompnet/app/branch_layout_repairs.py create mode 100644 src/decompnet/app/callee_prototype_repairs.py create mode 100644 src/decompnet/app/integer_lifetime_repairs.py create mode 100644 src/decompnet/app/integer_repair_support.py create mode 100644 src/decompnet/app/local_widening_repairs.py create mode 100644 tests/test_integer_call_repairs.py diff --git a/README.md b/README.md index 2cc984ebd..9504a0612 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,14 @@ local declaration order before random permutation. It keeps original line layout These hypotheses run through the same compiler and exact judge as agents, in ROM and supported 32-bit imported projects. They use local CPU, with no model API calls. Rejected candidates do not replace the best saved C. +The same pass can widen a byte/short local when arithmetic or a later conversion suggests truncation, test +candidate-owned callee parameter/return widths, and split or merge bounded integer lifetimes. It also tries +calls inside argument-selecting branches, copies terminal goto leaves, and reorders switch cases with no +fallthrough. Missing no-op zero cases can restore a compiler's original dispatch layout. Shared headers and +project-wide prototypes stay unchanged. These are hypotheses: an improved similarity score is still a draft, +and only a byte-exact result counts as matched. The families also reach iterative/deep search and +`try_local_repairs`; loops, incoming jumps, shadowing and ambiguous declarations limit what is proposed. + For close ROM misses it also tests cached compiler flag alternatives, two probes per pass. Later passes skip already tested choices and can reach optimization levels as well as debug flags. Only exact results qualify for a narrow compiler-range change. When automatic compiler changes are disabled, the result becomes a suggestion for diff --git a/ROADMAP.md b/ROADMAP.md index a7ae8e8b7..d204aaec9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,10 @@ # Roadmap +Bounded integer/call repairs now include evidence-led local widening, candidate-owned callee prototype widths, +integer lifetime splits/merges, call distribution into branches, terminal goto leaves and switch layout variants. +They share the ROM/imported-project judge and existing budgets; a better similarity score remains a draft. +General CFG liveness, fallthrough switch rewriting and shared-header prototype inference remain future work. + What's planned for decompdecomp, roughly in order, and why. Every item keeps the rule that matters most: a match is byte-identical after linking (ROM mode) or objdiff 100% (fork mode). Nothing here accepts code that only does the same thing. diff --git a/src/decompnet/app/branch_layout_repairs.py b/src/decompnet/app/branch_layout_repairs.py new file mode 100644 index 000000000..303498ac4 --- /dev/null +++ b/src/decompnet/app/branch_layout_repairs.py @@ -0,0 +1,203 @@ +"""Alternative call/leaf and switch layouts, with bounded work and explicit control-flow guards.""" +import re + +from .integer_repair_support import IDENT, INTEGER, NUMBER, edit, integer, view +from .pattern_repairs import close + + +def call_statement(text, start): + """A direct call statement (optionally assigned), with nested argument parentheses kept intact.""" + hit = re.match(rf'\s*(?:{IDENT}\s*=\s*)?({IDENT})\s*\(', text[start:]) + if not hit or hit[1] in {'if', 'for', 'while', 'switch', 'sizeof', 'return'}: + return None + end = close(text, start+hit.end()-1) + tail = re.match(r'\s*;', text[end:]) if end else None + return (start+hit.start(), end+tail.end()) if tail else None + + +def flat_assignments(text): + return bool(re.fullmatch(rf'(?:\s*{IDENT}\s*=(?!=)[^;{{}}]*;)+\s*', text)) and not re.search( + r'\b(?:return|goto|break|continue|case|default|for|while|do)\b', text) + + +def terminal_case(text): + terminal = re.search(r'\b(?:break\s*;|return\b[^;{}]*;|goto\s+'+IDENT+r'\s*;)\s*$', text) + if not terminal or text[:terminal.start()].count('{') != text[:terminal.start()].count('}'): + return False + prefix = text[:terminal.start()].rstrip() + return not prefix or prefix[-1] in ';}' + + +def controls(part): + """Braced controls and their ends; an unrecognised scope is a boundary for leaf traversal.""" + result = {} + for pos in range(part.opening+1, part.end-1): + if part.clean[pos] != '{': + continue + prefix = part.clean[:pos].rstrip() + kind = 'else' if re.search(r'\belse$', prefix) else '' + if prefix.endswith(')'): + depth = 1 + start = len(prefix)-2 + while start >= 0 and depth: + depth += (prefix[start] == ')')-(prefix[start] == '(') + start -= 1 + hit = re.search(r'\b(if|switch|for|while)\s*$', prefix[:start+1]) + kind = hit[1] if hit else '' + if re.search(r'\bdo$', prefix): + kind = 'do' + end = close(part.clean, pos, '{', '}') + if end: + result[end-1] = (pos, kind) + return result + + +def leaf_transfer(part, start, scopes): + """Follow closing if arms past their else arms, stopping at one explicit terminal transfer.""" + pos = start + while True: + while pos < part.end and part.clean[pos].isspace(): + pos += 1 + if part.clean[pos:pos+1] != '}': + break + if pos not in scopes or scopes[pos][1] not in {'if', 'else'}: + return None + pos += 1 + other = re.match(r'\s*else\s*{', part.clean[pos:]) + if other: + end = close(part.clean, pos+other.end()-1, '{', '}') + if not end: + return None + pos = end + elif re.match(r'\s*else\b', part.clean[pos:]): + return None # shortcut: leave unbraced/else-if continuation to a full CFG parser + terminal = re.match(rf'\s*(?:goto\s+({IDENT})|return(?:\s+[^;{{}}]+)?|break)\s*;', part.clean[pos:]) + return (pos, pos+terminal.end(), terminal[1]) if terminal else None + + +def variants(source, name): + part = view(source, name) + if not part: + return [] + result = [] + clean = part.clean + # Distribute a call following an if/else which only selects scalar arguments. Retain assignments: + # they may still be read afterwards, and evaluating an RHS twice would change its meaning. + for hit in re.finditer(r'\bif\s*\(', part.body): + cond_end = close(clean, part.opening+1+hit.end()-1) + arm = re.match(r'\s*{', clean[cond_end:]) if cond_end else None + if not arm: + continue + left = cond_end+arm.end()-1 + left_end = close(clean, left, '{', '}') + other = re.match(r'\s*else\s*{', clean[left_end:]) if left_end else None + if not other: + continue + right = left_end+other.end()-1 + right_end = close(clean, right, '{', '}') + if not right_end or not all(flat_assignments(clean[a+1:b-1]) for a, b in [(left, left_end), (right, right_end)]): + continue + call = call_statement(clean, right_end) + if not call: + continue + statement = source[call[0]:call[1]] + result.append(('branch_call', edit(source, [(left_end-1, left_end-1, statement), + (right_end-1, right_end-1, statement), (*call, '')]))) + if sum(family == 'branch_call' for family, _ in result) >= 4: + break + # Inline a leaf with its terminal transfer, never a label whose following code would fall through. + # Duplicate local names could change lookup at the new site, so leave those functions alone. + decls = list(re.finditer(rf'(?:^|[;{{}}])\s*(?!goto\b|return\b|break\b|continue\b)' + rf'(?:{IDENT}\s+)+\**\s*({IDENT})\s*(?=[;=,\[])', part.body)) + declarations = [decl[1] for decl in decls] + nested = any(part.body[:decl.start(1)].count('{') != part.body[:decl.start(1)].count('}') for decl in decls) + if not nested and len(declarations) == len(set(declarations)): + scopes = controls(part) + # Unbraced loops can change a copied break's destination too; don't reason through them. + unbraced = False + for loop in re.finditer(r'\b(?:for|while)\s*\(', part.body): + loop_start = part.opening+1+loop.start() + if loop[0].startswith('while') and any(kind == 'do' and ending < loop_start + and not clean[ending+1:loop_start].strip() for ending, (_, kind) in scopes.items()): + continue # the latch of a braced do/while is not a second, unbraced loop + loop_end = close(clean, part.opening+1+loop.end()-1) + if loop_end and not re.match(r'\s*{', clean[loop_end:]): + unbraced = True + + def break_scope(pos): + enclosing = [(opening, ending) for ending, (opening, kind) in scopes.items() + if opening < pos < ending and kind in {'switch', 'for', 'while', 'do'}] + return max(enclosing, default=None) + + for label in re.finditer(rf'\b({IDENT})\s*:(?!:)', part.body): + start = part.opening+1+label.end() + call = call_statement(clean, start) + if not call: + continue + terminal = leaf_transfer(part, call[1], scopes) + if not terminal: + continue + transfer_start, end, destination = terminal + leaf = source[start:call[1]]+source[transfer_start:end] + if destination == label[1]: + continue + for jump in re.finditer(rf'\bgoto\s+{re.escape(label[1])}\s*;', part.body): + pos = part.opening+1+jump.start() + if start <= pos < end: + continue + if re.search(r'\bbreak\b', clean[transfer_start:end]) and ( + unbraced or break_scope(pos) is None or break_scope(pos) != break_scope(start)): + continue + result.append(('goto_leaf', edit(source, [(pos, part.opening+1+jump.end(), '{'+leaf+'}')])) ) + if sum(family == 'goto_leaf' for family, _ in result) >= 4: + break + if sum(family == 'goto_leaf' for family, _ in result) >= 4: + break + for switch in re.finditer(r'\bswitch\s*\(', part.body): + condition = close(clean, part.opening+1+switch.end()-1) + arm = re.match(r'\s*{', clean[condition:]) if condition else None + if not arm: + continue + opening = condition+arm.end()-1 + end = close(clean, opening, '{', '}') + if not end: + continue + inside = clean[opening+1:end-1] + labels = [hit for hit in re.finditer(rf'\b(?:case\s+(-?{NUMBER})|default)\s*:', inside) + if inside[:hit.start()].count('{') == inside[:hit.start()].count('}')] + if not labels or inside[:labels[0].start()].strip(): + continue + # Reject non-literal/range labels, not merely the subset our regex understands. + all_labels = [hit for hit in re.finditer(r'\b(?:case\b[^:]*|default)\s*:', inside) + if inside[:hit.start()].count('{') == inside[:hit.start()].count('}')] + if len(labels) != len(all_labels): + continue + if not any(label[1] is None or integer(label[1].lstrip('-')) == 0 for label in labels): + result.append(('switch_layout', edit(source, [(opening+1, opening+1, '\ncase 0: break;\n')]))) + starts = [] + pending = labels[0].start() + valid = True + for index, label in enumerate(labels): + stop = labels[index+1].start() if index+1 < len(labels) else len(inside) + code = inside[label.end():stop] + if not code.strip(): + continue # grouped labels travel with their one body + # Only block-scoped declarations can move with a case. No fallthrough or cross-case labels. + if re.search(rf'\b(?:{INTEGER}|void|float|double)\s*\**\s*{IDENT}', code): + valid = False + break + if not terminal_case(code): + valid = False + break + starts.append((pending, stop)) + pending = stop + if not valid or not 2 <= len(starts) <= 6: + continue + chunks = [source[opening+1+a:opening+1+b] for a, b in starts] + orders = [list(range(rotation, len(chunks)))+list(range(rotation)) for rotation in range(1, len(chunks))] + orders += [list(reversed(range(len(chunks))))] + for order in orders[:6]: + result.append(('switch_layout', edit(source, [(opening+1+starts[0][0], opening+1+starts[-1][1], + ''.join(chunks[index] for index in order))]))) + break # bounded per function, not factorial permutations of every nested switch + return list(dict.fromkeys(result)) diff --git a/src/decompnet/app/callee_prototype_repairs.py b/src/decompnet/app/callee_prototype_repairs.py new file mode 100644 index 000000000..c62c5fae4 --- /dev/null +++ b/src/decompnet/app/callee_prototype_repairs.py @@ -0,0 +1,60 @@ +"""Repair candidate-owned scalar callee prototypes without editing a project's shared declarations. + +Wrong short formals can add conversions at every call. Masked results can also reveal a wrongly inferred +return width. Unknown, conditional, variadic and indirect declarations remain with the real compiler. +""" +import re + +from ..forks.csource import find +from .integer_repair_support import INTEGER, calls, edit, split_args, view +from .repair_preflight import _PROTO + +NARROW = {'s8', 'u8', 's16', 'u16', 'char', 'short', 'signed char', 'unsigned char', 'signed short', 'unsigned short'} + + +def variants(source, name): + part = view(source, name) + if not part or re.search(r'^\s*#', part.clean, re.M): + return [] + prototypes = list(_PROTO.finditer(part.clean[:part.opening])) + result = [] + for proto in prototypes: + callee = proto[1] + if callee == name or sum(p[1] == callee for p in prototypes) != 1 or find(source, callee): + continue + sites = list(calls(part, callee)) + if not sites or not proto[2].strip() or '...' in proto[2]: + continue + # A local object/parameter can shadow this function name. + without_proto = part.clean[:proto.start()]+part.clean[proto.end():] + if re.search(rf'\b{re.escape(callee)}\b(?!\s*\()', without_proto): + continue + params = split_args(proto[2]) + for start, end in params: + param = proto[2][start:end] + match = re.fullmatch(rf'\s*({INTEGER})(?:\s+[A-Za-z_]\w*)?\s*', param) + if not match or ' '.join(match[1].split()) not in NARROW: + continue + offset = proto.start(2)+start + result.append(('callee_parameter_width', edit(source, [(offset+match.start(1), offset+match.end(1), 'int')]))) + if len(result) >= 6: + break + # Infer return-width choices only from a captured direct result subsequently masked in this caller. + returns = re.match(rf'\s*(?:extern\s+)?({INTEGER})\s+{re.escape(callee)}\s*\(', proto[0]) + if returns and ' '.join(returns[1].split()) in {'s32', 'u32', 'int', 'signed int', 'unsigned int'}: + for start, _ in sites: + # Call whitespace can vary; locate the assignment ending at the callee token instead. + assignment = re.search(rf'\b([A-Za-z_]\w*)\s*=\s*{re.escape(callee)}\s*\($', part.clean[part.opening+1:start]) + if not assignment: + continue + variable = re.escape(assignment[1]) + masks = re.findall(rf'\b{variable}\s*&\s*0x(FF|FFFF)\b', part.body, re.I) + for mask in dict.fromkeys(masks): + for typ in (('unsigned char', 'signed char') if len(mask) == 2 else ('unsigned short', 'short')): + offset = proto.start()+returns.start(1) + result.append(('callee_return_width', edit(source, [(offset, proto.start()+returns.end(1), typ)]))) + if masks: + break + if len(result) >= 10: + break + return list(dict.fromkeys(result))[:10] diff --git a/src/decompnet/app/draft_repairs.py b/src/decompnet/app/draft_repairs.py index 663359d59..6d1799b1b 100644 --- a/src/decompnet/app/draft_repairs.py +++ b/src/decompnet/app/draft_repairs.py @@ -9,11 +9,15 @@ from .. import m2cfix, seeds from ..forks.csource import _blank, find from . import ( + branch_layout_repairs, branch_locals, + callee_prototype_repairs, captured_repairs, expression_repairs, + integer_lifetime_repairs, lifetime_repairs, load_pair_repairs, + local_widening_repairs, local_width_repairs, mmio_repairs, near_repairs, @@ -74,6 +78,13 @@ def variants(source: str, name: str, arch: str, asm: str = '', flags=(), labels= for text in local_width_repairs.variants(source, name): proposals.append(text) labels[text] = 'local_width' + for text in local_widening_repairs.variants(source, name): + proposals.append(text) + labels[text] = 'local_widening' + for module in (callee_prototype_repairs, integer_lifetime_repairs, branch_layout_repairs): + for family, text in module.variants(source, name): + proposals.append(text) + labels[text] = family for text in pointer_cast_repairs.variants(source, name, arch): proposals.append(text) labels[text] = 'pointer_cast_inline' diff --git a/src/decompnet/app/guide/workflow.md b/src/decompnet/app/guide/workflow.md index 4f46e2d67..ada5023ae 100644 --- a/src/decompnet/app/guide/workflow.md +++ b/src/decompnet/app/guide/workflow.md @@ -30,3 +30,10 @@ Goal: C that the game's own compiler turns into exactly the bytes in the ROM. No ## Priorities Get the instruction count and control flow right first, then operands, then register allocation. Plausible original code beats contortions: games were written by people, quickly. + +Before sending many hand-written variants, use `try_local_repairs` on the current revision. Its bounded +families include `local_widening`, `callee_parameter_width`, `callee_return_width`, `split_integer_lifetime`, +`merge_integer_lifetimes`, `branch_call`, `goto_leaf` and `switch_layout`. They can test lost coordinate bits, +unwanted call conversions, integer register lifetimes and alternative dispatch layouts without model-written +source variants. Candidate-owned prototypes change only in that candidate; confirm a discovered signature +before suggesting a shared header edit. A lower score is useful evidence, not a proof of correctness. diff --git a/src/decompnet/app/integer_lifetime_repairs.py b/src/decompnet/app/integer_lifetime_repairs.py new file mode 100644 index 000000000..2f388cb8a --- /dev/null +++ b/src/decompnet/app/integer_lifetime_repairs.py @@ -0,0 +1,96 @@ +"""Bounded integer lifetime splits and non-overlapping merges for old register allocators. + +No address-taken variable is rewritten. Loops, shadowing and incoming jumps are boundaries, not reasons +to guess liveness from textual order. The ordinary judge still decides every generated hypothesis. +""" +import re + +from .integer_repair_support import IDENT, edit, locals_for, refs, view +from .pattern_repairs import close + + +def depth(text, end): + return text[:end].count('{')-text[:end].count('}') + + +def loop_ranges(part): + ranges = [] + for hit in re.finditer(r'\b(for|while|do)\b', part.clean[part.opening:part.end]): + start = part.opening+hit.start() + pos = part.opening+hit.end() + if hit[1] != 'do': + opening = part.clean.find('(', pos) + pos = close(part.clean, opening) if opening >= 0 else None + if pos is None: + continue + while pos < part.end and part.clean[pos].isspace(): + pos += 1 + end = close(part.clean, pos, '{', '}') if part.clean[pos:pos+1] == '{' else part.clean.find(';', pos)+1 + if end and end > pos: + ranges.append((start, end)) + return ranges + + +def jump_crosses(part, start, end): + labels = {hit[1]: hit.start() for hit in re.finditer(rf'\b({IDENT})\s*:(?!:)', part.clean)} + for jump in re.finditer(rf'\bgoto\s+({IDENT})\s*;', part.clean): + target = labels.get(jump[1]) + if target is None: + return True + if (start <= jump.start() < end) != (start <= target < end): + return True + return False + + +def variants(source, name): + part = view(source, name) + if not part: + return [] + loops = loop_ranges(part) + declarations = [decl for decl in locals_for(part) if depth(part.clean, decl.start()) == depth(part.clean, part.opening)+1] + active = [(decl, refs(part, decl)) for decl in declarations] + active = [(decl, hits) for decl, hits in active if hits] + result = [] + # Splitting an independent top-level definition preserves every evaluation and update in its segment. + for decl, hits in active: + variable = decl['name'] + definitions = [] + for hit in hits: + definition = re.match(r'\s*=(?!=)([^;{}]*);', part.clean[hit.end():]) + prefix = part.clean[part.opening+1:hit.start()].rstrip() + if (definition and not re.search(rf'\b{re.escape(variable)}\b', definition[1]) + and prefix[-1:] in {';', '}'} and depth(part.clean, hit.start()) == depth(part.clean, decl.start()) + and not any(a <= hit.start() < b for a, b in loops)): + definitions.append(hit) + for index, definition in enumerate(definitions[1:3], 1): + stop = definitions[index+1].start() if index+1 < len(definitions) else part.end-1 + if jump_crosses(part, definition.start(), stop): + continue + fresh = f'__repair_{variable}_{index}' + if re.search(rf'\b{fresh}\b', part.clean): + continue + edits = [(decl.end(), decl.end(), f'\n {decl["type"]} {fresh};')] + edits += [(hit.start(), hit.end(), fresh) for hit in hits if definition.start() <= hit.start() < stop] + result.append(('split_integer_lifetime', edit(source, edits))) + if len(result) >= 4: + break + pairs = [] + for left, left_refs in active: + for right, right_refs in active: + if left['type'] != right['type'] or left_refs[-1].end() >= right_refs[0].start(): + continue + start, end = left_refs[0].start(), right_refs[-1].end() + region = part.clean[start:end] + if re.search(r'\b(?:for|while|do|goto|case|default|break|continue)\b', region): + continue + if any(a <= start < b or a <= right_refs[0].start() < b for a, b in loops) or jump_crosses(part, start, end): + continue + if not re.match(r'\s*=(?!=)', part.clean[right_refs[0].end():]): + continue + # Scope crossings through if arms are fine; a declaration's scope and type stay unchanged. + same_stem = re.sub(r'_\d+$', '', left['name']) == re.sub(r'_\d+$', '', right['name']) + pairs.append((not same_stem, right_refs[0].start()-left_refs[-1].end(), left, right, right_refs)) + for _, _, left, right, hits in sorted(pairs, key=lambda pair: pair[:2])[:6]: + edits = [(right.start(), right.end(), '')]+[(hit.start(), hit.end(), left['name']) for hit in hits] + result.append(('merge_integer_lifetimes', edit(source, edits))) + return list(dict.fromkeys(result)) diff --git a/src/decompnet/app/integer_repair_support.py b/src/decompnet/app/integer_repair_support.py new file mode 100644 index 000000000..158f3ac7f --- /dev/null +++ b/src/decompnet/app/integer_repair_support.py @@ -0,0 +1,89 @@ +"""Small, conservative C spans shared by integer repair hypotheses, not a replacement C parser.""" +import re +from dataclasses import dataclass + +from ..forks.csource import _blank, find +from .pattern_repairs import close + +IDENT = r'[A-Za-z_]\w*' +INTEGER = r'(?:unsigned\s+(?:char|short|int)|signed\s+(?:char|short|int)|[su](?:8|16|32)|char|short|int)' +DECL = re.compile(rf'\b(?P{INTEGER})\s+(?P{IDENT})\s*;') +NUMBER = r'(?:0[xX][0-9a-fA-F]+|[0-9]+)[uUlL]*' + + +@dataclass +class View: + source: str + clean: str + opening: int + end: int + + @property + def body(self): + return self.clean[self.opening+1:self.end-1] + + +def view(source, name): + span = find(source, name) + if not span: + return None + clean = _blank(source) + opening = clean.find('{', *span) + if opening < 0 or '#' in clean[opening:span[1]] or re.search(r'::|\b(?:class|template|namespace)\b', clean): + return None + return View(source, clean, opening, span[1]) + + +def locals_for(part): + """Ordinary, unescaped scalar declarations; leave shadows and qualified objects to the compiler.""" + found = [] + for decl in DECL.finditer(part.clean, part.opening+1, part.end): + prefix = part.clean[part.opening+1:decl.start()].rstrip() + if prefix and prefix[-1] not in ';{}': + continue + name = decl['name'] + # Also notice shadows declared as pointers, qualified objects or for-loop variables. + declarations = re.findall(rf'\b(?:{INTEGER}|void|float|double)\s*\**\s*{re.escape(name)}\s*(?=[;=,\[])', part.body) + if len(declarations) != 1 or re.search(rf'&\s*{re.escape(name)}\b', part.body): + continue + found.append(decl) + return found + + +def refs(part, decl): + return [hit for hit in re.finditer(rf'\b{re.escape(decl["name"])}\b', part.clean) + if part.opening < hit.start() < part.end and not decl.start() <= hit.start() < decl.end() + and not re.search(r'(?:\.|->|\bgoto)\s*$', part.clean[:hit.start()]) + and not re.match(r'\s*:(?!:)', part.clean[hit.end():])] + + +def edit(source, edits): + for start, end, replacement in sorted(edits, reverse=True): + source = source[:start]+replacement+source[end:] + return source + + +def integer(text): + text = re.sub(r'[uUlL]+$', '', text) + return int(text, 16 if text.lower().startswith('0x') else 10) + + +def split_args(text): + start = depth = 0 + result = [] + for pos, char in enumerate(text): + depth += (char in '([{')-(char in ')]}') + if char == ',' and depth == 0: + result.append((start, pos)) + start = pos+1 + if text.strip(): + result.append((start, len(text))) + return result + + +def calls(part, name): + for hit in re.finditer(rf'(?]){re.escape(name)}\s*\(', part.clean[part.opening+1:part.end-1]): + start = part.opening+1+hit.end() + end = close(part.clean, start-1) + if end: + yield start, end-1 diff --git a/src/decompnet/app/local_experiments.py b/src/decompnet/app/local_experiments.py index 457520186..bc308825e 100644 --- a/src/decompnet/app/local_experiments.py +++ b/src/decompnet/app/local_experiments.py @@ -8,10 +8,14 @@ from . import ( agent_targets, agent_tokens, + branch_layout_repairs, + callee_prototype_repairs, captured_repairs, expression_repairs, + integer_lifetime_repairs, lifetime_repairs, load_pair_repairs, + local_widening_repairs, local_width_repairs, mmio_repairs, pattern_repairs, @@ -24,7 +28,8 @@ FAMILIES = ('signedness','declaration_order','typed_temporary','branch_orientation','narrowing','source_lines', 'address_temporary','constant_local','comparison_load','stack_array','loop_induction', - 'parameter_width','value_lifetime','sum_shape','call_result_inline','comparison_reuse','integer_address_order','mmio_address','call_arity','struct_byte_offset','switch_call','paired_load_order','array_index_width','split_scalar_lifetime','pointer_before_store','store_address_lifetime','stack_padding','local_width','pointer_cast_inline') + 'parameter_width','value_lifetime','sum_shape','call_result_inline','comparison_reuse','integer_address_order','mmio_address','call_arity','struct_byte_offset','switch_call','paired_load_order','array_index_width','split_scalar_lifetime','pointer_before_store','store_address_lifetime','stack_padding','local_width','pointer_cast_inline', + 'local_widening','callee_parameter_width','callee_return_width','split_integer_lifetime','merge_integer_lifetimes','branch_call','goto_leaf','switch_layout') def run(net, agent, slug, function, revision, families=None, max_trials=6): @@ -64,6 +69,12 @@ def run(net, agent, slug, function, revision, families=None, max_trials=6): for family,candidate in lifetime_repairs.variants(source,fn['name']): if family in pools: pools[family].append(candidate) + for module in (callee_prototype_repairs, integer_lifetime_repairs, branch_layout_repairs): + for family, candidate in module.variants(source, fn['name']): + if family in pools: + pools[family].append(candidate) + if 'local_widening' in pools: + pools['local_widening'] = local_widening_repairs.variants(source, fn['name']) if 'pointer_cast_inline' in pools: pools['pointer_cast_inline'] = pointer_cast_repairs.variants(source,fn['name'],proj['arch']) if 'local_width' in pools: diff --git a/src/decompnet/app/local_widening_repairs.py b/src/decompnet/app/local_widening_repairs.py new file mode 100644 index 000000000..4d4ac164d --- /dev/null +++ b/src/decompnet/app/local_widening_repairs.py @@ -0,0 +1,62 @@ +"""Try wider locals when arithmetic or a later conversion contradicts a byte/halfword declaration. + +A high similarity score can hide a truncated screen coordinate. These bounded hypotheses use source +evidence; they never change a matched function or replace the ordinary compile/link judge. +""" +import re + +from .integer_repair_support import NUMBER, edit, integer, locals_for, refs, view +from .pattern_repairs import close + +WIDTHS = {'s8': 8, 'u8': 8, 'char': 8, 'signed char': 8, 'unsigned char': 8, + 's16': 16, 'u16': 16, 'short': 16, 'signed short': 16, 'unsigned short': 16} + + +def large_arithmetic(rhs, maximum): + """A large coefficient/value at expression depth zero, not a pointer's field offset or cast type.""" + rhs = rhs.strip() + while rhs.startswith('(') and close(rhs, 0) == len(rhs): + rhs = rhs[1:-1].strip() + if re.search(r'\bsizeof\b', rhs): + return False + depth = 0 + for token in re.finditer(rf'(? maximum: + return bool(re.fullmatch(NUMBER, rhs) or re.search(r'[+*\-]|<<', rhs)) + return False + + +def variants(source, name): + part = view(source, name) + if not part: + return [] + result = [] + for decl in locals_for(part): + typ = ' '.join(decl['type'].split()) + bits = WIDTHS.get(typ) + if not bits: + continue + unsigned = typ.startswith('u') or typ.startswith('unsigned') + maximum = (1 << (bits if unsigned else bits-1))-1 + arithmetic = conversion = False + for hit in refs(part, decl): + tail = part.clean[hit.end():] + assignment = re.match(r'\s*=(?!=)([^;{}]+);', tail) + if assignment: + arithmetic |= large_arithmetic(assignment[1], maximum) + if bits == 8: + conversion |= bool(re.search(r'\(\s*(?:[su]16|(?:unsigned\s+)?short)\s*\)\s*$', part.clean[:hit.start()])) + shift = re.match(rf'\s*<<\s*({NUMBER})', tail) + conversion |= bool(shift and integer(shift[1]) >= bits) + if not arithmetic and not conversion: + continue + choices = ('short', 'unsigned short', 'int') if bits == 8 else ('int', 'unsigned int') + for choice in choices: + result.append(edit(source, [(decl.start('type'), decl.end('type'), choice)])) + if len(result) >= 8: + break + return result[:8] diff --git a/src/decompnet/app/repair_ranking.py b/src/decompnet/app/repair_ranking.py index 1b7aa66b7..f91a7cafa 100644 --- a/src/decompnet/app/repair_ranking.py +++ b/src/decompnet/app/repair_ranking.py @@ -5,16 +5,18 @@ def order(families, rows): focus = classify(rows) markers = {row[0] for row in rows if row[0] != ' '} if focus == 'call': - preferred = ('call_arity', 'call_result_inline', 'parameter_width', 'parameter_storage') + preferred = ('call_arity', 'callee_parameter_width', 'callee_return_width', 'call_result_inline', 'parameter_width', 'parameter_storage') elif markers and markers <= {'r', 's'}: preferred = ('stack_padding', 'unused_scalar_local', 'local_storage', 'parameter_storage', 'local_declaration_order', 'stack_array', 'parameter_width', 'declaration_order') if 's' in markers else ( 'store_address_lifetime', 'pointer_cast_inline', 'comparison_reuse', 'integer_address_order', - 'typed_temporary', 'value_lifetime', 'split_scalar_lifetime', 'array_index_width', 'local_width', 'declaration_order') + 'typed_temporary', 'value_lifetime', 'split_integer_lifetime', 'merge_integer_lifetimes', + 'split_scalar_lifetime', 'array_index_width', 'local_width', 'declaration_order') elif markers & {'<', '>', '|', 'o'}: - preferred = ('paired_load_order', 'pointer_before_store', 'branch_orientation', 'loop_induction', 'call_result_inline', 'parameter_width') + preferred = ('paired_load_order', 'pointer_before_store', 'branch_call', 'goto_leaf', 'switch_layout', + 'branch_orientation', 'loop_induction', 'callee_parameter_width', 'call_result_inline', 'parameter_width') elif 'i' in markers: - preferred = ('integer_address_order', 'signedness', 'narrowing', 'constant_local') + preferred = ('local_widening', 'integer_address_order', 'signedness', 'narrowing', 'constant_local') else: return list(families) return sorted(families, key=lambda f: preferred.index(f) if f in preferred else len(preferred)) diff --git a/src/decompnet/app/tools.py b/src/decompnet/app/tools.py index 9770567e1..0a86937a1 100644 --- a/src/decompnet/app/tools.py +++ b/src/decompnet/app/tools.py @@ -623,7 +623,7 @@ async def get_compiler_trial(ctx: Context, function: str, trial_id: str) -> str: @tool async def try_local_repairs(ctx: Context, function: str, revision: str, families: list[str] | None = None, max_trials: int = 6) -> str: - """Try applicable local repair families against one revision; at most six normal compiles and 60 seconds. Omit families to discover applicable repairs; retain best continuation. Use instead of parallel family probes.""" + """Try applicable local repairs against one revision; at most six normal compiles and 60 seconds. Omit families to discover repairs. Includes local_widening, callee_parameter_width, callee_return_width, split_integer_lifetime, merge_integer_lifetimes, branch_call, goto_leaf and switch_layout. Retain best continuation; use instead of parallel family probes.""" return await run(ctx,'try_local_repairs',function=function,revision=revision,families=families,max_trials=max_trials) @tool diff --git a/tests/test_integer_call_repairs.py b/tests/test_integer_call_repairs.py new file mode 100644 index 000000000..23e5a0fbf --- /dev/null +++ b/tests/test_integer_call_repairs.py @@ -0,0 +1,211 @@ +"""Synthetic counterexamples and real compiler checks for bounded integer/call layout hypotheses.""" +import json + +import pytest + +from decompnet.app import ( + branch_layout_repairs, + callee_prototype_repairs, + draft_repairs, + integer_lifetime_repairs, + local_experiments, + local_widening_repairs, +) + +WIDE = 'int f(int offset) { signed char coordinate; coordinate = 320-offset; return coordinate; }' +PROTO = 'int draw(int,short);int f(int y){return draw(7,y-2);}' +MERGE = 'int f(int x){int value;int value_2;value=x+3;use(value);value_2=x+5;return value_2;}' +BRANCH = 'int draw(int);void f(int flag){int x;if(flag){x=3;}else{x=7;}draw(x);}' + + +def test_widening_requires_arithmetic_or_a_wider_conversion_and_keeps_evaluations(): + choices = local_widening_repairs.variants(WIDE, 'f') + assert len(choices) == 3 and any('int coordinate;' in text for text in choices) + assert all('coordinate = 320-offset;' in text and 'return coordinate;' in text for text in choices) + assert local_widening_repairs.variants('int f(int x){u8 index;index=x/10;return (s16)index;}', 'f') + assert local_widening_repairs.variants('int f(int x){u8 index;index=x/10;return (index<<16)>>15;}', 'f') + + +@pytest.mark.parametrize('source', [ + WIDE.replace('320-offset', '7-offset'), + WIDE.replace('320-offset', 'read8000()'), + WIDE.replace('320-offset', '*(unsigned char *)(base+320)'), + WIDE.replace('signed char coordinate;', 'volatile signed char coordinate;'), + WIDE.replace('coordinate =', 'escape(&coordinate); coordinate ='), + WIDE.replace('return coordinate;', '{int coordinate;use(coordinate);}return coordinate;'), + WIDE.replace('coordinate =', '#define VALUE 320\ncoordinate ='), +]) +def test_widening_skips_offsets_qualified_objects_escapes_shadows_and_macros(source): + assert not local_widening_repairs.variants(source, 'f') + + +def test_callee_repair_changes_only_the_owned_prototype_and_not_the_calls(): + choices = callee_prototype_repairs.variants(PROTO, 'f') + assert choices == [('callee_parameter_width', 'int draw(int,int);int f(int y){return draw(7,y-2);}')] + source = 'int decode(int);int f(int bits){int index;index=decode(bits);return index&0xff;}' + returns = [text for family, text in callee_prototype_repairs.variants(source, 'f') if family == 'callee_return_width'] + assert len(returns) == 2 and all('index=decode(bits);return index&0xff;' in text for text in returns) + assert any('unsigned char decode(int)' in text for text in returns) + + +@pytest.mark.parametrize('source', [ + PROTO.replace('int draw(int,short);', 'int draw(int,short);int draw(int,short);'), + PROTO.replace('int draw(int,short);', 'int draw(int,short,...);'), + PROTO.replace('int draw(int,short);', 'int (*draw)(int,short);'), + PROTO.replace('int draw(int,short);', '#if EXTERNAL\nint draw(int,short);\n#endif\n'), + PROTO.replace('int f(int y){', 'int f(int y){int draw;'), + PROTO.replace('int f(int y)', 'int f(int y, int draw)'), + PROTO.replace('int draw(int,short);', 'int draw(int x,short y){return x+y;}'), +]) +def test_callee_repair_leaves_ambiguous_variadic_indirect_shadowed_and_defined_callees_alone(source): + assert not callee_prototype_repairs.variants(source, 'f') + + +def test_integer_lifetimes_split_independent_segments_and_merge_only_disjoint_same_types(): + source = 'int f(int x){int value;value=x+3;use(value);value=x+7;use(value);value=x+9;return value;}' + splits = [text for family, text in integer_lifetime_repairs.variants(source, 'f') if family == 'split_integer_lifetime'] + assert splits and 'value=x+3;use(value)' in splits[0] + assert '__repair_value_1=x+7;use(__repair_value_1)' in splits[0] + assert 'value=x+9;return value' in splits[0] + merges = [text for family, text in integer_lifetime_repairs.variants(MERGE, 'f') if family == 'merge_integer_lifetimes'] + assert merges and 'int value_2;' not in merges[0] and 'value=x+5;return value;' in merges[0] + assert not any(family == 'merge_integer_lifetimes' for family, _ in integer_lifetime_repairs.variants(MERGE.replace('int value_2;', 'short value_2;'), 'f')) + + +@pytest.mark.parametrize('source', [ + MERGE.replace('return value_2;', 'return value_2+value;'), + MERGE.replace('use(value);', 'escape(&value);'), + MERGE.replace('value=x+3;', 'do{value=x+3;').replace('return value_2;', 'use(value_2);}while(x--);return 0;'), + MERGE.replace('value=x+3;', 'again:value=x+3;').replace('return value_2;', 'if(x--)goto again;return value_2;'), + MERGE.replace('return value_2;', '{int value;use(value);}return value_2;'), +]) +def test_integer_merge_rejects_overlapping_escaped_looped_jump_crossed_and_shadowed_values(source): + assert not any(family == 'merge_integer_lifetimes' for family, _ in integer_lifetime_repairs.variants(source, 'f')) + + +def test_integer_split_rejects_a_jump_that_can_bypass_the_new_definition(): + source = 'int f(int x){int value;value=x;use(value);if(x)goto later;value=x+3;later:return value;}' + assert not any(family == 'split_integer_lifetime' for family, _ in integer_lifetime_repairs.variants(source, 'f')) + + +def test_lifetime_edits_do_not_rename_fields_or_labels_sharing_a_local_name(): + source = MERGE.replace('return value_2;', 'obj.value_2=9;return value_2;') + choices = [text for family, text in integer_lifetime_repairs.variants(source, 'f') if family == 'merge_integer_lifetimes'] + assert choices and all('obj.value_2=9;' in text for text in choices) + + +def test_branch_call_keeps_assignments_and_executes_one_copy_per_path(): + candidates = [text for family, text in branch_layout_repairs.variants(BRANCH, 'f') if family == 'branch_call'] + assert candidates and candidates[0].count('draw(x);') == 2 + assert 'x=3;' in candidates[0] and 'x=7;' in candidates[0] + assert not any(family == 'branch_call' for family, _ in branch_layout_repairs.variants(BRANCH.replace('x=3;', 'int x=3;'), 'f')) + assert not any(family == 'branch_call' for family, _ in branch_layout_repairs.variants(BRANCH.replace('x=3;', 'return;'), 'f')) + + +def test_goto_leaf_copies_its_terminal_transfer_and_rejects_fallthrough_or_shadowing(): + source = 'void f(int x){if(x)goto leaf;work();goto done;leaf:draw(x);goto done;done:return;}' + choices = [text for family, text in branch_layout_repairs.variants(source, 'f') if family == 'goto_leaf'] + assert choices and 'if(x){draw(x);goto done;}' in choices[0] + fallthrough = source.replace('draw(x);goto done;', 'draw(x);x++;goto done;') + assert not any(family == 'goto_leaf' for family, _ in branch_layout_repairs.variants(fallthrough, 'f')) + scoped = source.replace('leaf:draw(x);goto done;', '{int x;leaf:draw(x);goto done;}') + assert not any(family == 'goto_leaf' for family, _ in branch_layout_repairs.variants(scoped, 'f')) + + +def test_goto_leaf_can_exit_an_if_arm_with_the_same_enclosing_switch_break(): + source = 'void f(int mode,int flag){switch(mode){case 1:if(flag){leaf:draw(mode);}else{other();}break;case 2:goto leaf;}}' + choices = [text for family, text in branch_layout_repairs.variants(source, 'f') if family == 'goto_leaf'] + assert choices and 'case 2:{draw(mode);break;}' in choices[0] + nested_loop = source.replace('case 2:goto leaf;', 'case 2:while(flag){goto leaf;}') + assert not any(family == 'goto_leaf' for family, _ in branch_layout_repairs.variants(nested_loop, 'f')) + unbraced = source.replace('case 2:goto leaf;', 'case 2:while(flag)goto leaf;') + assert not any(family == 'goto_leaf' for family, _ in branch_layout_repairs.variants(unbraced, 'f')) + preceding_loop = source.replace('switch(mode)', 'do{work();}while(flag);switch(mode)') + assert any(family == 'goto_leaf' for family, _ in branch_layout_repairs.variants(preceding_loop, 'f')) + + +def test_switch_reordering_keeps_grouped_cases_and_adds_a_missing_noop_zero(): + source = 'void f(int x){switch(x){case 1:case 2:draw(3);break;case 5:draw(7);break;}}' + choices = [text for family, text in branch_layout_repairs.variants(source, 'f') if family == 'switch_layout'] + assert choices and any('case 0: break;' in text for text in choices) + assert all('case 1:case 2:draw(3);break;' in text and 'case 5:draw(7);break;' in text for text in choices) + default = source.replace('case 5:', 'default:') + assert not any('case 0: break;' in text for _, text in branch_layout_repairs.variants(default, 'f')) + goto = 'void f(int x){switch(x){case 1:leaf:draw(3);break;case 5:goto leaf;}}' + assert any('case 5:goto leaf;case 1:leaf:draw(3);break;' in text for _, text in branch_layout_repairs.variants(goto, 'f')) + + +@pytest.mark.parametrize('source', [ + 'void f(int x){switch(x){case 1:draw(3);case 5:draw(7);break;}}', + 'void f(int x){switch(x){case 1:if(x)break;case 5:draw(7);break;}}', + 'int f(int x){switch(x){case 1:if(x)return 3;case 5:return 7;}}', + 'void f(int x){switch(x){case 1:int y=3;draw(y);break;case 5:draw(7);break;}}', + 'void f(int x){switch(x){case VALUE:draw(3);break;case 5:draw(7);break;}}', +]) +def test_switch_reordering_rejects_fallthrough_conditional_terminals_declarations_and_symbolic_labels(source): + # A no-op zero insertion does not move the existing fallthrough/declarations and is independently safe. + assert not any(family == 'switch_layout' and 'case 0: break;' not in text + for family, text in branch_layout_repairs.variants(source, 'f')) + + +@pytest.mark.parametrize('arch', ['mips', 'ppc', 'x86', 'arm', 'thumb']) +def test_new_hypotheses_reach_shared_draft_and_manual_family_searches(arch): + examples = [('local_widening', WIDE), ('callee_parameter_width', PROTO), ('merge_integer_lifetimes', MERGE), ('branch_call', BRANCH)] + for expected, source in examples: + prefix = 'void other(void){int untouched;use(untouched);}\n' + labels = {} + choices = draft_repairs.variants(prefix+source, 'f', arch, labels=labels, limit=None) + assert any(labels[text] == expected for text in choices), (expected, labels) + assert all(text.startswith(prefix) for text in choices if labels[text] == expected) + assert expected in local_experiments.FAMILIES + + +def test_real_mips_compiler_confirms_widening_and_callee_repair_can_recover_exact_objects(tmp_path): + from decompnet import scoring, toolchains + from decompnet.compiler import compile_source + from decompnet.elfobj import extract_function + from decompnet.project import compiler_config + if not toolchains.installed('gcc-2.7.2-kmc'): + pytest.skip('needs pinned gcc-2.7.2-kmc') + cfg = compiler_config(tmp_path, {'compiler': {'toolchain': 'gcc-2.7.2-kmc'}}) + examples = [(WIDE, 'int f(int offset){return 320-offset;}', local_widening_repairs.variants(WIDE, 'f')), + (PROTO, 'int draw(int,int);int f(int y){return draw(7,y-2);}', [text for _, text in callee_prototype_repairs.variants(PROTO, 'f')])] + for original, target, candidates in examples: + expected = compile_source(cfg, '', target) + before = compile_source(cfg, '', original) + assert expected.ok and before.ok + obj = extract_function(expected.obj, 'f') + assert scoring.diff(obj, extract_function(before.obj, 'f')).score > 0 + scores = [] + for candidate in candidates: + compiled = compile_source(cfg, '', candidate) + assert compiled.ok, compiled.log + scores.append(scoring.diff(obj, extract_function(compiled.obj, 'f')).score) + assert 0 in scores + + +@pytest.mark.parametrize('mode', ['rom', 'fork']) +def test_manual_new_families_share_one_budget_revision_and_normal_compile_path(tmp_path, monkeypatch, mode): + from decompnet.app import agent_tokens + from tests.test_claims import _game + net, _ = _game(tmp_path, [('f', 10, 'open')]) + agent, _ = net.local_agent('a') + source = 'int draw(short);int f(int x){signed char coordinate;coordinate=320-x;return draw(coordinate);}' + with net.store.tx() as db: + db.execute('UPDATE projects SET config=?', (json.dumps({'kind': mode, 'compiler': {'toolchain': 'example', 'flags': []}}),)) + db.execute('UPDATE functions SET best_source=?', (source,)) + net.claim_row(agent, 'game', 'f') + fn = net._function('f', 'game') + net._record(fn, agent, 10, source, 1) + seen = [] + def compile(agent, function, text, slug): + seen.append(text) + net._record(fn, agent, 20, text, 1) + return 'score 20' + monkeypatch.setattr(net, 'compile_and_diff', compile) + _, revision = agent_tokens.candidate(net, agent, 'game', 'f') + result = local_experiments.run(net, agent, 'game', 'f', revision, + families=['local_widening', 'callee_parameter_width'], max_trials=2) + assert len(seen) == 2 and '1: local_widening, 2: callee_parameter_width' in result + assert 'starting candidate retained' in result + assert agent_tokens.candidate(net, agent, 'game', 'f')[0] == source