From 84c03fd01d85e589616b208b6759097e682f082b Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sun, 18 Oct 2026 10:46:58 -0300 Subject: [PATCH] Compute scope regions from native AST positions Scope.get_region() no longer patches the whole module AST through patchedast to borrow node regions. It now resolves each scope's region directly from the native AST positions via codeanalyze.ASTLinesAdapter, and extends it to the enclosing decorator by searching backwards for the "@" token with _Source.rfind_token. This keeps regions correct for sources the patched walker cannot round-trip: normalized Unicode identifiers (e.g. "u\u0301" aliases), decorated definitions, and raw string sources using LF, CRLF, or CR newlines. Modules resolve to the full source range, and nodes without complete end positions raise RopeError instead of silently producing partial regions. Note: the original internal change also fixed _Source tokenization for CR/CRLF sources; that code path was removed by the patchedast cleanup (#896), which already computes positions through ASTLinesAdapter, so only the pyscopes resolution is carried over. --- CHANGELOG.md | 1 + rope/base/pyscopes.py | 32 ++++++--- ropetest/pyscopestest.py | 141 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 164 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e7311ec..96dc856b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # **Upcoming release** +- Compute scope regions from AST positions without patching the whole module, preserving normalized Unicode identifiers, decorators, and LF, CRLF, and CR newlines - #895 patchedast Starred and keyword now consumes their syntactically expected and ** (@lieryan) - #896 patchedast cleanup and refactoring (@lieryan) diff --git a/rope/base/pyscopes.py b/rope/base/pyscopes.py index 565541903..7d0a90cb4 100644 --- a/rope/base/pyscopes.py +++ b/rope/base/pyscopes.py @@ -100,13 +100,22 @@ def get_kind(self): pass def get_region(self): - self._calculate_scope_regions_for_module() + global_scope = self._get_global_scope() node = self.pyobject.get_ast() - region = patchedast.node_region(node) - return region - - def _calculate_scope_regions_for_module(self): - self._get_global_scope()._calculate_scope_regions() + if isinstance(node, ast.Module): + return (0, len(global_scope._get_source())) + start, end = global_scope._ast_lines[node] + if start is None or end is None: + raise exceptions.RopeError("Scope AST has no complete source positions") + decorators = getattr(node, "decorator_list", ()) + if decorators: + decorator_start, _ = global_scope._ast_lines[decorators[0]] + if decorator_start is None: + raise exceptions.RopeError("Decorator AST has no source positions") + start = global_scope._region_source.rfind_token("@", 0, decorator_start) + if start is None: + raise exceptions.RopeError("Scope decorator has no source marker") + return (start, end) def in_region(self, offset): """Checks if offset is in scope region""" @@ -134,10 +143,15 @@ def get_name(self, name): return self.builtin_names[name] raise exceptions.NameNotFoundError("name %s not found" % name) + @property + @utils.saveit + def _ast_lines(self): + return codeanalyze.ASTLinesAdapter(self._get_source()) + + @property @utils.saveit - def _calculate_scope_regions(self): - source = self._get_source() - patchedast.patch_ast(self.pyobject.get_ast(), source) + def _region_source(self): + return patchedast._Source(self._get_source()) def _get_source(self): return self.pyobject.source_code diff --git a/ropetest/pyscopestest.py b/ropetest/pyscopestest.py index d60702b32..6c23588e9 100644 --- a/ropetest/pyscopestest.py +++ b/ropetest/pyscopestest.py @@ -1,8 +1,9 @@ import unittest from textwrap import dedent -from rope.base import libutils +from rope.base import exceptions, libutils from rope.base.pyobjects import get_base_type +from rope.refactor import patchedast from ropetest import testutils @@ -560,3 +561,141 @@ def func2(o): ) self.assertEqual(scope.get_scopes()[1].get_region(), (26, 47)) + + def test_scope_regions_with_normalized_import_alias(self): + alias = "u\u0301" + code = ( + f"from os import path as {alias}\n" + f"def outer({alias}):\n" + " def inner():\n" + f" marker = 'π'; return {alias}.sep\n" + " return inner\n" + ) + scope = libutils.get_string_scope(self.project, code) + outer = scope.get_scopes()[0] + inner = outer.get_scopes()[0] + + self.assertIs(scope, scope.get_inner_scope_for_offset(code.index(alias))) + self.assertIs( + inner, scope.get_inner_scope_for_offset(code.index(alias + ".sep")) + ) + self.assertIs(inner.lookup("ú"), outer.lookup("ú")) + self.assertIsNot(inner.lookup("ú"), scope.lookup("ú")) + + def test_decorated_scope_regions_preserve_raw_string_newlines(self): + for newline in ("\n", "\r\n", "\r"): + for definition in ( + "def sample(value):\n marker = 'π'; return value\n", + "async def sample(value):\n marker = 'π'; return value\n", + "class Sample:\n marker = 'π'; value = 1\n", + ): + with self.subTest(newline=newline, definition=definition): + code = ( + "from os import path as u\u0301\n" + "@(\n" + " # @ is not the decorator marker\n" + " decorate\n" + ")\n" + "@other\n" + + definition + ).replace("\n", newline) + module = libutils.get_string_module(self.project, code) + self.assertEqual(code, module.source_code) + scope = module.get_scope() + child = scope.get_scopes()[0] + expected = (code.index("@("), len(code.rstrip("\r\n"))) + + self.assertEqual((0, len(code)), scope.get_region()) + self.assertEqual(expected, child.get_region()) + self.assertIs( + child, + scope.get_inner_scope_for_offset(code.index("decorate") + 1), + ) + self.assertFalse(child.in_region(expected[0])) + self.assertTrue(child.in_region(expected[0] + 1)) + self.assertTrue(child.in_region(expected[1] - 1)) + self.assertFalse(child.in_region(expected[1])) + + def test_scope_regions_use_the_decoded_resource_source(self): + for index, newline in enumerate(("\n", "\r\n", "\r")): + with self.subTest(newline=newline): + resource = testutils.create_module(self.project, "regions_%s" % index) + resource.write( + "@(\n decorate\n)\ndef sample():\n return 'π'\n".replace( + "\n", newline + ) + ) + module = self.project.get_pymodule(resource) + source = module.source_code + self.assertNotIn("\r", source) + scope = module.get_scope() + + self.assertEqual((0, len(source)), scope.get_region()) + self.assertEqual( + (source.index("@("), len(source.rstrip("\n"))), + scope.get_scopes()[0].get_region(), + ) + + def test_scope_regions_with_normalized_alias_in_comprehensions(self): + alias = "u\u0301" + expressions = ( + f"[({alias}, item) for item in values]", + f"{{({alias}, item) for item in values}}", + f"{{item: {alias} for item in values}}", + f"(({alias}, item) for item in values)", + ) + for expression in expressions: + with self.subTest(expression=expression): + code = ( + f"from os import path as {alias}\n" + "def sample(values):\n" + f" result = {expression}\n" + " return result\n" + ) + scope = libutils.get_string_scope(self.project, code) + function = scope.get_scopes()[0] + comprehension = function.get_scopes()[0] + + self.assertIs( + comprehension, + scope.get_inner_scope_for_offset(code.rindex(alias)), + ) + self.assertIs(comprehension.lookup("ú"), scope.lookup("ú")) + + def test_scope_region_does_not_partially_patch_the_module_ast(self): + code = "def sample(value):\n return value\n" + module = libutils.get_string_module(self.project, code) + module.get_scope().get_scopes()[0].get_region() + patched = patchedast.patch_ast(module.get_ast(), code, sorted_children=True) + + self.assertEqual(code, patchedast.write_ast(patched)) + + def test_scope_region_rejects_incomplete_source_positions(self): + scope = libutils.get_string_scope(self.project, "def sample(): pass\n") + child = scope.get_scopes()[0] + child.pyobject.get_ast().end_lineno = None + + with self.assertRaisesRegex(exceptions.RopeError, "complete source positions"): + child.get_region() + + def test_scope_decorator_marker_uses_python_line_boundaries(self): + for prefix in ("\f\n", "# leading\vcomment\n"): + with self.subTest(prefix=prefix): + code = ( + prefix + + "@(\n" + " # @ is not the decorator marker\n" + " decorate\n" + ")\n" + "def sample():\n" + " pass\n" + ) + scope = libutils.get_string_scope(self.project, code) + child = scope.get_scopes()[0] + + self.assertEqual( + (code.index("@("), len(code.rstrip("\n"))), child.get_region() + ) + self.assertIs( + child, scope.get_inner_scope_for_offset(code.index("decorate")) + )