From 6666928ab7014d8fb0c9d2113f8660804e4d002a Mon Sep 17 00:00:00 2001 From: Junaid Date: Tue, 13 Oct 2026 22:40:35 +0530 Subject: [PATCH] Keep normalized whitespace in sync after filtering --- sqlparse/filters/others.py | 1 + tests/test_format.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 95bc436c..3cb009ea 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -94,6 +94,7 @@ def _stripws_default(tlist): for token in tlist.tokens: if token.is_whitespace: token.value = '' if last_was_ws or is_first_char else ' ' + token.normalized = token.value last_was_ws = token.is_whitespace is_first_char = False diff --git a/tests/test_format.py b/tests/test_format.py index 93495067..0ced0a01 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -150,6 +150,24 @@ def test_strip_ws_invalid_option(self): with pytest.raises(SQLParseError): sqlparse.format(s, strip_whitespace=None) + @pytest.mark.parametrize('text', [ + '\t select\n\t* from foo', + 'select (a\t +\n b)', + 'select a\t,\n b from foo', + ]) + def test_strip_ws_updates_normalized_value(self, text): + # Regression for issue547: match() must see the filtered whitespace. + from sqlparse.filters import StripWhitespaceFilter + + stmt = sqlparse.parse(text)[0] + StripWhitespaceFilter().process(stmt) + for token in stmt.flatten(): + if token.is_whitespace: + assert token.normalized == token.value + assert token.match(token.ttype, token.value) + pattern = '^ $' if token.value else '^$' + assert token.match(token.ttype, pattern, regex=True) + def test_preserve_ws(self): # preserve at least one whitespace after subgroups f = lambda sql: sqlparse.format(sql, strip_whitespace=True)