From 0a1680368f8e77575ece1898b94e9942f39f011e Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 30 Oct 2026 21:18:34 +0100 Subject: [PATCH 1/3] Lex PHP attributes natively so fixers format inside #[...] The lexer used to fold a whole #[...] attribute into one opaque Comment token, so no fixer could see the arrays, named arguments or operators inside it. That left attribute-internal code diverging from PHP ECS (e.g. concat spacing in #[WithEnv('X', __DIR__.'/y')]). Attributes now lex natively: "#[" is an AttributeStart token, the closing "]" is retagged as AttributeEnd, and the contents lex as ordinary tokens, mirroring php-cs-fixer's T_ATTRIBUTE / T_ATTRIBUTE_CLOSE. Both delimiters are distinct kinds, so generic bracket-depth scans never mistake them for array brackets. Fixers that reasoned about an attribute as a whole were ported to the native tokens: a shared helper set (AttributeEnd/AttributeStartOf, attributeStartsAt/EndsAt, prevMeaningfulSkippingAttributes) plus a text bridge that re-lexes the existing string reflows (symfony standalone line, line-length break, arg collapse, anon-class flatten). The old string hacks on the comment value are gone. --- blink/internal/fixer/rules/attribute.go | 70 ++++++++++++++++++ blink/internal/fixer/rules/blocks.go | 8 ++ blink/internal/fixer/rules/braces_position.go | 4 +- .../internal/fixer/rules/class_definition.go | 22 ++++-- blink/internal/fixer/rules/constructs.go | 2 +- blink/internal/fixer/rules/gen3_class_sep.go | 24 +++--- blink/internal/fixer/rules/line_length.go | 22 +++--- .../fixer/rules/method_argument_space.go | 74 ++++++++++++------- .../fixer/rules/no_superfluous_phpdoc_tags.go | 10 ++- blink/internal/fixer/rules/phpunit_fixers.go | 16 ++-- .../rules/standalone_line_required_param.go | 16 ++-- ...standalone_line_symfony_attribute_param.go | 20 +++-- .../fixer/rules/statement_indentation.go | 4 +- blink/internal/fixer/rules/token_batch.go | 37 ++++++++-- blink/internal/lexer/lexer.go | 73 ++++++++---------- blink/internal/lexer/lexer_test.go | 54 ++++++++++++-- blink/internal/token/token.go | 36 +++++---- blink/internal/tokens/stream.go | 29 ++++++++ 18 files changed, 368 insertions(+), 153 deletions(-) create mode 100644 blink/internal/fixer/rules/attribute.go diff --git a/blink/internal/fixer/rules/attribute.go b/blink/internal/fixer/rules/attribute.go new file mode 100644 index 0000000000..5935f1b1db --- /dev/null +++ b/blink/internal/fixer/rules/attribute.go @@ -0,0 +1,70 @@ +package rules + +import ( + "strings" + + "blink/internal/lexer" + "blink/internal/token" + "blink/internal/tokens" +) + +// Attributes lex natively ("#[" is an AttributeStart token, its contents and the +// closing "]" are ordinary tokens), so most fixers see inside them for free. The +// helpers here cover the few rules that still reason about an attribute as a whole. + +// attributeStartsAt reports whether an attribute ("#[") begins at i. +func attributeStartsAt(s *tokens.Stream, i int) bool { + return i >= 0 && i < s.Len() && s.At(i).Kind == token.AttributeStart +} + +// attributeEndsAt reports whether the attribute-closing "]" is at i. +func attributeEndsAt(s *tokens.Stream, i int) bool { + return s.AttributeStartOf(i) >= 0 +} + +// prevMeaningfulSkippingAttributes is prevMeaningfulIndex but also steps back over +// any attributes ("new #[A] #[B] class" -> the "new"), since an attribute decorates +// the token that follows it. +func prevMeaningfulSkippingAttributes(s *tokens.Stream, i int) int { + p := prevMeaningfulIndex(s, i) + for p >= 0 && attributeEndsAt(s, p) { + p = prevMeaningfulIndex(s, s.AttributeStartOf(p)) + } + return p +} + +// attributeText renders the attribute tokens [start..end] back to source text. +func attributeText(s *tokens.Stream, start, end int) string { + var b strings.Builder + for j := start; j <= end; j++ { + b.WriteString(s.At(j).Value) + } + return b.String() +} + +// rewriteAttributeText feeds the attribute [start..end] to transform as its source +// text ("#[...]"), re-lexes the result and splices it back in place. It bridges the +// text-shaped attribute transforms to the native token stream. Returns whether the +// stream changed. +func rewriteAttributeText(s *tokens.Stream, start, end int, transform func(string) (string, bool)) bool { + old := attributeText(s, start, end) + nv, ok := transform(old) + if !ok || nv == old { + return false + } + s.ReplaceRange(start, end, lexAttributeFragment(nv)) + return true +} + +// lexAttributeFragment lexes a standalone "#[...]" fragment into its tokens, from +// the AttributeStart through the end (the opening PHP tag used to drive the lexer +// is dropped). +func lexAttributeFragment(text string) []token.Token { + toks := lexer.Lex(" k { + k = end + } + continue + } if expect { starts = append(starts, k) expect = false diff --git a/blink/internal/fixer/rules/braces_position.go b/blink/internal/fixer/rules/braces_position.go index 7df23a2e76..a758559625 100644 --- a/blink/internal/fixer/rules/braces_position.go +++ b/blink/internal/fixer/rules/braces_position.go @@ -86,9 +86,9 @@ func (f BracesPosition) Fix(s *tokens.Stream) bool { case braceClassLike: category = "classes" // an anonymous class ("new class ... {", also "new #[Attr] class ... {") - // is configured separately; prevMeaningfulIndex steps over the attribute + // is configured separately; step back over any leading attribute if kw >= 0 { - if p := prevMeaningfulIndex(s, kw); p >= 0 { + if p := prevMeaningfulSkippingAttributes(s, kw); p >= 0 { if prev := s.At(p); prev.Kind == token.Keyword && strings.EqualFold(prev.Value, "new") { category = "anon_classes" } diff --git a/blink/internal/fixer/rules/class_definition.go b/blink/internal/fixer/rules/class_definition.go index d8738481fd..bd6db910c1 100644 --- a/blink/internal/fixer/rules/class_definition.go +++ b/blink/internal/fixer/rules/class_definition.go @@ -102,7 +102,7 @@ func (f ClassDefinition) Fix(s *tokens.Stream) bool { // anonymous class whose keyword is at classIdx, or -1 for a named declaration. // prevMeaningfulIndex steps over a leading attribute ("new #[Attr] class"). func anonClassNewIndex(s *tokens.Stream, classIdx int) int { - p := prevMeaningfulIndex(s, classIdx) + p := prevMeaningfulSkippingAttributes(s, classIdx) if p >= 0 && s.At(p).Kind == token.Keyword && strings.EqualFold(s.At(p).Value, "new") { return p } @@ -115,14 +115,22 @@ func anonClassNewIndex(s *tokens.Stream, classIdx int) int { func classDefFlattenAnonHeaderAttributes(s *tokens.Stream, newIdx, classIdx int) bool { changed := false for j := newIdx + 1; j < classIdx; j++ { - t := s.At(j) - if !isAttributeComment(t) || !strings.ContainsRune(t.Value, '\n') { + if !attributeStartsAt(s, j) { + continue + } + end := s.AttributeEnd(j) + if end < 0 { continue } - if nv, ok := collapseAttributeText(t.Value, false); ok && nv != t.Value { - s.SetValue(j, nv) + before := s.Len() + if strings.ContainsRune(attributeText(s, j, end), '\n') && + rewriteAttributeText(s, j, end, func(v string) (string, bool) { + return collapseAttributeText(v, false) + }) { changed = true + classIdx -= before - s.Len() // the splice shifted the "class" keyword } + j = s.AttributeEnd(j) // skip past the attribute we just handled } return changed } @@ -169,10 +177,10 @@ func (f ClassDefinition) classDefFlattenHeader(s *tokens.Stream, kw int) bool { if t.Kind != token.Whitespace || !hasNewline(t.Value) { continue } - if j-1 >= 0 && (s.At(j-1).Kind == token.DocComment || isAttribute(s.At(j-1))) { + if j-1 >= 0 && (s.At(j-1).Kind == token.DocComment || attributeEndsAt(s, j-1)) { continue } - if j+1 < s.Len() && (s.At(j+1).Kind == token.DocComment || isAttribute(s.At(j+1))) { + if j+1 < s.Len() && (s.At(j+1).Kind == token.DocComment || attributeStartsAt(s, j+1)) { continue } s.SetValue(j, " ") diff --git a/blink/internal/fixer/rules/constructs.go b/blink/internal/fixer/rules/constructs.go index 7d17a7c950..ec9b3a66a4 100644 --- a/blink/internal/fixer/rules/constructs.go +++ b/blink/internal/fixer/rules/constructs.go @@ -187,7 +187,7 @@ func (f SingleSpaceAroundConstruct) fixNamedArgument(s *tokens.Stream) bool { if next.Kind == token.Whitespace { // a newline into an attribute or doc comment is kept, as php-cs-fixer does if hasNewline(next.Value) { - if nn := i + 2; nn < s.Len() && (isAttributeComment(s.At(nn)) || s.At(nn).Kind == token.DocComment) { + if nn := i + 2; nn < s.Len() && (attributeStartsAt(s, nn) || s.At(nn).Kind == token.DocComment) { continue } } diff --git a/blink/internal/fixer/rules/gen3_class_sep.go b/blink/internal/fixer/rules/gen3_class_sep.go index 504bce8dfc..0cd1026050 100644 --- a/blink/internal/fixer/rules/gen3_class_sep.go +++ b/blink/internal/fixer/rules/gen3_class_sep.go @@ -145,13 +145,22 @@ func (f ClassAttributesSeparation) fixMemberGap(s *tokens.Stream, prevStart, nex wsCount++ wsIdx = j case token.DocComment, token.Comment: - // a docblock, attribute (#[...]) or a line/block comment leading the next - // member is its trivia: the blank line goes above the block. A trailing - // comment on the previous member's line is rejected by the guards below - // (its whitespace has no newline), so it is still left alone. + // a docblock or a line/block comment leading the next member is its + // trivia: the blank line goes above the block. A trailing comment on the + // previous member's line is rejected by the guards below (its whitespace + // has no newline), so it is still left alone. if firstTrivia < 0 { firstTrivia = j } + case token.AttributeStart: + // an attribute (#[...]) leading the next member is trivia too; skip its + // inner tokens so the blank line goes above the whole attribute block + if firstTrivia < 0 { + firstTrivia = j + } + if end := s.AttributeEnd(j); end > j { + j = end + } default: return false } @@ -258,12 +267,7 @@ func hasDocOrAttrAbove(s *tokens.Stream, start int) bool { if j < 0 { return false } - t := s.At(j) - return t.Kind == token.DocComment || isAttribute(t) -} - -func isAttribute(t token.Token) bool { - return t.Kind == token.Comment && strings.HasPrefix(t.Value, "#[") + return s.At(j).Kind == token.DocComment || s.AttributeStartOf(j) >= 0 } // setNewlineCount rewrites the whitespace token at idx to hold exactly req diff --git a/blink/internal/fixer/rules/line_length.go b/blink/internal/fixer/rules/line_length.go index db4c2ab875..93d1c054fc 100644 --- a/blink/internal/fixer/rules/line_length.go +++ b/blink/internal/fixer/rules/line_length.go @@ -393,13 +393,17 @@ func (f LineLength) inlineItems(s *tokens.Stream, open, closeIdx int) bool { return changed } -// processAttribute expands a too-long "#[Attr(...)]" (one comment token) onto -// multiple lines by rewriting its text. +// processAttribute expands a too-long "#[Attr(...)]" onto multiple lines by +// rewriting the attribute's text and re-lexing it back into the stream. func (f LineLength) processAttribute(s *tokens.Stream, pos int) bool { if !f.breakLongLines { return false } - v := s.At(pos).Value + end := s.AttributeEnd(pos) + if end < 0 { + return false + } + v := attributeText(s, pos, end) if !strings.Contains(v, "(") { return false } @@ -411,12 +415,10 @@ func (f LineLength) processAttribute(s *tokens.Stream, pos int) bool { return false } base := lineIndent(s, pos) - nv := llExpandAttribute(v, base) - if nv == "" || nv == v { - return false - } - s.SetValue(pos, nv) - return true + return rewriteAttributeText(s, pos, end, func(v string) (string, bool) { + nv := llExpandAttribute(v, base) + return nv, nv != "" + }) } // attributeColumn is the rendered column at which the attribute token starts. @@ -582,7 +584,7 @@ func llHasPromotedProperty(s *tokens.Stream, open, closeIdx int) bool { } func llIsAttribute(t token.Token) bool { - return t.Kind == token.Comment && strings.HasPrefix(t.Value, "#[") + return t.Kind == token.AttributeStart } func llIsHeredoc(t token.Token) bool { diff --git a/blink/internal/fixer/rules/method_argument_space.go b/blink/internal/fixer/rules/method_argument_space.go index 023e9c5678..3513201c52 100644 --- a/blink/internal/fixer/rules/method_argument_space.go +++ b/blink/internal/fixer/rules/method_argument_space.go @@ -295,6 +295,12 @@ func countTopLevelArgs(s *tokens.Stream, open, closeIdx int) int { count := 1 depth := 0 for j := open + 1; j < closeIdx; j++ { + if attributeStartsAt(s, j) { + if end := s.AttributeEnd(j); end > j { + j = end + } + continue + } t := s.At(j) if t.Kind != token.Punct { continue @@ -319,7 +325,7 @@ func countTopLevelArgs(s *tokens.Stream, open, closeIdx int) int { func argumentContentIsMultiline(s *tokens.Stream, open, closeIdx int) bool { for i := open + 1; i < closeIdx; i++ { t := s.At(i) - if (t.Kind == token.Comment || t.Kind == token.DocComment) && !isAttributeComment(t) { + if t.Kind == token.Comment || t.Kind == token.DocComment { return true } if i == open+1 || i == closeIdx-1 { @@ -429,26 +435,25 @@ func isGroupingReflowParen(s *tokens.Stream, open int) bool { return t.Kind == token.Punct && (t.Value == "!" || t.Value == "&&" || t.Value == "||") } -// isAttributeComment reports whether t is a "#[...]" attribute (the lexer keeps -// it as one comment token), as opposed to a "//" or "#" line comment. -func isAttributeComment(t token.Token) bool { - return t.Kind == token.Comment && strings.HasPrefix(t.Value, "#[") -} - // collapseAttributeArgs collapses a multiline argument list inside an attribute -// (e.g. "#[Attr(\n 'foo'\n)]" -> "#[Attr('foo')]") on the attribute's token -// text. With singleArgOnly it only collapses a call that has a single argument, -// matching on_multiline=ensure_single_line_for_single_argument. +// (e.g. "#[Attr(\n 'foo'\n)]" -> "#[Attr('foo')]") on the attribute's text, then +// re-lexes it. With singleArgOnly it only collapses a call that has a single +// argument, matching on_multiline=ensure_single_line_for_single_argument. func collapseAttributeArgs(s *tokens.Stream, singleArgOnly bool) bool { changed := false for i := 0; i < s.Len(); i++ { - t := s.At(i) - if !isAttributeComment(t) || !strings.ContainsRune(t.Value, '\n') { + if !attributeStartsAt(s, i) { continue } - if nv, ok := collapseAttributeText(t.Value, singleArgOnly); ok && nv != t.Value { - s.SetValue(i, nv) + end := s.AttributeEnd(i) + if end < 0 || !strings.ContainsRune(attributeText(s, i, end), '\n') { + continue + } + if rewriteAttributeText(s, i, end, func(v string) (string, bool) { + return collapseAttributeText(v, singleArgOnly) + }) { changed = true + i = s.AttributeEnd(i) } } return changed @@ -508,17 +513,19 @@ func placeAttributesInParen(s *tokens.Stream, open, closeIdx int, placement stri var attrs []int depth := 0 for j := open + 1; j < closeIdx; j++ { - t := s.At(j) - if isAttributeComment(t) { + if attributeStartsAt(s, j) { if depth == 0 { attrs = append(attrs, j) } + if end := s.AttributeEnd(j); end > j { + j = end + } continue } - if t.Kind != token.Punct { + if s.At(j).Kind != token.Punct { continue } - switch t.Value { + switch s.At(j).Value { case "(", "[", "{": depth++ case ")", "]", "}": @@ -533,7 +540,8 @@ func placeAttributesInParen(s *tokens.Stream, open, closeIdx int, placement stri } else { want = "\n" + lineIndentBefore(s, a) } - if editSlotAfter(s, a, want) { + // the whitespace to normalize sits after the attribute's closing "]" + if editSlotAfter(s, s.AttributeEnd(a), want) { changed = true } } @@ -550,6 +558,12 @@ func reflowParen(s *tokens.Stream, open, closeIdx int, collapseBlanks bool) bool var commas []int depth := 0 for j := open + 1; j < closeIdx; j++ { + if attributeStartsAt(s, j) { + if end := s.AttributeEnd(j); end > j { + j = end // an attribute's inner commas are not argument separators + } + continue + } t := s.At(j) if t.Kind != token.Punct { continue @@ -605,7 +619,7 @@ func reflowBreakAfterComments(s *tokens.Stream, open, closeIdx int, base string) continue } if depth != 0 || (t.Kind != token.Comment && t.Kind != token.DocComment) || - isLineComment(t) || isAttributeComment(t) { + isLineComment(t) { continue // attributes are placed by applyAttributePlacement, not here } // the comment must start its own line (preceded by a newline) to count as a @@ -645,7 +659,7 @@ func reflowAfterComma(s *tokens.Stream, comma int, base string, collapseBlanks b ws = n n++ } - if n < s.Len() && !isAttributeComment(s.At(n)) && + if n < s.Len() && (isLineComment(s.At(n)) || s.At(n).Kind == token.Comment || s.At(n).Kind == token.DocComment) { changed := false // only method_argument_space normalizes the comma-to-comment gap to one @@ -686,11 +700,18 @@ func argListIsMultiline(s *tokens.Stream, open, closeIdx int) bool { } depth := 0 for j := open + 1; j < closeIdx; j++ { - t := s.At(j) - // a top-level attribute that itself spans lines makes the list multiline - if depth == 0 && isAttributeComment(t) && hasNewline(t.Value) { - return true + if attributeStartsAt(s, j) { + end := s.AttributeEnd(j) + if end > j { + // a top-level attribute that itself spans lines makes the list multiline + if depth == 0 && strings.ContainsRune(attributeText(s, j, end), '\n') { + return true + } + j = end + continue + } } + t := s.At(j) if t.Kind != token.Punct { continue } @@ -770,8 +791,7 @@ func argNLAfterComma(s *tokens.Stream, comma int, base string, collapseBlanks bo if newlines := strings.Count(ws.Value, "\n"); newlines >= 2 { keep := !collapseBlanks if collapseBlanks { - if nx := nextSignificantIndex(s, comma); nx >= 0 && - isComment(s.At(nx)) && !isAttributeComment(s.At(nx)) { + if nx := nextSignificantIndex(s, comma); nx >= 0 && isComment(s.At(nx)) { keep = true } } diff --git a/blink/internal/fixer/rules/no_superfluous_phpdoc_tags.go b/blink/internal/fixer/rules/no_superfluous_phpdoc_tags.go index c155dc3be1..7edeed8975 100644 --- a/blink/internal/fixer/rules/no_superfluous_phpdoc_tags.go +++ b/blink/internal/fixer/rules/no_superfluous_phpdoc_tags.go @@ -283,8 +283,8 @@ type funcSig struct { // signatureAfter finds the function declaration a docblock at index i documents // and extracts its parameter and return types. func signatureAfter(s *tokens.Stream, doc int) (funcSig, bool) { - // Walk forward across whitespace, attributes (#[...] Comment) and modifier - // keywords to the "function" keyword. + // Walk forward across whitespace, attributes (#[...]) and modifier keywords to + // the "function" keyword. sawModifier := false j := doc + 1 for j < s.Len() { @@ -293,6 +293,12 @@ func signatureAfter(s *tokens.Stream, doc int) (funcSig, bool) { case token.Whitespace, token.Comment: j++ continue + case token.AttributeStart: + if end := s.AttributeEnd(j); end > j { + j = end + 1 + continue + } + return funcSig{}, false case token.Keyword: lv := strings.ToLower(t.Value) if lv == "function" { diff --git a/blink/internal/fixer/rules/phpunit_fixers.go b/blink/internal/fixer/rules/phpunit_fixers.go index 88d3bf5dd0..c85c541a86 100644 --- a/blink/internal/fixer/rules/phpunit_fixers.go +++ b/blink/internal/fixer/rules/phpunit_fixers.go @@ -109,7 +109,11 @@ func puDocBlockIndex(s *tokens.Stream, index int) int { return idx } t := s.At(p) - if t.Kind == token.Comment { // includes folded #[...] attributes + if t.Kind == token.AttributeEnd { + idx = s.AttributeStartOf(p) + continue + } + if t.Kind == token.Comment { idx = p continue } @@ -266,17 +270,17 @@ func isTestMethod(s *tokens.Stream, index int) bool { return docIdx >= 0 && s.At(docIdx).Kind == token.DocComment && strings.Contains(s.At(docIdx).Value, "@test") } -// isTestAttributePresent reports whether the method carries a #[Test] attribute, -// which the lexer folds into a single comment token starting with "#[". +// isTestAttributePresent reports whether the method carries a #[Test] attribute. func isTestAttributePresent(s *tokens.Stream, index int) bool { p := getPrevNonWhitespace(s, index) for p >= 0 { t := s.At(p) - if t.Kind == token.Comment { - if strings.HasPrefix(t.Value, "#[") && phpUnitTestAttrRe(t.Value) { + if t.Kind == token.AttributeEnd { + start := s.AttributeStartOf(p) + if phpUnitTestAttrRe(attributeText(s, start, p)) { return true } - p = getPrevNonWhitespace(s, p) + p = getPrevNonWhitespace(s, start) continue } if t.Kind == token.Keyword && puModifierKw[strings.ToLower(t.Value)] { diff --git a/blink/internal/fixer/rules/standalone_line_required_param.go b/blink/internal/fixer/rules/standalone_line_required_param.go index c00b00c32c..bdb5963af4 100644 --- a/blink/internal/fixer/rules/standalone_line_required_param.go +++ b/blink/internal/fixer/rules/standalone_line_required_param.go @@ -17,8 +17,7 @@ type StandaloneLineRequiredParam struct{} var ( requiredAnnotationRe = regexp.MustCompile(`(?i)@required\b`) - // the lexer folds a #[Required] attribute into a single "#..." comment token - requiredAttributeRe = regexp.MustCompile(`(?i)^#\[.*\bRequired\b`) + requiredAttributeRe = regexp.MustCompile(`(?i)^#\[.*\bRequired\b`) ) func (StandaloneLineRequiredParam) Name() string { @@ -65,13 +64,16 @@ func isPublicRequiredMethod(s *tokens.Stream, fnPos int) bool { isRequired := false for j := fnPos - 1; j >= 0; j-- { t := s.At(j) - switch t.Kind { - case token.Whitespace: - continue - case token.Comment: - if requiredAttributeRe.MatchString(t.Value) { + if attributeEndsAt(s, j) { + start := s.AttributeStartOf(j) + if requiredAttributeRe.MatchString(attributeText(s, start, j)) { isRequired = true } + j = start + continue + } + switch t.Kind { + case token.Whitespace, token.Comment: continue case token.DocComment: if requiredAnnotationRe.MatchString(t.Value) { diff --git a/blink/internal/fixer/rules/standalone_line_symfony_attribute_param.go b/blink/internal/fixer/rules/standalone_line_symfony_attribute_param.go index 5f21c82f99..40aa76a640 100644 --- a/blink/internal/fixer/rules/standalone_line_symfony_attribute_param.go +++ b/blink/internal/fixer/rules/standalone_line_symfony_attribute_param.go @@ -3,7 +3,6 @@ package rules import ( "strings" - "blink/internal/token" "blink/internal/tokens" ) @@ -11,8 +10,8 @@ import ( // // StandaloneLineSymfonyAttributeParam puts each argument of an allow-listed // Symfony attribute on its own line. #[AsCommand] always breaks; every other -// listed attribute only when it has 2 or more arguments. The lexer keeps an -// attribute as a single comment token, so the reformat happens on that string. +// listed attribute only when it has 2 or more arguments. The reflow runs on the +// attribute's source text, which is then re-lexed back into the stream. type StandaloneLineSymfonyAttributeParam struct{} const alwaysBreakAttributeShortName = "AsCommand" @@ -48,14 +47,19 @@ func (StandaloneLineSymfonyAttributeParam) SourceURL() string { func (StandaloneLineSymfonyAttributeParam) Fix(s *tokens.Stream) bool { changed := false for i := 0; i < s.Len(); i++ { - t := s.At(i) - if t.Kind != token.Comment || !strings.HasPrefix(t.Value, "#[") { + if !attributeStartsAt(s, i) { continue } - reflowed, ok := reflowSymfonyAttribute(t.Value, lineIndentBefore(s, i)) - if ok && reflowed != t.Value { - s.SetValue(i, reflowed) + end := s.AttributeEnd(i) + if end < 0 { + continue + } + base := lineIndentBefore(s, i) + if rewriteAttributeText(s, i, end, func(v string) (string, bool) { + return reflowSymfonyAttribute(v, base) + }) { changed = true + i = s.AttributeEnd(i) } } return changed diff --git a/blink/internal/fixer/rules/statement_indentation.go b/blink/internal/fixer/rules/statement_indentation.go index a03fb8ed7b..25db2529ea 100644 --- a/blink/internal/fixer/rules/statement_indentation.go +++ b/blink/internal/fixer/rules/statement_indentation.go @@ -507,7 +507,7 @@ func stmtIsComment(s *tokens.Stream, i int) bool { if t.Kind == token.DocComment { return true } - return t.Kind == token.Comment && !isAttributeComment(t) + return t.Kind == token.Comment } // stmtIsPlainComment reports whether token i is a non-doc line/block comment @@ -517,7 +517,7 @@ func stmtIsPlainComment(s *tokens.Stream, i int) bool { return false } t := s.At(i) - return t.Kind == token.Comment && !isAttributeComment(t) + return t.Kind == token.Comment } // stmtNextValue returns the index of the next token equal to value, or -1. diff --git a/blink/internal/fixer/rules/token_batch.go b/blink/internal/fixer/rules/token_batch.go index d3c8cc0b55..9d3614a5b7 100644 --- a/blink/internal/fixer/rules/token_batch.go +++ b/blink/internal/fixer/rules/token_batch.go @@ -45,21 +45,44 @@ func (AttributeBlockNoSpaces) SourceURL() string { func (AttributeBlockNoSpaces) Fix(s *tokens.Stream) bool { changed := false for i := 0; i < s.Len(); i++ { - t := s.At(i) - if t.Kind != token.Comment || !strings.HasPrefix(t.Value, "#[") || !strings.HasSuffix(t.Value, "]") { + if !attributeStartsAt(s, i) { continue } - inner := t.Value[2 : len(t.Value)-1] - trimmed := strings.TrimRight(strings.TrimLeft(inner, " \t"), " \t") - if trimmed == inner { + end := s.AttributeEnd(i) + if end < 0 { continue } - s.SetValue(i, "#["+trimmed+"]") - changed = true + // trailing space before "]" first (higher index), so its removal never shifts + // the leading token we trim next; only same-line spaces/tabs are stripped + if j := end - 1; j > i && s.At(j).Kind == token.Whitespace { + if trimAttributeEdgeSpace(s, j, strings.TrimRight) { + changed = true + } + } + if j := i + 1; j < s.Len() && s.At(j).Kind == token.Whitespace { + if trimAttributeEdgeSpace(s, j, strings.TrimLeft) { + changed = true + } + } } return changed } +// trimAttributeEdgeSpace strips same-line spaces/tabs from the whitespace token at +// j using trim (TrimLeft or TrimRight), removing it when nothing is left. +func trimAttributeEdgeSpace(s *tokens.Stream, j int, trim func(string, string) string) bool { + nv := trim(s.At(j).Value, " \t") + if nv == s.At(j).Value { + return false + } + if nv == "" { + s.RemoveAt(j) + } else { + s.SetValue(j, nv) + } + return true +} + // PHP-CS-Fixer: https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/blob/master/src/Fixer/Operator/NoSpaceAroundDoubleColonFixer.php // // NoSpaceAroundDoubleColon removes single-line whitespace around a "::" operator diff --git a/blink/internal/lexer/lexer.go b/blink/internal/lexer/lexer.go index 46fd0693a1..8cc62b415e 100644 --- a/blink/internal/lexer/lexer.go +++ b/blink/internal/lexer/lexer.go @@ -21,9 +21,37 @@ func Lex(src string) []token.Token { // most tokens are a few bytes; presize to avoid repeated slice regrowth l := &lexer{src: src, toks: make([]token.Token, 0, len(src)/3+8)} l.run() + markAttributeCloses(l.toks) return l.toks } +// markAttributeCloses retags the "]" that closes each "#[" attribute as an +// AttributeEnd token, so generic bracket-depth scans treat neither attribute +// delimiter as an array bracket (mirroring php-cs-fixer's T_ATTRIBUTE_CLOSE). +func markAttributeCloses(toks []token.Token) { + for i := range toks { + if toks[i].Kind != token.AttributeStart { + continue + } + depth := 1 + for j := i + 1; j < len(toks); j++ { + if toks[j].Kind != token.Punct { + continue + } + switch toks[j].Value { + case "[": + depth++ + case "]": + depth-- + if depth == 0 { + toks[j].Kind = token.AttributeEnd + j = len(toks) + } + } + } + } +} + type lexer struct { src string pos int @@ -341,49 +369,12 @@ func (l *lexer) hasPrefix(s string) bool { return strings.HasPrefix(l.src[l.pos:], s) } -// lexAttribute consumes a "#[ ... ]" attribute as one opaque token, matching -// brackets across lines and skipping string and heredoc/nowdoc content inside -// (so a multiline nowdoc in an attribute never leaks out as code). +// lexAttribute emits the "#[" that opens an attribute as one token; its contents +// (and the closing "]") then lex as ordinary PHP tokens, so fixers see the arrays, +// named arguments and operators inside, matching php-cs-fixer's T_ATTRIBUTE. func (l *lexer) lexAttribute(start int) { l.pos += 2 // consume "#[" - depth := 1 - for l.pos < len(l.src) && depth > 0 { - switch c := l.src[l.pos]; { - case c == '[': - depth++ - l.pos++ - case c == ']': - depth-- - l.pos++ - case c == '\'' || c == '"': - l.skipString(c) - case l.hasPrefix("<<<"): - if end := l.scanHeredoc(); end > l.pos { - l.pos = end - } else { - l.pos++ - } - default: - l.pos++ - } - } - l.emit(token.Comment, start) -} - -// skipString advances past a single- or double-quoted string, honoring escapes. -func (l *lexer) skipString(quote byte) { - l.pos++ // opening quote - for l.pos < len(l.src) { - c := l.src[l.pos] - if c == '\\' && l.pos+1 < len(l.src) { - l.pos += 2 - continue - } - l.pos++ - if c == quote { - return - } - } + l.emit(token.AttributeStart, start) } // scanHeredoc returns the end offset of a heredoc/nowdoc starting at "<<<", or diff --git a/blink/internal/lexer/lexer_test.go b/blink/internal/lexer/lexer_test.go index 736010585a..50a0cf3bd6 100644 --- a/blink/internal/lexer/lexer_test.go +++ b/blink/internal/lexer/lexer_test.go @@ -99,17 +99,22 @@ func TestLexHeredocNowdocOpaque(t *testing.T) { func TestLexMultilineAttributeWithNowdoc(t *testing.T) { src := " 2 && tk.Value[:2] == "#[" { - attr = tk.Value + if tk.Kind == token.AttributeStart { + sawAttrStart = true + } + if tk.Kind == token.String && strings.Contains(tk.Value, "prose e.g. with. dots") { + nowdoc = tk.Value } } - if attr == "" { - t.Fatal("multiline attribute not captured as one token") + if !sawAttrStart { + t.Fatal(`attribute opener "#[" not lexed as an AttributeStart token`) } - if !strings.Contains(attr, "prose e.g. with. dots") { - t.Errorf("nowdoc body not inside the attribute token: %q", attr) + // the nowdoc stays one opaque String token, so its body never leaks out as code + if nowdoc == "" { + t.Error("nowdoc body not captured as a single string token") } // lossless var got strings.Builder @@ -121,6 +126,41 @@ func TestLexMultilineAttributeWithNowdoc(t *testing.T) { } } +func TestLexAttributeIsNative(t *testing.T) { + toks := Lex("= len(s.toks) || s.toks[i].Kind != token.AttributeStart { + return -1 + } + for j := i + 1; j < len(s.toks); j++ { + if s.toks[j].Kind == token.AttributeEnd { + return j + } + } + return -1 +} + +// AttributeStartOf returns the index of the AttributeStart ("#[") matching the +// AttributeEnd ("]") at i, or -1 when i is not an attribute close. +func (s *Stream) AttributeStartOf(i int) int { + if i < 0 || i >= len(s.toks) || s.toks[i].Kind != token.AttributeEnd { + return -1 + } + for j := i - 1; j >= 0; j-- { + if s.toks[j].Kind == token.AttributeStart { + return j + } + } + return -1 +} + // ReplaceRange swaps tokens [start, end] (inclusive) for repl. The splice is // done in place: when len(repl) matches the range the tail never moves, and // otherwise only the tail shifts, so no fresh tail copy is allocated. From 63d7112f4419dda3bbac17ab0403bef3caed6d53 Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 30 Oct 2026 21:54:24 +0100 Subject: [PATCH 2/3] Fix fixers misreading native attribute tokens Making #[...] lex as real tokens exposed four fixers that did not account for an attribute in the token stream: - NoUnusedImports: an attribute name (#[AsCommand(...)]) now counts as a class use even with a following (, so its import is not dropped. - BracesPosition: a class header scan no longer reads a leading attribute's ) as a multiline signature, so the class brace moves to its own line. - NoSuperfluousPhpdocTags: parseParams skips a param attribute, so a @param covered by the native type is removed even with #[\SensitiveParameter]. - StatementIndentation: a promoted param with a leading attribute is still a param, not a class property, so following params keep their indent. --- blink/internal/fixer/rules/braces_position.go | 3 ++ .../fixer/rules/braces_position_test.go | 11 +++++ blink/internal/fixer/rules/imports_unused.go | 41 +++++++++++++++++++ .../fixer/rules/imports_unused_test.go | 10 +++++ .../fixer/rules/no_superfluous_phpdoc_tags.go | 6 +++ .../rules/no_superfluous_phpdoc_tags_test.go | 11 +++++ .../fixer/rules/statement_indentation.go | 5 ++- .../statement_indentation_regression_test.go | 10 +++++ 8 files changed, 95 insertions(+), 2 deletions(-) diff --git a/blink/internal/fixer/rules/braces_position.go b/blink/internal/fixer/rules/braces_position.go index a758559625..301624ac19 100644 --- a/blink/internal/fixer/rules/braces_position.go +++ b/blink/internal/fixer/rules/braces_position.go @@ -230,6 +230,9 @@ func funcSignatureMultiline(s *tokens.Stream, brace int) bool { closeParen := -1 for j := brace - 1; j >= 0; j-- { t := s.At(j) + if t.Kind == token.AttributeEnd { + return false // a leading attribute's parens are not the signature (e.g. a class header) + } if t.Kind != token.Punct { continue } diff --git a/blink/internal/fixer/rules/braces_position_test.go b/blink/internal/fixer/rules/braces_position_test.go index e73b468897..15d753c880 100644 --- a/blink/internal/fixer/rules/braces_position_test.go +++ b/blink/internal/fixer/rules/braces_position_test.go @@ -111,3 +111,14 @@ func TestClassifyTraitBlockIsOther(t *testing.T) { t.Fatalf("trait block classify = %d, want braceOther", kind) } } + +// A class preceded by a multiline attribute still gets its opening brace moved to +// the next line: the attribute's own ")" must not be read as a multiline signature. +func TestBracesPositionClassAfterMultilineAttribute(t *testing.T) { + t.Parallel() + src := "= 0; j-- { + t := s.At(j) + if t.Kind == token.AttributeStart && depth == 0 { + return true + } + if t.Kind == token.AttributeEnd { + return false + } + if t.Kind == token.Punct { + switch t.Value { + case ")", "]", "}": + depth++ + case "(", "[", "{": + if depth == 0 { + return false + } + depth-- + } + } + } + return false +} + // usageMatchesKind reports whether the identifier at k is a reference of the // given import kind. func usageMatchesKind(s *tokens.Stream, k int, kind string) bool { + if attributeNameAt(s, k) { + return kind == "class" // an attribute is always a class reference, even with "(" after + } prev, hasPrev := prevSignificant(s, k) if hasPrev { if prev.Kind == token.Punct { diff --git a/blink/internal/fixer/rules/imports_unused_test.go b/blink/internal/fixer/rules/imports_unused_test.go index a7af8b0b91..2414593c1e 100644 --- a/blink/internal/fixer/rules/imports_unused_test.go +++ b/blink/internal/fixer/rules/imports_unused_test.go @@ -29,3 +29,13 @@ func TestNoUnusedImportsBraceNamespaceUnused(t *testing.T) { t.Fatalf("unused brace-namespace import must be removed; changed=%v got=%q want=%q", changed, got, want) } } + +// An import used only as an attribute name ("#[AsCommand(...)]") is a class usage, +// even though a "(" follows the name, so it must be kept. +func TestNoUnusedImportsAttributeUsage(t *testing.T) { + t.Parallel() + src := "= 0 { + k = end // a param attribute ("#[\SensitiveParameter] array $x") is not its type + } + continue + } if t.Kind == token.Whitespace || t.Kind == token.Comment || t.Kind == token.DocComment { if t.Kind == token.Comment || t.Kind == token.DocComment { for _, name := range nspCommentParamNames(t.Value) { diff --git a/blink/internal/fixer/rules/no_superfluous_phpdoc_tags_test.go b/blink/internal/fixer/rules/no_superfluous_phpdoc_tags_test.go index bd4620d319..c14cb8d9f8 100644 --- a/blink/internal/fixer/rules/no_superfluous_phpdoc_tags_test.go +++ b/blink/internal/fixer/rules/no_superfluous_phpdoc_tags_test.go @@ -117,3 +117,14 @@ func TestNoSuperfluousPhpdocTagsSourceURL(t *testing.T) { t.Fatalf("SourceURL %q, want %q", got, want) } } + +// A @param whose native type is fully covered is superfluous even when the +// parameter carries an attribute ("#[\SensitiveParameter] array $x"). +func TestNoSuperfluousPhpdocTagsAttributedParam(t *testing.T) { + t.Parallel() + src := "= 0 && s.At(before).Kind == token.Punct && (s.At(before).Value == "(" || s.At(before).Value == ",") { return false } diff --git a/blink/internal/fixer/rules/statement_indentation_regression_test.go b/blink/internal/fixer/rules/statement_indentation_regression_test.go index 9a744f6369..d15080feab 100644 --- a/blink/internal/fixer/rules/statement_indentation_regression_test.go +++ b/blink/internal/fixer/rules/statement_indentation_regression_test.go @@ -10,3 +10,13 @@ func TestStatementIndentationNoPanicOnMalformed(t *testing.T) { apply(t, StatementIndentation{}, src) // must not panic } } + +// A promoted constructor parameter with a leading attribute is still a param, not +// a class property, so the params after it must not gain an extra indent level. +func TestStatementIndentationAttributedPromotedParam(t *testing.T) { + t.Parallel() + src := " Date: Fri, 30 Oct 2026 22:17:41 +0100 Subject: [PATCH 3/3] Lock rector-src parity to 0% after native attribute lexing rector-src now matches PHP ECS on all 1568 files, so ratchet the locker from 0.25% to 0% - any future divergence fails CI. --- blink/parity/corpora/rector-src.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blink/parity/corpora/rector-src.json b/blink/parity/corpora/rector-src.json index 22b43432be..e3063da1e8 100644 --- a/blink/parity/corpora/rector-src.json +++ b/blink/parity/corpora/rector-src.json @@ -4,5 +4,5 @@ "paths": ["src", "rules", "tests"], "skip": ["*/node_modules/*", "*/Fixture/*"], "sets": ["psr12", "common"], - "max_diff_percent": 0.25 + "max_diff_percent": 0.0 }