From 3c8e318042561abea32576674585bf6dc7d2b161 Mon Sep 17 00:00:00 2001 From: Ben Gribaudo Date: Thu, 15 Oct 2026 17:25:50 -0400 Subject: [PATCH] Implementing suppot for verbatim literals --- src/powerquery-parser/language/ast/ast.ts | 1 + .../language/ast/astUtils/astUtils.ts | 11 ++++++++++- src/powerquery-parser/language/token.ts | 1 + .../language/type/typeUtils/typeUtils.ts | 1 + src/powerquery-parser/lexer/lexer.ts | 16 ++++++++++++++++ .../localization/localization.ts | 1 + .../parser/parseState/parseStateUtils.ts | 1 + .../combinatorialParserV2.ts | 1 + .../readOperatorsAndOperands.ts | 1 + .../parser/parsers/naiveParseSteps.ts | 3 +++ src/test/libraryTest/lexer/lexError.test.ts | 2 ++ src/test/libraryTest/lexer/lexSimple.test.ts | 16 ++++++++++++++++ src/test/libraryTest/parser/parseSimple.test.ts | 4 ++++ 13 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/powerquery-parser/language/ast/ast.ts b/src/powerquery-parser/language/ast/ast.ts index ccfbceea..9aaa9127 100644 --- a/src/powerquery-parser/language/ast/ast.ts +++ b/src/powerquery-parser/language/ast/ast.ts @@ -364,6 +364,7 @@ export enum LiteralKind { Numeric = "Numeric", Record = "Record", Text = "Text", + Verbatim = "Verbatim", } export interface LiteralExpression extends INode { diff --git a/src/powerquery-parser/language/ast/astUtils/astUtils.ts b/src/powerquery-parser/language/ast/astUtils/astUtils.ts index 035d3281..a77dbdd9 100644 --- a/src/powerquery-parser/language/ast/astUtils/astUtils.ts +++ b/src/powerquery-parser/language/ast/astUtils/astUtils.ts @@ -121,7 +121,13 @@ export function isTUnaryExpression(node: Ast.TNode): node is Ast.TUnaryExpressio export function literalKindFrom( tokenKind: TokenKind | undefined, -): Ast.LiteralKind.Numeric | Ast.LiteralKind.Logical | Ast.LiteralKind.Null | Ast.LiteralKind.Text | undefined { +): + | Ast.LiteralKind.Numeric + | Ast.LiteralKind.Logical + | Ast.LiteralKind.Null + | Ast.LiteralKind.Text + | Ast.LiteralKind.Verbatim + | undefined { switch (tokenKind) { case TokenKind.HexLiteral: case TokenKind.KeywordHashNan: @@ -139,6 +145,9 @@ export function literalKindFrom( case TokenKind.TextLiteral: return Ast.LiteralKind.Text; + case TokenKind.VerbatimLiteral: + return Ast.LiteralKind.Verbatim; + case TokenKind.Ampersand: case TokenKind.Asterisk: case TokenKind.AtSign: diff --git a/src/powerquery-parser/language/token.ts b/src/powerquery-parser/language/token.ts index dafb69b5..c65e0100 100644 --- a/src/powerquery-parser/language/token.ts +++ b/src/powerquery-parser/language/token.ts @@ -64,6 +64,7 @@ export enum TokenKind { RightParenthesis = "RightParenthesis", Semicolon = "Semicolon", TextLiteral = "TextLiteral", + VerbatimLiteral = "VerbatimLiteral", } export interface IToken { diff --git a/src/powerquery-parser/language/type/typeUtils/typeUtils.ts b/src/powerquery-parser/language/type/typeUtils/typeUtils.ts index c88db980..b05ea9b3 100644 --- a/src/powerquery-parser/language/type/typeUtils/typeUtils.ts +++ b/src/powerquery-parser/language/type/typeUtils/typeUtils.ts @@ -31,6 +31,7 @@ export function typeKindFromLiteralKind(literalKind: Ast.LiteralKind): Type.Type return Type.TypeKind.Record; case Ast.LiteralKind.Text: + case Ast.LiteralKind.Verbatim: return Type.TypeKind.Text; default: diff --git a/src/powerquery-parser/lexer/lexer.ts b/src/powerquery-parser/lexer/lexer.ts index 375c67b8..339fe50d 100644 --- a/src/powerquery-parser/lexer/lexer.ts +++ b/src/powerquery-parser/lexer/lexer.ts @@ -268,6 +268,8 @@ function tokenize( if (chr2 === '"') { token = readQuotedToken(text, lineNumber, positionStart, locale); + } else if (chr2 === "!" && text[positionStart + 2] === '"') { + token = readVerbatimLiteral(text, lineNumber, positionStart, locale); } else { token = readKeyword(text, lineNumber, positionStart, locale); } @@ -315,6 +317,20 @@ function readQuotedToken(text: string, lineNumber: number, positionStart: number } } +function readVerbatimLiteral(text: string, lineNumber: number, positionStart: number, locale: string): ReadToken { + const positionEnd: number | undefined = indexOfTextEnd(text, positionStart + 3); + + if (positionEnd !== undefined) { + return readTokenFrom(Token.TokenKind.VerbatimLiteral, text, positionStart, positionEnd + 1); + } else { + throw new LexError.UnterminatedMultilineTokenError( + locale, + graphemePositionFrom(text, lineNumber, positionStart), + LexError.UnterminatedMultilineTokenKind.Text, + ); + } +} + function readHexLiteral(text: string, lineNumber: number, positionStart: number, locale: string): ReadToken { const positionEnd: number | undefined = indexOfRegexEnd(Pattern.Hex, text, positionStart); diff --git a/src/powerquery-parser/localization/localization.ts b/src/powerquery-parser/localization/localization.ts index 2476064b..849c39d9 100644 --- a/src/powerquery-parser/localization/localization.ts +++ b/src/powerquery-parser/localization/localization.ts @@ -180,6 +180,7 @@ export function localizeTokenKind(localizationTemplates: ILocalizationTemplates, case Token.TokenKind.Semicolon: return localizationTemplates.tokenKind_semicolon; case Token.TokenKind.TextLiteral: + case Token.TokenKind.VerbatimLiteral: return localizationTemplates.tokenKind_textLiteral; default: diff --git a/src/powerquery-parser/parser/parseState/parseStateUtils.ts b/src/powerquery-parser/parser/parseState/parseStateUtils.ts index 8d050a9b..95568168 100644 --- a/src/powerquery-parser/parser/parseState/parseStateUtils.ts +++ b/src/powerquery-parser/parser/parseState/parseStateUtils.ts @@ -255,6 +255,7 @@ export function isOnGeneralizedIdentifierStart(state: ParseState, tokenIndex: nu case Token.TokenKind.RightParenthesis: case Token.TokenKind.Semicolon: case Token.TokenKind.TextLiteral: + case Token.TokenKind.VerbatimLiteral: return false; default: diff --git a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts index feb607db..1317f2e1 100644 --- a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts +++ b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts @@ -246,6 +246,7 @@ async function readUnaryExpression( case Token.TokenKind.NumericLiteral: case Token.TokenKind.NullLiteral: case Token.TokenKind.TextLiteral: + case Token.TokenKind.VerbatimLiteral: trace.exit(); return NaiveParseSteps.readLiteralExpression(state, parser, trace.id); diff --git a/src/powerquery-parser/parser/parsers/combinatorialParserV2/readOperatorsAndOperands.ts b/src/powerquery-parser/parser/parsers/combinatorialParserV2/readOperatorsAndOperands.ts index 75c68dce..441fb51a 100644 --- a/src/powerquery-parser/parser/parsers/combinatorialParserV2/readOperatorsAndOperands.ts +++ b/src/powerquery-parser/parser/parsers/combinatorialParserV2/readOperatorsAndOperands.ts @@ -349,6 +349,7 @@ function nextOperatorAndOperandRead(tokenKind: Token.TokenKind): NextOperatorAnd case Token.TokenKind.RightParenthesis: case Token.TokenKind.Semicolon: case Token.TokenKind.TextLiteral: + case Token.TokenKind.VerbatimLiteral: return undefined; default: diff --git a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts index 6e1201e6..e9a7e204 100644 --- a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts +++ b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts @@ -494,6 +494,7 @@ export async function readExpression( case TokenKind.RightParenthesis: case TokenKind.Semicolon: case TokenKind.TextLiteral: + case TokenKind.VerbatimLiteral: case undefined: expression = await parser.readNullCoalescingExpression(state, parser, trace.id); break; @@ -991,6 +992,7 @@ export async function readPrimaryExpression( case TokenKind.RightParenthesis: case TokenKind.Semicolon: case TokenKind.TextLiteral: + case TokenKind.VerbatimLiteral: case undefined: primaryExpression = parser.readLiteralExpression(state, parser, trace.id); break; @@ -1121,6 +1123,7 @@ export function readLiteralExpression( TokenKind.NumericLiteral, TokenKind.NullLiteral, TokenKind.TextLiteral, + TokenKind.VerbatimLiteral, ]; const error: ParseError.ExpectedAnyTokenKindError | undefined = ParseStateUtils.testIsOnAnyTokenKind( diff --git a/src/test/libraryTest/lexer/lexError.test.ts b/src/test/libraryTest/lexer/lexError.test.ts index d75344c1..6d08e85f 100644 --- a/src/test/libraryTest/lexer/lexError.test.ts +++ b/src/test/libraryTest/lexer/lexError.test.ts @@ -42,6 +42,7 @@ describe("Lexer.Error", () => { ["/*", Lexer.LexError.UnterminatedMultilineTokenKind.MultilineComment], ['"', Lexer.LexError.UnterminatedMultilineTokenKind.Text], ['#"', Lexer.LexError.UnterminatedMultilineTokenKind.QuotedIdentifier], + ['#!"', Lexer.LexError.UnterminatedMultilineTokenKind.Text], ] as const) { it(kind, () => { const error: Lexer.LexError.TInnerLexError = assertGetLexError(opening); @@ -67,6 +68,7 @@ describe("Lexer.Error", () => { for (const [opening, kind] of [ ['"', Lexer.LexError.UnterminatedMultilineTokenKind.Text], ['#"', Lexer.LexError.UnterminatedMultilineTokenKind.QuotedIdentifier], + ['#!"', Lexer.LexError.UnterminatedMultilineTokenKind.Text], ] as const) { it(`${kind} does not treat a doubled quote at EOF as a closing delimiter`, () => { const error: Lexer.LexError.TInnerLexError = assertGetLexError(`${opening}a""""`); diff --git a/src/test/libraryTest/lexer/lexSimple.test.ts b/src/test/libraryTest/lexer/lexSimple.test.ts index 9a357a74..81ba617e 100644 --- a/src/test/libraryTest/lexer/lexSimple.test.ts +++ b/src/test/libraryTest/lexer/lexSimple.test.ts @@ -228,6 +228,22 @@ type assertGetSnapshotAbridgedTokens(text, expected, true); }); + + it(`VerbatimLiteral`, () => { + const text: string = ` +#!"" +#!"a" +#!"a""b" +`; + + const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [ + [Language.Token.TokenKind.VerbatimLiteral, `#!""`], + [Language.Token.TokenKind.VerbatimLiteral, `#!"a"`], + [Language.Token.TokenKind.VerbatimLiteral, `#!"a""b"`], + ]; + + assertGetSnapshotAbridgedTokens(text, expected, true); + }); }); describe(`Lexer.Simple.Whitespace`, () => { diff --git a/src/test/libraryTest/parser/parseSimple.test.ts b/src/test/libraryTest/parser/parseSimple.test.ts index ee84d0f8..b2c1a65b 100644 --- a/src/test/libraryTest/parser/parseSimple.test.ts +++ b/src/test/libraryTest/parser/parseSimple.test.ts @@ -1172,6 +1172,10 @@ describe("Parser.AbridgedNode", () => { await ParserTestUtils.runAbridgedNodeTest(`false`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); + it(`#!"a"`, async () => { + await ParserTestUtils.runAbridgedNodeTest(`#!"a"`, [[Ast.NodeKind.LiteralExpression, undefined]]); + }); + it(`1`, async () => { await ParserTestUtils.runAbridgedNodeTest(`1`, [[Ast.NodeKind.LiteralExpression, undefined]]); });