Implementing suppot for verbatim literals

src/powerquery-parser/language/ast/ast.ts

@@ -364,6 +364,7 @@ export enum LiteralKind {
     Numeric = "Numeric",
     Record = "Record",
     Text = "Text",
+    Verbatim = "Verbatim",
 }
 
 export interface LiteralExpression extends INode {

src/powerquery-parser/language/ast/astUtils/astUtils.ts

@@ -121,7 +121,13 @@ export function isTUnaryExpression(node: Ast.TNode): node is Ast.TUnaryExpressio
 
 export function literalKindFrom(
     tokenKind: TokenKind | undefined,
-): Ast.LiteralKind.Numeric | Ast.LiteralKind.Logical | Ast.LiteralKind.Null | Ast.LiteralKind.Text | undefined {
+):
+    | Ast.LiteralKind.Numeric
+    | Ast.LiteralKind.Logical
+    | Ast.LiteralKind.Null
+    | Ast.LiteralKind.Text
+    | Ast.LiteralKind.Verbatim
+    | undefined {
     switch (tokenKind) {
         case TokenKind.HexLiteral:
         case TokenKind.KeywordHashNan:
@@ -139,6 +145,9 @@ export function literalKindFrom(
         case TokenKind.TextLiteral:
             return Ast.LiteralKind.Text;
 
+        case TokenKind.VerbatimLiteral:
+            return Ast.LiteralKind.Verbatim;
+
         case TokenKind.Ampersand:
         case TokenKind.Asterisk:
         case TokenKind.AtSign:

src/powerquery-parser/language/token.ts

@@ -64,6 +64,7 @@ export enum TokenKind {
     RightParenthesis = "RightParenthesis",
     Semicolon = "Semicolon",
     TextLiteral = "TextLiteral",
+    VerbatimLiteral = "VerbatimLiteral",
 }
 
 export interface IToken<Kind, Position> {

src/powerquery-parser/language/type/typeUtils/typeUtils.ts

@@ -31,6 +31,7 @@ export function typeKindFromLiteralKind(literalKind: Ast.LiteralKind): Type.Type
             return Type.TypeKind.Record;
 
         case Ast.LiteralKind.Text:
+        case Ast.LiteralKind.Verbatim:
             return Type.TypeKind.Text;
 
         default:

src/powerquery-parser/lexer/lexer.ts

@@ -268,6 +268,8 @@ function tokenize(
 
         if (chr2 === '"') {
             token = readQuotedToken(text, lineNumber, positionStart, locale);
+        } else if (chr2 === "!" && text[positionStart + 2] === '"') {
+            token = readVerbatimLiteral(text, lineNumber, positionStart, locale);
         } else {
             token = readKeyword(text, lineNumber, positionStart, locale);
         }
@@ -315,6 +317,20 @@ function readQuotedToken(text: string, lineNumber: number, positionStart: number
     }
 }
 
+function readVerbatimLiteral(text: string, lineNumber: number, positionStart: number, locale: string): ReadToken {
+    const positionEnd: number | undefined = indexOfTextEnd(text, positionStart + 3);
+
+    if (positionEnd !== undefined) {
+        return readTokenFrom(Token.TokenKind.VerbatimLiteral, text, positionStart, positionEnd + 1);
+    } else {
+        throw new LexError.UnterminatedMultilineTokenError(
+            locale,
+            graphemePositionFrom(text, lineNumber, positionStart),
+            LexError.UnterminatedMultilineTokenKind.Text,
+        );
+    }
+}
+
 function readHexLiteral(text: string, lineNumber: number, positionStart: number, locale: string): ReadToken {
     const positionEnd: number | undefined = indexOfRegexEnd(Pattern.Hex, text, positionStart);
 

src/powerquery-parser/localization/localization.ts

@@ -180,6 +180,7 @@ export function localizeTokenKind(localizationTemplates: ILocalizationTemplates,
         case Token.TokenKind.Semicolon:
             return localizationTemplates.tokenKind_semicolon;
         case Token.TokenKind.TextLiteral:
+        case Token.TokenKind.VerbatimLiteral:
             return localizationTemplates.tokenKind_textLiteral;
 
         default:

src/powerquery-parser/parser/parseState/parseStateUtils.ts

@@ -255,6 +255,7 @@ export function isOnGeneralizedIdentifierStart(state: ParseState, tokenIndex: nu
         case Token.TokenKind.RightParenthesis:
         case Token.TokenKind.Semicolon:
         case Token.TokenKind.TextLiteral:
+        case Token.TokenKind.VerbatimLiteral:
             return false;
 
         default:

src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts

@@ -246,6 +246,7 @@ async function readUnaryExpression(
         case Token.TokenKind.NumericLiteral:
         case Token.TokenKind.NullLiteral:
         case Token.TokenKind.TextLiteral:
+        case Token.TokenKind.VerbatimLiteral:
             trace.exit();
 
             return NaiveParseSteps.readLiteralExpression(state, parser, trace.id);

src/powerquery-parser/parser/parsers/combinatorialParserV2/readOperatorsAndOperands.ts

@@ -349,6 +349,7 @@ function nextOperatorAndOperandRead(tokenKind: Token.TokenKind): NextOperatorAnd
         case Token.TokenKind.RightParenthesis:
         case Token.TokenKind.Semicolon:
         case Token.TokenKind.TextLiteral:
+        case Token.TokenKind.VerbatimLiteral:
             return undefined;
 
         default:

src/powerquery-parser/parser/parsers/naiveParseSteps.ts

@@ -494,6 +494,7 @@ export async function readExpression(
         case TokenKind.RightParenthesis:
         case TokenKind.Semicolon:
         case TokenKind.TextLiteral:
+        case TokenKind.VerbatimLiteral:
         case undefined:
             expression = await parser.readNullCoalescingExpression(state, parser, trace.id);
             break;
@@ -991,6 +992,7 @@ export async function readPrimaryExpression(
             case TokenKind.RightParenthesis:
             case TokenKind.Semicolon:
             case TokenKind.TextLiteral:
+            case TokenKind.VerbatimLiteral:
             case undefined:
                 primaryExpression = parser.readLiteralExpression(state, parser, trace.id);
                 break;
@@ -1121,6 +1123,7 @@ export function readLiteralExpression(
         TokenKind.NumericLiteral,
         TokenKind.NullLiteral,
         TokenKind.TextLiteral,
+        TokenKind.VerbatimLiteral,
     ];
 
     const error: ParseError.ExpectedAnyTokenKindError | undefined = ParseStateUtils.testIsOnAnyTokenKind(

src/test/libraryTest/lexer/lexError.test.ts

@@ -42,6 +42,7 @@ describe("Lexer.Error", () => {
             ["/*", Lexer.LexError.UnterminatedMultilineTokenKind.MultilineComment],
             ['"', Lexer.LexError.UnterminatedMultilineTokenKind.Text],
             ['#"', Lexer.LexError.UnterminatedMultilineTokenKind.QuotedIdentifier],
+            ['#!"', Lexer.LexError.UnterminatedMultilineTokenKind.Text],
         ] as const) {
             it(kind, () => {
                 const error: Lexer.LexError.TInnerLexError = assertGetLexError(opening);
@@ -67,6 +68,7 @@ describe("Lexer.Error", () => {
         for (const [opening, kind] of [
             ['"', Lexer.LexError.UnterminatedMultilineTokenKind.Text],
             ['#"', Lexer.LexError.UnterminatedMultilineTokenKind.QuotedIdentifier],
+            ['#!"', Lexer.LexError.UnterminatedMultilineTokenKind.Text],
         ] as const) {
             it(`${kind} does not treat a doubled quote at EOF as a closing delimiter`, () => {
                 const error: Lexer.LexError.TInnerLexError = assertGetLexError(`${opening}a""""`);

src/test/libraryTest/lexer/lexSimple.test.ts

@@ -228,6 +228,22 @@ type
 
         assertGetSnapshotAbridgedTokens(text, expected, true);
     });
+
+    it(`VerbatimLiteral`, () => {
+        const text: string = `
+#!""
+#!"a"
+#!"a""b"
+`;
+
+        const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [
+            [Language.Token.TokenKind.VerbatimLiteral, `#!""`],
+            [Language.Token.TokenKind.VerbatimLiteral, `#!"a"`],
+            [Language.Token.TokenKind.VerbatimLiteral, `#!"a""b"`],
+        ];
+
+        assertGetSnapshotAbridgedTokens(text, expected, true);
+    });
 });
 
 describe(`Lexer.Simple.Whitespace`, () => {

src/test/libraryTest/parser/parseSimple.test.ts

@@ -1172,6 +1172,10 @@ describe("Parser.AbridgedNode", () => {
             await ParserTestUtils.runAbridgedNodeTest(`false`, [[Ast.NodeKind.LiteralExpression, undefined]]);
         });
 
+        it(`#!"a"`, async () => {
+            await ParserTestUtils.runAbridgedNodeTest(`#!"a"`, [[Ast.NodeKind.LiteralExpression, undefined]]);
+        });
+
         it(`1`, async () => {
             await ParserTestUtils.runAbridgedNodeTest(`1`, [[Ast.NodeKind.LiteralExpression, undefined]]);
         });