From 60b80a2b548b2d52cf05f4cb90a99a1c1ad1b1ab Mon Sep 17 00:00:00 2001 From: minleejae Date: Fri, 23 Oct 2026 13:26:53 +0900 Subject: [PATCH 1/2] fix: end column DEFAULT values before a top-level COLLATE In MySQL and PostgreSQL column definitions, a COLLATE after the DEFAULT value is a column attribute: MySQL's DEFAULT takes a literal and PostgreSQL's a b_expr, and neither includes COLLATE. The column DEFAULT parsed a full expression, so DEFAULT 'N' COLLATE x became one DEFAULT option holding a CollateExpression. Parse column DEFAULT values with a ColumnDefaultValue production that stops before a COLLATE at its top level. That COLLATE becomes the column's COLLATE option. A COLLATE inside parentheses, function arguments or CASE stays in the expression. SET DEFAULT, an a_expr in PostgreSQL, and CREATE DOMAIN are unchanged. SQL output does not change. The nesting walk of slice subscripts moves to the package-private TokenNesting, like RoutineBodyBoundary, and serves both checks. The generated CCJSqlParser class is close to the size JaCoCo can still instrument. Signed-off-by: minleejae --- .../sf/jsqlparser/parser/TokenNesting.java | 38 +++++++ .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 50 ++++++--- src/site/sphinx/usage.rst | 12 ++ .../create/ColumnDefaultExpressionTest.java | 106 +++++++++++++++++- 4 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 src/main/java/net/sf/jsqlparser/parser/TokenNesting.java diff --git a/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java b/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java new file mode 100644 index 0000000000..e41c6a0dd8 --- /dev/null +++ b/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java @@ -0,0 +1,38 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser; + +import static net.sf.jsqlparser.parser.CCJSqlParserConstants.*; + +/** + * Token nesting checks used by grammar lookaheads. Kept outside the generated parser, whose class + * is close to the size JaCoCo can still instrument. + */ +final class TokenNesting { + private TokenNesting() {} + + /** + * Whether the target is not nested in brackets, braces, parentheses or a CASE expression opened + * from the first token on. + */ + static boolean isTopLevelFrom(Token first, Token target) { + int depth = 0; + for (Token t = first; t != null && t != target; t = t.next) { + if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image) + || "{".equals(t.image)) { + depth++; + } else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image) + || "}".equals(t.image)) { + depth--; + } + } + return depth == 0; + } +} diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 755d965fc0..3f2ada7df9 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -224,20 +224,19 @@ public class CCJSqlParser extends AbstractJSqlParser { */ private boolean isSliceColonAhead() { Token colon = getToken(1); - if (subscriptOpenings.isEmpty() || !":".equals(colon.image)) { - return false; - } - int depth = 0; - for (Token t = subscriptOpenings.peek().next; t != null && t != colon; t = t.next) { - if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image) - || "{".equals(t.image)) { - depth++; - } else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image) - || "}".equals(t.image)) { - depth--; - } - } - return depth == 0; + return !subscriptOpenings.isEmpty() && ":".equals(colon.image) + && TokenNesting.isTopLevelFrom(subscriptOpenings.peek().next, colon); + } + + // first tokens of the enclosing column DEFAULT values: a COLLATE at their top level is the + // column's own attribute, as in MySQL's column_attribute and PostgreSQL's ColQualList + private final Deque columnDefaultStarts = new ArrayDeque(); + + /** Whether the pending COLLATE follows an unparenthesized column DEFAULT value. */ + private boolean isColumnCollateAhead() { + Token collate = getToken(1); + return !columnDefaultStarts.isEmpty() && collate.kind == K_COLLATE + && TokenNesting.isTopLevelFrom(columnDefaultStarts.peek(), collate); } private Expression createOutputArgument(Expression argument) throws ParseException { @@ -13149,7 +13148,7 @@ Expression PrimaryExpression() #PrimaryExpression: ) [ - LOOKAHEAD(2) (token= | token= | token=) { retval = new CollateExpression(retval, token.image); } + LOOKAHEAD(2, { !isColumnCollateAhead() }) (token= | token= | token=) { retval = new CollateExpression(retval, token.image); } ] [ @@ -16389,7 +16388,7 @@ ColumnOption ColumnDefinitionOption(): { LOOKAHEAD() { option = ColumnOption.raw("DEFAULT"); } // Db2 permits an omitted default value. Keep these implicit defaults raw. [ LOOKAHEAD(2, { !(getToken(1).kind == K_NOT && getToken(2).kind == K_NULL) }) - defaultExpression=Expression() { option = ColumnOption.defaultValue(defaultExpression); } ] + defaultExpression=ColumnDefaultValue() { option = ColumnOption.defaultValue(defaultExpression); } ] | parameter=ColumnDefinitionParameter() { option = ColumnOption.raw(parameter); } @@ -16397,6 +16396,25 @@ ColumnOption ColumnDefinitionOption(): { { return option; } } +/** + * An explicit column DEFAULT value. MySQL's DEFAULT literal and PostgreSQL's DEFAULT b_expr exclude + * COLLATE, so the value ends before a COLLATE at its top level, which is the column's own + * attribute. A COLLATE inside parentheses, function arguments or CASE stays in the expression. + */ +Expression ColumnDefaultValue(): +{ + Expression expression; +} +{ + { columnDefaultStarts.push(getToken(1)); } + try { + expression=Expression() + } finally { + columnDefaultStarts.pop(); + } + { return expression; } +} + GeneratedColumnDefinition GeneratedColumnDefinition(): { Expression expression; diff --git a/src/site/sphinx/usage.rst b/src/site/sphinx/usage.rst index 75b5c758d0..a07b72f088 100644 --- a/src/site/sphinx/usage.rst +++ b/src/site/sphinx/usage.rst @@ -935,6 +935,18 @@ Charset remains in ``ColDataType.getCharacterSet()``; column collation is in its COLLATE option. Consumers can map both directly without reconstructing tokens. StringValue comment bodies retain their SQL escape representation. +A column DEFAULT value ends before a COLLATE at its top level, which becomes the +column's COLLATE option: MySQL's DEFAULT literal and PostgreSQL's DEFAULT ``b_expr`` +do not include COLLATE. A COLLATE inside parentheses, function arguments or a CASE +expression stays in the default expression. ``SET DEFAULT`` actions and CREATE +DOMAIN keep an expression COLLATE. + +Migration: ``DEFAULT 'N' COLLATE utf8mb4_bin`` in a column definition previously +produced one DEFAULT option holding a ``CollateExpression``. It now produces a +DEFAULT option with ``'N'`` and a COLLATE option with ``getCollation()``; +``getColumnSpecs()`` returns ``[DEFAULT, 'N', COLLATE, utf8mb4_bin]``. SQL output +is unchanged. + MySQL column attributes have their own kinds: ``SRID`` (``getSrid()``), ``COLUMN_FORMAT`` (``getColumnFormat()``: FIXED, DYNAMIC or DEFAULT), and ``ENGINE_ATTRIBUTE``/``SECONDARY_ENGINE_ATTRIBUTE`` (``getEngineAttribute()`` diff --git a/src/test/java/net/sf/jsqlparser/statement/create/ColumnDefaultExpressionTest.java b/src/test/java/net/sf/jsqlparser/statement/create/ColumnDefaultExpressionTest.java index 4afc42d80a..2e403e5524 100644 --- a/src/test/java/net/sf/jsqlparser/statement/create/ColumnDefaultExpressionTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/create/ColumnDefaultExpressionTest.java @@ -9,27 +9,36 @@ */ package net.sf.jsqlparser.statement.create; +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; import static net.sf.jsqlparser.util.validation.ValidationTestAsserts.validateNotAllowed; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Stream; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.CollateExpression; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.parser.CCJSqlParser; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.parser.feature.Feature; -import net.sf.jsqlparser.util.validation.feature.FeaturesAllowed; -import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.StatementVisitorAdapter; import net.sf.jsqlparser.statement.alter.Alter; +import net.sf.jsqlparser.statement.alter.AlterDomain; +import net.sf.jsqlparser.statement.alter.AlterRelation; +import net.sf.jsqlparser.statement.create.domain.CreateDomain; import net.sf.jsqlparser.statement.create.table.ColumnDefinition; import net.sf.jsqlparser.statement.create.table.ColumnOption; import net.sf.jsqlparser.statement.create.table.CreateTable; @@ -37,8 +46,11 @@ import net.sf.jsqlparser.util.deparser.ExpressionDeParser; import net.sf.jsqlparser.util.deparser.SelectDeParser; import net.sf.jsqlparser.util.deparser.StatementDeParser; +import net.sf.jsqlparser.util.validation.feature.FeaturesAllowed; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; class ColumnDefaultExpressionTest { @@ -161,6 +173,94 @@ void neighboringAndLegacyColumnOptionsStillRoundTrip(String sql) throws JSQLPars assertRoundTrip(CCJSqlParserUtil.parse(sql)); } + @ParameterizedTest + @ValueSource( + strings = {"CREATE TABLE t (c VARCHAR(30) ", "ALTER TABLE t ADD COLUMN c VARCHAR(30) ", + "ALTER TABLE t MODIFY COLUMN c VARCHAR(30) ", + "ALTER TABLE t CHANGE COLUMN old_c c VARCHAR(30) "}) + void collateAfterDefaultIsAColumnAttribute(String prefix) throws JSQLParserException { + String sql = prefix + "NOT NULL DEFAULT 'N' COLLATE utf8mb4_unicode_ci COMMENT 'flag'" + + (prefix.startsWith("CREATE") ? ")" : ""); + List> configurations = Arrays.asList(parser -> { + }, parser -> parser.withDialect(Dialect.MYSQL)); + for (Consumer configuration : configurations) { + Statement statement = assertSqlCanBeParsedAndDeparsed(sql, true, configuration); + ColumnDefinition column = firstColumn(statement); + assertThat(column.getColumnOptions()).extracting(ColumnOption::getKind) + .containsExactly(ColumnOption.Kind.NULLABILITY, ColumnOption.Kind.DEFAULT, + ColumnOption.Kind.COLLATE, ColumnOption.Kind.COMMENT); + assertInstanceOf(StringValue.class, defaultOption(column).getDefaultExpression()); + assertEquals("utf8mb4_unicode_ci", column.getColumnOptions().get(2).getCollation()); + assertThat(column.getColumnSpecs()).containsExactly("NOT", "NULL", "DEFAULT", "'N'", + "COLLATE", "utf8mb4_unicode_ci", "COMMENT", "'flag'"); + assertRoundTrip(statement); + } + } + + static Stream defaultValuesBeforeCollate() { + return Stream.of(Arguments.of(Dialect.MYSQL, "_utf8mb4'N'", "utf8mb4_bin"), + Arguments.of(Dialect.MYSQL, "'N'", "'utf8mb4_bin'"), + Arguments.of(Dialect.MYSQL, "1", "utf8mb4_bin"), + Arguments.of(Dialect.MYSQL, "-1", "utf8mb4_bin"), + Arguments.of(Dialect.POSTGRESQL, "'a' || 'b'", "\"C\""), + Arguments.of(Dialect.POSTGRESQL, "lower('A' COLLATE \"C\")", "\"C\""), + Arguments.of(Dialect.POSTGRESQL, "CASE WHEN true THEN 'a' END", "\"C\""), + Arguments.of(Dialect.POSTGRESQL, "'x'::text", "\"C\""), + Arguments.of(Dialect.POSTGRESQL, "'x'", "pg_catalog.\"default\"")); + } + + @ParameterizedTest + @MethodSource("defaultValuesBeforeCollate") + void defaultValueEndsBeforeTopLevelCollate(Dialect dialect, String value, String collation) + throws JSQLParserException { + String definition = " VARCHAR(30) DEFAULT " + value + " COLLATE " + collation + " NOT NULL"; + String alter = dialect == Dialect.MYSQL ? "ALTER TABLE t MODIFY COLUMN c" + : "ALTER TABLE t ADD COLUMN c"; + for (String sql : Arrays.asList("CREATE TABLE t (c" + definition + ")", + alter + definition)) { + ColumnDefinition column = firstColumn(assertSqlCanBeParsedAndDeparsed(sql, true, + parser -> parser.withDialect(dialect))); + assertThat(column.getColumnOptions()).extracting(ColumnOption::getKind) + .containsExactly(ColumnOption.Kind.DEFAULT, ColumnOption.Kind.COLLATE, + ColumnOption.Kind.NULLABILITY); + assertEquals(value, defaultOption(column).getDefaultExpression().toString()); + assertEquals(collation, column.getColumnOptions().get(1).getCollation()); + } + } + + @Test + void parenthesizedCollateStaysInTheDefaultValue() throws JSQLParserException { + ColumnDefinition column = firstColumn(assertSqlCanBeParsedAndDeparsed( + "ALTER TABLE t MODIFY COLUMN c VARCHAR(3) DEFAULT ('N' COLLATE utf8mb4_bin) NOT NULL", + true, parser -> parser.withDialect(Dialect.MYSQL))); + assertThat(column.getColumnOptions()).extracting(ColumnOption::getKind) + .containsExactly(ColumnOption.Kind.DEFAULT, ColumnOption.Kind.NULLABILITY); + Expression value = defaultOption(column).getDefaultExpression(); + assertInstanceOf(ParenthesedExpressionList.class, value); + assertInstanceOf(CollateExpression.class, ((ParenthesedExpressionList) value).get(0)); + } + + @Test + void setDefaultAndDomainDefaultsKeepTheExpressionCollate() throws JSQLParserException { + // PostgreSQL's SET DEFAULT takes an a_expr, which includes COLLATE. CREATE DOMAIN is + // unchanged. + Consumer postgres = parser -> parser.withDialect(Dialect.POSTGRESQL); + Alter table = (Alter) assertSqlCanBeParsedAndDeparsed( + "ALTER TABLE t ALTER COLUMN c SET DEFAULT 'x' COLLATE \"C\"", true, postgres); + assertInstanceOf(CollateExpression.class, table.getAlterExpressions().get(0) + .getColumnDefaults().get(0).getDefaultExpression()); + AlterRelation view = (AlterRelation) assertSqlCanBeParsedAndDeparsed( + "ALTER VIEW v ALTER COLUMN c SET DEFAULT 'x' COLLATE \"C\"", true, postgres); + assertInstanceOf(CollateExpression.class, + view.getActions().get(0).getDefaultExpression()); + AlterDomain alterDomain = (AlterDomain) assertSqlCanBeParsedAndDeparsed( + "ALTER DOMAIN d SET DEFAULT 'x' COLLATE \"C\"", true, postgres); + assertInstanceOf(CollateExpression.class, alterDomain.getDefaultExpression()); + CreateDomain createDomain = (CreateDomain) assertSqlCanBeParsedAndDeparsed( + "CREATE DOMAIN d AS text DEFAULT 'x' COLLATE \"C\"", true, postgres); + assertInstanceOf(CollateExpression.class, createDomain.getDefaultExpression()); + } + @ParameterizedTest @ValueSource(strings = {"CREATE TABLE t (a INT DEFAULT (1 + 2", "CREATE TABLE t (a INT DEFAULT (1 +))", From 36ec002658ca0f0860813e35c53ea05b83896ce7 Mon Sep 17 00:00:00 2001 From: minleejae Date: Fri, 23 Oct 2026 17:28:48 +0900 Subject: [PATCH 2/2] refactor: keep the token nesting walk in the grammar Move TokenNesting from its own file into the grammar as a private nested class next to the lookaheads that use it, as asked in review. The nested class compiles to its own class file, so its branches stay out of CCJSqlParser.class. Inlining the walk as a parser method took 18 more of the constant-pool entries JaCoCo can still instrument. Behaviour does not change. Signed-off-by: minleejae --- .../sf/jsqlparser/parser/TokenNesting.java | 38 ------------------- .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 27 +++++++++++++ 2 files changed, 27 insertions(+), 38 deletions(-) delete mode 100644 src/main/java/net/sf/jsqlparser/parser/TokenNesting.java diff --git a/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java b/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java deleted file mode 100644 index e41c6a0dd8..0000000000 --- a/src/main/java/net/sf/jsqlparser/parser/TokenNesting.java +++ /dev/null @@ -1,38 +0,0 @@ -/*- - * #%L - * JSQLParser library - * %% - * Copyright (C) 2004 - 2026 JSQLParser - * %% - * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 - * #L% - */ -package net.sf.jsqlparser.parser; - -import static net.sf.jsqlparser.parser.CCJSqlParserConstants.*; - -/** - * Token nesting checks used by grammar lookaheads. Kept outside the generated parser, whose class - * is close to the size JaCoCo can still instrument. - */ -final class TokenNesting { - private TokenNesting() {} - - /** - * Whether the target is not nested in brackets, braces, parentheses or a CASE expression opened - * from the first token on. - */ - static boolean isTopLevelFrom(Token first, Token target) { - int depth = 0; - for (Token t = first; t != null && t != target; t = t.next) { - if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image) - || "{".equals(t.image)) { - depth++; - } else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image) - || "}".equals(t.image)) { - depth--; - } - } - return depth == 0; - } -} diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 3f2ada7df9..bd3ffb657b 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -228,6 +228,33 @@ public class CCJSqlParser extends AbstractJSqlParser { && TokenNesting.isTopLevelFrom(subscriptOpenings.peek().next, colon); } + /** + * Token nesting checks shared by the lookaheads above and below. A nested class keeps its + * branches out of CCJSqlParser.class, whose constant pool is close to what JaCoCo can still + * instrument. + */ + private static final class TokenNesting { + private TokenNesting() {} + + /** + * Whether the target is not nested in brackets, braces, parentheses or a CASE expression + * opened from the first token on. + */ + static boolean isTopLevelFrom(Token first, Token target) { + int depth = 0; + for (Token t = first; t != null && t != target; t = t.next) { + if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image) + || "{".equals(t.image)) { + depth++; + } else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image) + || "}".equals(t.image)) { + depth--; + } + } + return depth == 0; + } + } + // first tokens of the enclosing column DEFAULT values: a COLLATE at their top level is the // column's own attribute, as in MySQL's column_attribute and PostgreSQL's ColQualList private final Deque columnDefaultStarts = new ArrayDeque();