From 99dd9e917ef22b7b64874e7e8ba4ceb574fbb0b0 Mon Sep 17 00:00:00 2001 From: minleejae Date: Wed, 14 Oct 2026 18:29:16 +0900 Subject: [PATCH] feat: parse PostgreSQL Unicode escape string constants With Dialect.POSTGRESQL, U&'\0041BC \00e9' was read as the column U, the & operator and a string, and U&'d!0061t!+000061' UESCAPE '!' failed, for example in COMMENT ON ... IS, column defaults and expressions. Lex U&'...' with an optional UESCAPE 'c' clause as one string literal under the PostgreSQL dialect. StringValue records it with the new POSTGRESQL_UNICODE_ESCAPE escaping and its escape character, and getNotExcapedValue() resolves \XXXX and \+XXXXXX escapes, surrogate pairs and doubled escape characters in linear time. The parser rejects the escapes and escape characters that PostgreSQL rejects. Rendering keeps the literal and its UESCAPE clause. Other dialects and the default configuration keep reading U&'x' as before. Signed-off-by: minleejae --- .../sf/jsqlparser/expression/StringValue.java | 220 +++++++++++++++++- .../util/deparser/ExpressionDeParser.java | 4 + .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 41 +++- src/site/sphinx/usage.rst | 35 +++ .../expression/StringValueTest.java | 107 +++++++++ .../parser/CCJSqlParserUtilTest.java | 78 +++++++ 6 files changed, 478 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/sf/jsqlparser/expression/StringValue.java b/src/main/java/net/sf/jsqlparser/expression/StringValue.java index 0885154a5..45afea1dd 100644 --- a/src/main/java/net/sf/jsqlparser/expression/StringValue.java +++ b/src/main/java/net/sf/jsqlparser/expression/StringValue.java @@ -25,6 +25,9 @@ public final class StringValue extends ASTNodeAccessImpl implements Expression { public static final List ALLOWED_PREFIXES = Arrays.asList("N", "U", "E", "R", "B", "RB", "_utf8", "Q"); + /** The prefix of a PostgreSQL Unicode escape string, as in {@code U&'\0041'}. */ + private static final String UNICODE_ESCAPE_PREFIX = "U&"; + /** * How a backslash in the literal body was read. The parser records the escaping of the * configuration that lexed the literal; constructed values default to {@link #STANDARD}. @@ -35,13 +38,21 @@ public enum Escaping { /** MySQL/MariaDB backslash escapes, for example {@code \n}, {@code \'} and {@code \\}. */ MYSQL_BACKSLASH, /** PostgreSQL escape string escapes, for example {@code E'\n'} and {@code E'\x41'}. */ - POSTGRESQL_ESCAPE_STRING + POSTGRESQL_ESCAPE_STRING, + /** + * PostgreSQL Unicode escapes of a {@code U&'...'} string: the escape character followed by + * four hexadecimal digits or by {@code +} and six, and a doubled escape character. The + * escape character is a backslash unless {@link #getUnicodeEscapeCharacter()} names another + * one. + */ + POSTGRESQL_UNICODE_ESCAPE } private String value = ""; private String prefix = null; private String quoteStr = "'"; private Escaping escaping = Escaping.STANDARD; + private Character unicodeEscapeCharacter = null; public StringValue() { // empty constructor @@ -70,6 +81,10 @@ public StringValue(String escapedValue) { return; } + if (readUnicodeEscapeString(escapedValue)) { + return; + } + if (escapedValue.length() > 2) { for (String p : ALLOWED_PREFIXES) { if (escapedValue.length() > p.length() @@ -85,6 +100,46 @@ public StringValue(String escapedValue) { value = escapedValue; } + /** + * Reads {@code U&'body'} with an optional {@code UESCAPE 'c'} clause. Returns false, leaving + * this value unchanged, when the text has another shape. + */ + private boolean readUnicodeEscapeString(String text) { + if (!text.regionMatches(true, 0, UNICODE_ESCAPE_PREFIX + "'", 0, 3)) { + return false; + } + int close = 3; + while (close < text.length()) { + if (text.charAt(close) != '\'') { + close++; + } else if (close + 1 < text.length() && text.charAt(close + 1) == '\'') { + close += 2; + } else { + break; + } + } + if (close >= text.length()) { + return false; + } + String clause = text.substring(close + 1).trim(); + Character escape = null; + if (!clause.isEmpty()) { + if (!clause.regionMatches(true, 0, "UESCAPE", 0, 7)) { + return false; + } + String quoted = clause.substring(7).trim(); + if (quoted.length() != 3 || quoted.charAt(0) != '\'' || quoted.charAt(2) != '\'') { + return false; + } + escape = quoted.charAt(1); + } + prefix = UNICODE_ESCAPE_PREFIX; + value = text.substring(3, close); + escaping = Escaping.POSTGRESQL_UNICODE_ESCAPE; + unicodeEscapeCharacter = escape; + return true; + } + /** * Returns the opening PostgreSQL dollar-quote delimiter, or null if there is none. A tag * follows unquoted identifier rules, excluding dollar signs. This method does not require the @@ -149,17 +204,47 @@ public StringValue withEscaping(Escaping escaping) { return this; } + /** + * Returns the escape character named by the {@code UESCAPE} clause of a PostgreSQL Unicode + * escape string, or null when the literal has no such clause and the escape character is a + * backslash. + */ + public Character getUnicodeEscapeCharacter() { + return unicodeEscapeCharacter; + } + + /** + * Sets the escape character rendered as {@code UESCAPE 'c'} after the literal; null removes the + * clause. It applies to {@link Escaping#POSTGRESQL_UNICODE_ESCAPE} values, whose + * {@link #getNotExcapedValue()} rejects a character that PostgreSQL rejects. + */ + public void setUnicodeEscapeCharacter(Character unicodeEscapeCharacter) { + this.unicodeEscapeCharacter = unicodeEscapeCharacter; + } + + public StringValue withUnicodeEscapeCharacter(Character unicodeEscapeCharacter) { + setUnicodeEscapeCharacter(unicodeEscapeCharacter); + return this; + } + /** * Returns the literal's characters with its escapes resolved: a doubled quote character and, - * depending on {@link #getEscaping()}, backslash escape sequences. PostgreSQL octal and - * hexadecimal escapes denote bytes, which are decoded as UTF-8. Dollar-quoted bodies are + * depending on {@link #getEscaping()}, backslash or Unicode escape sequences. PostgreSQL octal + * and hexadecimal escapes denote bytes, which are decoded as UTF-8. Dollar-quoted bodies are * returned unchanged. + * + * @throws IllegalArgumentException if a {@link Escaping#POSTGRESQL_UNICODE_ESCAPE} value has an + * escape that PostgreSQL rejects; the parser rejects such literals */ public String getNotExcapedValue() { if (quoteStr != null && quoteStr.startsWith("$")) { return value; } char quote = "\"".equals(quoteStr) ? '"' : '\''; + if (escaping == Escaping.POSTGRESQL_UNICODE_ESCAPE) { + return unescapeUnicode(value, quote, + unicodeEscapeCharacter == null ? '\\' : unicodeEscapeCharacter); + } Unescaped result = new Unescaped(value.length()); int i = 0; while (i < value.length()) { @@ -270,6 +355,82 @@ private static int appendNumber(Unescaped result, String text, int start, int ra return end; } + /** + * Resolves PostgreSQL Unicode escapes the way the server does: {@code escape} followed by four + * hexadecimal digits, or by {@code +} and six, names a code point, and a UTF-16 surrogate pair + * must be written as two consecutive escapes. A doubled escape or quote character stands for + * itself. + */ + private static String unescapeUnicode(String text, char quote, char escape) { + checkUnicodeEscapeCharacter(escape); + UnicodeUnescaped result = new UnicodeUnescaped(text.length()); + int i = 0; + while (i < text.length()) { + char c = text.charAt(i); + boolean doubled = i + 1 < text.length() && text.charAt(i + 1) == c; + if (c == escape && !doubled) { + i = appendUnicodeEscape(result, text, i, escape); + } else { + result.append(c, i); + i += doubled && (c == escape || c == quote) ? 2 : 1; + } + } + return result.finish(text.length()); + } + + private static void checkUnicodeEscapeCharacter(char escape) { + if (escape > 0x7F || isHexDigit(escape) || "+'\" \t\n\r\f\u000B".indexOf(escape) >= 0) { + throw new IllegalArgumentException("invalid Unicode escape character '" + escape + "'"); + } + } + + /** + * Appends the code point of the escape at {@code position}, {@code escape} followed by four + * hexadecimal digits or by {@code +} and six, and returns the next index. + */ + private static int appendUnicodeEscape(UnicodeUnescaped result, String text, int position, + char escape) { + boolean plus = position + 1 < text.length() && text.charAt(position + 1) == '+'; + int start = plus ? position + 2 : position + 1; + int end = start + (plus ? 6 : 4); + int codePoint = parseHex(text, start, end); + if (codePoint < 0) { + throw new IllegalArgumentException("invalid Unicode escape at position " + position + + ": Unicode escapes must be " + escape + "XXXX or " + escape + "+XXXXXX"); + } + if (codePoint == 0 || codePoint > Character.MAX_CODE_POINT) { + throw new IllegalArgumentException( + "invalid Unicode escape value at position " + position); + } + result.appendCodePoint(codePoint, position); + return end; + } + + private static IllegalArgumentException invalidSurrogatePair(int position) { + return new IllegalArgumentException( + "invalid Unicode surrogate pair at position " + position); + } + + private static boolean isHexDigit(char c) { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'; + } + + /** Returns the ASCII hexadecimal number spelled by text[start, end), or -1. */ + private static int parseHex(String text, int start, int end) { + if (end > text.length()) { + return -1; + } + int number = 0; + for (int i = start; i < end; i++) { + char c = text.charAt(i); + if (!isHexDigit(c)) { + return -1; + } + number = number * 16 + Character.digit(c, 16); + } + return number; + } + /** Collects unescaped characters; consecutive escaped bytes are decoded as UTF-8. */ private static final class Unescaped { private final StringBuilder chars; @@ -307,6 +468,54 @@ public String toString() { } } + /** + * Collects resolved Unicode escapes. A UTF-16 surrogate pair must be written as two consecutive + * escapes, as PostgreSQL requires. + */ + private static final class UnicodeUnescaped { + private final StringBuilder chars; + private char highSurrogate; + + UnicodeUnescaped(int capacity) { + chars = new StringBuilder(capacity); + } + + void append(char c, int position) { + requireNoPendingSurrogate(position); + chars.append(c); + } + + void appendCodePoint(int codePoint, int position) { + if (codePoint >= Character.MIN_LOW_SURROGATE + && codePoint <= Character.MAX_LOW_SURROGATE) { + if (highSurrogate == 0) { + throw invalidSurrogatePair(position); + } + chars.appendCodePoint(Character.toCodePoint(highSurrogate, (char) codePoint)); + highSurrogate = 0; + return; + } + requireNoPendingSurrogate(position); + if (codePoint >= Character.MIN_HIGH_SURROGATE + && codePoint <= Character.MAX_HIGH_SURROGATE) { + highSurrogate = (char) codePoint; + } else { + chars.appendCodePoint(codePoint); + } + } + + String finish(int position) { + requireNoPendingSurrogate(position); + return chars.toString(); + } + + private void requireNoPendingSurrogate(int position) { + if (highSurrogate != 0) { + throw invalidSurrogatePair(position); + } + } + } + @Override public T accept(ExpressionVisitor expressionVisitor, S context) { return expressionVisitor.visit(this, context); @@ -314,7 +523,10 @@ public T accept(ExpressionVisitor expressionVisitor, S context) { @Override public String toString() { - return (prefix != null ? prefix : "") + quoteStr + value + quoteStr; + String unicodeEscape = unicodeEscapeCharacter != null + ? " UESCAPE '" + unicodeEscapeCharacter + "'" + : ""; + return (prefix != null ? prefix : "") + quoteStr + value + quoteStr + unicodeEscape; } public StringValue withPrefix(String prefix) { diff --git a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java index 32a6d20ca..8183fd13a 100644 --- a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java +++ b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java @@ -715,6 +715,10 @@ public StringBuilder visit(StringValue stringValue, S context) { } builder.append(stringValue.getQuoteStr()).append(stringValue.getValue()) .append(stringValue.getQuoteStr()); + if (stringValue.getUnicodeEscapeCharacter() != null) { + builder.append(" UESCAPE '").append(stringValue.getUnicodeEscapeCharacter()) + .append('\''); + } return builder; } diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 2210dbe9d..1db1d6c5f 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -2573,7 +2573,9 @@ public class CCJSqlParser extends AbstractJSqlParser { /** Creates a string literal that remembers how this configuration lexed its backslashes. */ private StringValue stringValue(Token token) { StringValue value = new StringValue(token.image); - if (value.getQuoteStr() == null || value.getQuoteStr().startsWith("$")) { + // a U&'...' string brings its own escaping + if (value.getQuoteStr() == null || value.getQuoteStr().startsWith("$") + || value.getEscaping() == StringValue.Escaping.POSTGRESQL_UNICODE_ESCAPE) { return value; } String dialect = getAsString(Feature.dialect); @@ -2744,14 +2746,40 @@ TOKEN_MGR_DECLS : { } // Like closingQuote for a single-quoted literal after its optional prefix. Oracle q'..' - // literals have their own delimiters and are left unchanged. + // literals have their own delimiters and PostgreSQL U&'..' strings their own lexing, so + // both are left unchanged. public static int closingSingleQuote(String s, boolean backslashEscapes) { - if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\'') { + if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\'' + || isUnicodeEscapeString(s)) { return -1; } return closingQuote(s, s.indexOf('\''), '\'', backslashEscapes); } + private static boolean isUnicodeEscapeString(String s) { + return s.regionMatches(true, 0, "U&'", 0, 3); + } + + // A PostgreSQL Unicode escape string U&'...' [UESCAPE 'c'] is one literal with + // Dialect.POSTGRESQL, which rejects invalid escapes as the server does. Other dialects keep + // reading the identifier U followed by the & operator. + private void lexUnicodeEscapeString(Token token) { + if (!AbstractJSqlParser.Dialect.POSTGRESQL.name().equals(configuration.getValue(Feature.dialect))) { + truncateIdentifierToken(token, 1); + token.kind = S_IDENTIFIER; + return; + } + try { + new StringValue(token.image).getNotExcapedValue(); + } catch (IllegalArgumentException e) { + // Rewind so that every later attempt to read past the literal fails as well + input_stream.backup(token.image.length()); + throw new TokenMgrException("Lexical error at line " + token.beginLine + ", column " + + token.beginColumn + ". Invalid Unicode escape string " + token.image + ": " + + e.getMessage(), TokenMgrException.LEXICAL_ERROR); + } + } + private RowPatternTokenContext rowPatternContext; private void reportMySqlCommentError() { @@ -3722,6 +3750,10 @@ TOKEN: | ("q''" (~[])* "''") // | ("q'\\" (~[])* "\\'") <--- Does not work ) + // PostgreSQL Unicode escape string with an optional UESCAPE clause, see + // lexUnicodeEscapeString(). Backslashes do not escape its quotes. + | "U&'" ("''" | ~["'"])* "'" + ( ([" ", "\t", "\r", "\n"])* "UESCAPE" ([" ", "\t", "\r", "\n"])* "'" ~["'"] "'" )? ) > { // The longest match may run past the closing quote of the active rules. A backslash @@ -3736,6 +3768,9 @@ TOKEN: matchedToken.image = image.substring(0, end); input_stream.backup(image.length() - end); } + if (isUnicodeEscapeString(matchedToken.image)) { + lexUnicodeEscapeString(matchedToken); + } } | < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\"" // MySQL double-quoted string with backslash escapes; trimmed below when not enabled diff --git a/src/site/sphinx/usage.rst b/src/site/sphinx/usage.rst index 901608d62..4c62ea815 100644 --- a/src/site/sphinx/usage.rst +++ b/src/site/sphinx/usage.rst @@ -1413,12 +1413,47 @@ backslashes: ``\0`` and ``\Z`` become their characters, and ``\%`` and ``\_`` keep their backslash. * ``POSTGRESQL_ESCAPE_STRING`` for ``E'...'`` with ``Dialect.POSTGRESQL``, including ``\x``, octal and ``\u`` escapes; octal and hexadecimal escapes are UTF-8 bytes. +* ``POSTGRESQL_UNICODE_ESCAPE`` for ``U&'...'``, see + `PostgreSQL Unicode escape strings`_. * ``STANDARD`` otherwise, including dollar-quoted strings and constructed values; backslashes stay literal. Other dialects with backslash escapes keep this reading. ``setEscaping()`` selects the reading for a constructed value. Escaping is not part of ``equals()``, like the quote character. +PostgreSQL Unicode escape strings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With ``Dialect.POSTGRESQL``, ``U&'...'`` and an optional ``UESCAPE 'c'`` clause form one +``StringValue``. Its prefix is ``U&``, ``getValue()`` returns the body as written, and +``toString()`` renders the literal with its clause. ``getUnicodeEscapeCharacter()`` returns +the ``UESCAPE`` character, or ``null`` without the clause, when the escape character is a +backslash. ``getNotExcapedValue()`` resolves the escapes as PostgreSQL does: + +* the escape character followed by four hexadecimal digits, or by ``+`` and six, is that + code point. A character above U+FFFF can also be written as a UTF-16 surrogate pair of + two escapes; +* a doubled escape character and a doubled quote stand for themselves. + +.. code-block:: java + + Comment comment = (Comment) CCJSqlParserUtil.parse( + "COMMENT ON TABLE t IS U&'d!0061t!+000061' UESCAPE '!'", + parser -> parser.withDialect(Dialect.POSTGRESQL)); + comment.getComment().getNotExcapedValue(); // "data" + comment.toString(); // COMMENT ON TABLE t IS U&'d!0061t!+000061' UESCAPE '!' + +* The parser rejects what PostgreSQL rejects: + an escape without its digits, code point 0 or one above U+10FFFF, an unpaired surrogate, + and an escape character that is a hexadecimal digit, ``+``, a quote, whitespace or not ASCII. + ``getNotExcapedValue()`` throws ``IllegalArgumentException`` for such a constructed value. +* Without ``Dialect.POSTGRESQL``, ``U&'x'`` keeps its previous reading: the column ``U``, + the ``&`` operator and the string ``'x'``. +* ``UESCAPE`` may follow the literal after spaces, tabs or newlines, but not after a comment + or a continuation line. +* ``U&"..."`` quoted identifiers are not supported. +* The ``UESCAPE`` character is not part of ``equals()``, like the escaping. + Other dialect-specific syntax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java b/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java index 420748c5a..9f5a0c573 100644 --- a/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java +++ b/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java @@ -10,16 +10,21 @@ package net.sf.jsqlparser.expression; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import java.time.Duration; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.alter.Alter; +import net.sf.jsqlparser.statement.comment.Comment; import net.sf.jsqlparser.statement.create.table.ColumnOption; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.test.TestUtils; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -142,6 +147,108 @@ void testPostgresEscapeStrings() throws JSQLParserException { .getNotExcapedValue()); } + @Test + void testPostgresUnicodeEscapeStrings() throws JSQLParserException { + StringValue value = literal("U&'\\0041BC \\00e9'", Dialect.POSTGRESQL); + assertEquals(StringValue.Escaping.POSTGRESQL_UNICODE_ESCAPE, value.getEscaping()); + assertEquals("U&", value.getPrefix()); + assertEquals("\\0041BC \\00e9", value.getValue()); + assertNull(value.getUnicodeEscapeCharacter()); + assertEquals("ABC \u00e9", value.getNotExcapedValue()); + + StringValue custom = literal("U&'d!0061t!+000061' UESCAPE '!'", Dialect.POSTGRESQL); + assertEquals("d!0061t!+000061", custom.getValue()); + assertEquals(Character.valueOf('!'), custom.getUnicodeEscapeCharacter()); + assertEquals("data", custom.getNotExcapedValue()); + assertEquals("U&'d!0061t!+000061' UESCAPE '!'", custom.toString()); + + String[][] cases = { + {"U&'\\+01F600'", "\uD83D\uDE00"}, + {"U&'a\\\\b'", "a\\b"}, + {"U&'it''s'", "it's"}, + // a surrogate pair as two escapes, in either form + {"u&'\\D83D\\DE00'", "\uD83D\uDE00"}, + {"U&'\\d83d\\+00de00'", "\uD83D\uDE00"}, + {"U&'caf\\00E9 \\+00AC00 \uD55C\uAE00'", "caf\u00e9 \uAC00 \uD55C\uAE00"}, + // a doubled escape character stands for itself; other backslashes stay + {"U&'a!!b\\n!0021' UESCAPE '!'", "a!b\\n!"}, + {"U&'\\' UESCAPE '!'", "\\"}, + {"U&'#0041' uescape '#'", "A"}, + {"U&'x' UESCAPE '\\'", "x"}, + {"U&''", ""}}; + for (String[] c : cases) { + assertEquals(c[1], literal(c[0], Dialect.POSTGRESQL).getNotExcapedValue(), c[0]); + } + } + + @Test + void testPostgresUnicodeEscapeStringsRoundTrip() throws JSQLParserException { + for (String sql : new String[] { + "COMMENT ON TABLE t IS U&'\\0041BC \\00e9'", + "COMMENT ON TABLE t IS U&'d!0061t!+000061' UESCAPE '!'", + "CREATE TABLE t (a text DEFAULT U&'\\+01F600', b text DEFAULT U&'it''s')", + "ALTER TABLE t ALTER COLUMN a SET DEFAULT U&'a\\\\b'", + "INSERT INTO t VALUES (U&'#0041' UESCAPE '#', U&'x')", + "SELECT * FROM t WHERE a = U&'\\00e9' AND b LIKE U&'%!0041' UESCAPE '!'"}) { + Statement statement = TestUtils.assertSqlCanBeParsedAndDeparsed(sql, false, + parser -> parser.withDialect(Dialect.POSTGRESQL)); + Statement reparsed = CCJSqlParserUtil.parse(statement.toString(), + parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertEquals(sql, reparsed.toString()); + } + Comment comment = (Comment) CCJSqlParserUtil.parse( + "COMMENT ON TABLE t IS U&'d!0061t!+000061' UESCAPE '!'", + parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertEquals("data", comment.getComment().getNotExcapedValue()); + // a continuation line belongs to the same Unicode escape string, as on the server + StringValue continued = literal("U&'\\0041'\n'\\0042'", Dialect.POSTGRESQL); + assertEquals("AB", continued.getNotExcapedValue()); + assertEquals("U&'\\0041\\0042'", continued.toString()); + } + + @Test + void testConstructedUnicodeEscapeStrings() { + StringValue value = new StringValue("U&'!0041' UESCAPE '!'"); + assertEquals("U&", value.getPrefix()); + assertEquals("!0041", value.getValue()); + assertEquals(StringValue.Escaping.POSTGRESQL_UNICODE_ESCAPE, value.getEscaping()); + assertEquals("A", value.getNotExcapedValue()); + assertEquals("U&'!0041' UESCAPE '!'", value.toString()); + assertEquals("U&'!0041'", value.withUnicodeEscapeCharacter(null).toString()); + assertEquals("!0041", value.getNotExcapedValue()); + + StringValue built = new StringValue("'#00e9'").withPrefix("U&") + .withEscaping(StringValue.Escaping.POSTGRESQL_UNICODE_ESCAPE) + .withUnicodeEscapeCharacter('#'); + assertEquals("U&'#00e9' UESCAPE '#'", built.toString()); + assertEquals("\u00e9", built.getNotExcapedValue()); + ExpressionDeParser deParser = new ExpressionDeParser(); + built.accept(deParser, null); + assertEquals("U&'#00e9' UESCAPE '#'", deParser.getBuilder().toString()); + + assertThrows(IllegalArgumentException.class, + () -> new StringValue("U&'\\00'").getNotExcapedValue()); + assertThrows(IllegalArgumentException.class, + () -> new StringValue("U&'x' UESCAPE '+'").getNotExcapedValue()); + // without the escaping a U& prefix only renders + assertEquals("\\0041", new StringValue("'\\0041'").withPrefix("U&").getNotExcapedValue()); + } + + @Test + void testUnicodeUnescapingIsLinear() { + int escapes = 500_000; + StringBuilder sb = new StringBuilder(13 * escapes + 4).append("U&'"); + for (int i = 0; i < escapes; i++) { + sb.append("\\0041\\+01F600"); + } + StringValue instance = new StringValue(sb.append('\'').toString()); + + String result = assertTimeoutPreemptively(Duration.ofSeconds(2), + instance::getNotExcapedValue); + + assertEquals(3 * escapes, result.length()); + } + @Test void testOtherBackslashDialectsKeepTheirSource() throws JSQLParserException { for (Dialect dialect : new Dialect[] {Dialect.BIGQUERY, Dialect.DATABRICKS, diff --git a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java index c8b797120..bf74eb6a3 100644 --- a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java +++ b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java @@ -14,6 +14,7 @@ import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.arithmetic.Addition; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseAnd; import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication; import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; import net.sf.jsqlparser.schema.Column; @@ -856,6 +857,83 @@ public void testSingleQuotedStringsMixBackslashEscapesAndDoubledQuotes() throws .parse("SELECT * FROM t WHERE a LIKE 'x\\_%' ESCAPE '\\'").toString()); } + @Test + public void testPostgresUnicodeEscapeStringsAreOneLiteral() throws Exception { + Consumer postgres = + p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL); + for (String literal : new String[] {"U&'\\0041BC \\00e9'", + "U&'d!0061t!+000061' UESCAPE '!'", "U&'it''s'", "U&'a\\\\b'", + "U&'\\' UESCAPE '!'", "U&'x' UESCAPE '\\'"}) { + PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse( + "SELECT " + literal + " FROM t", postgres)).getSelectBody(); + assertEquals(1, select.getSelectItems().size(), literal); + assertNull(select.getSelectItems().get(0).getAlias(), literal); + assertInstanceOf(StringValue.class, select.getSelectItems().get(0).getExpression()); + assertEquals("SELECT " + literal + " FROM t", select.toString()); + } + // the prefix is case-insensitive; UESCAPE may follow without or after any whitespace + assertEquals("SELECT U&'#0041' UESCAPE '#', U&'x' FROM t", CCJSqlParserUtil.parse( + "SELECT u&'#0041'\n\tuescape\n'#', U&'x'FROM t", postgres).toString()); + // spaces inside U&' make it an expression + assertInstanceOf(BitwiseAnd.class, ((PlainSelect) CCJSqlParserUtil + .parse("SELECT U & 'x' FROM t", postgres)).getSelectItem(0).getExpression()); + assertInstanceOf(BitwiseAnd.class, ((PlainSelect) CCJSqlParserUtil + .parse("SELECT U &'x' FROM t", postgres)).getSelectItem(0).getExpression()); + } + + @Test + public void testUnicodeEscapeStringsNeedThePostgresDialect() throws Exception { + for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {null, + AbstractJSqlParser.Dialect.MYSQL, AbstractJSqlParser.Dialect.ORACLE, + AbstractJSqlParser.Dialect.SQLSERVER, AbstractJSqlParser.Dialect.ANSI_SQL}) { + Consumer consumer = p -> { + if (dialect != null) { + p.withDialect(dialect); + } + }; + for (String sql : new String[] {"SELECT U&'x' FROM t", + "SELECT u&'\\0041' FROM t"}) { + BitwiseAnd and = assertInstanceOf(BitwiseAnd.class, ((PlainSelect) CCJSqlParserUtil + .parse(sql, consumer)).getSelectItem(0).getExpression(), sql); + assertInstanceOf(Column.class, and.getLeftExpression(), sql); + assertNull(assertInstanceOf(StringValue.class, and.getRightExpression(), sql) + .getPrefix(), sql); + } + for (String sql : new String[] {"SELECT a & b FROM t", "SELECT U & 1 FROM t", + "SELECT U&1 FROM t", "SELECT U & 'x' FROM t"}) { + assertInstanceOf(BitwiseAnd.class, ((PlainSelect) CCJSqlParserUtil.parse(sql, + consumer)).getSelectItem(0).getExpression(), sql); + } + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil + .parse("SELECT U&'d!0061' UESCAPE '!' FROM t", consumer)); + // literals that merely start with & are unaffected + assertEquals("SELECT '&', '&x', N'&', 'a' & 'b' FROM t", CCJSqlParserUtil + .parse("SELECT '&', '&x', N'&', 'a' & 'b' FROM t", consumer).toString()); + } + // the bitwise reading also stays for PostgreSQL's U & 1 and a & b + for (String sql : new String[] {"SELECT a & b FROM t", "SELECT U & 1 FROM t", + "SELECT U&1 FROM t", "SELECT '&' & 'b' FROM t"}) { + assertInstanceOf(BitwiseAnd.class, ((PlainSelect) CCJSqlParserUtil.parse(sql, + p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL))) + .getSelectItem(0).getExpression(), sql); + } + } + + @Test + public void testInvalidUnicodeEscapeStringsAreRejected() { + for (String literal : new String[] {"U&'\\00'", "U&'\\0000'", "U&'\\+110000'", + "U&'\\+12'", "U&'\\D800'", "U&'\\DC00'", "U&'\\D800x'", + "U&'\\D800\\0041'", "U&'\\D800\\\\'", "U&'x\\'", "U&'\\g000'", + "U&'x' UESCAPE '+'", "U&'x' UESCAPE 'a'", "U&'x' UESCAPE 'F'", + "U&'x' UESCAPE ' '", "U&'x' UESCAPE '\"'", "U&'x' UESCAPE '\u00e9'"}) { + for (String sql : new String[] {"SELECT " + literal, "SELECT f(" + literal + ")", + "COMMENT ON TABLE t IS " + literal}) { + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql, + p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL)), sql); + } + } + } + @Test public void testAdjacentStringLiteralsNewline() throws Exception { // default (off): the MySQL / SQL Server alias reading, unchanged