diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java index 2e48e0b82582..45a1cb1d97a2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java @@ -26,11 +26,9 @@ import org.apache.paimon.types.DataTypeFamily; import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.StringUtils; import java.util.ArrayList; import java.util.List; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,52 +112,15 @@ private String extractArrayContent(String str) { private List parseArrayElements( String content, CastExecutor elementCastExecutor) { List elements = new ArrayList<>(); - for (String token : splitArrayElements(content)) { - String trimmedToken = token.trim(); + for (TokenSplitter.Token token : TokenSplitter.split(content)) { + String value = token.value(); + // only an unquoted null is the null element; "null" is the four-character string Object element = - "null".equals(trimmedToken) + !token.literal() && "null".equals(value) ? null - : elementCastExecutor.cast(BinaryString.fromString(trimmedToken)); + : elementCastExecutor.cast(BinaryString.fromString(value)); elements.add(element); } return elements; } - - private List splitArrayElements(String content) { - List elements = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - Stack bracketStack = new Stack<>(); - boolean inQuotes = false; - boolean escaped = false; - - for (char c : content.toCharArray()) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - inQuotes = !inQuotes; - } else if (!inQuotes) { - if (StringUtils.isOpenBracket(c)) { - bracketStack.push(c); - } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { - bracketStack.pop(); - } else if (c == ',' && bracketStack.isEmpty()) { - addCurrentElement(elements, current); - continue; - } - } - current.append(c); - } - - addCurrentElement(elements, current); - return elements; - } - - private void addCurrentElement(List elements, StringBuilder current) { - if (current.length() > 0) { - elements.add(current.toString()); - current.setLength(0); - } - } } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java index 5a2379d273a2..1ceaf1c2bc60 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java @@ -26,11 +26,8 @@ import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.StringUtils; -import java.util.ArrayList; import java.util.List; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -93,7 +90,7 @@ private InternalRow parseRow( if (content.isEmpty()) { return createNullRow(fieldCount); } - List fieldValues = splitRowFields(content); + List fieldValues = TokenSplitter.split(content); if (fieldValues.size() != fieldCount) { throw new RuntimeException( "Row field count mismatch. Expected: " @@ -137,60 +134,22 @@ private GenericRow createNullRow(int fieldCount) { } private GenericRow createRowFromFields( - List fieldValues, + List fieldValues, CastExecutor[] fieldCastExecutors, int fieldCount) { GenericRow row = new GenericRow(fieldCount); for (int i = 0; i < fieldCount; i++) { - String fieldValue = fieldValues.get(i).trim(); - Object value = parseFieldValue(fieldValue, fieldCastExecutors[i]); - row.setField(i, value); + row.setField(i, parseFieldValue(fieldValues.get(i), fieldCastExecutors[i])); } return row; } private Object parseFieldValue( - String fieldValue, CastExecutor castExecutor) { - return "null".equals(fieldValue) + TokenSplitter.Token token, CastExecutor castExecutor) { + String value = token.value(); + // only an unquoted null is the null field; "null" is the four-character string + return !token.literal() && "null".equals(value) ? null - : castExecutor.cast(BinaryString.fromString(fieldValue)); - } - - private List splitRowFields(String content) { - List fields = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - Stack bracketStack = new Stack<>(); - boolean inQuotes = false; - boolean escaped = false; - - for (char c : content.toCharArray()) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - inQuotes = !inQuotes; - } else if (!inQuotes) { - if (StringUtils.isOpenBracket(c)) { - bracketStack.push(c); - } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { - bracketStack.pop(); - } else if (c == ',' && bracketStack.isEmpty()) { - addCurrentField(fields, current); - continue; - } - } - current.append(c); - } - - addCurrentField(fields, current); - return fields; - } - - private void addCurrentField(List fields, StringBuilder current) { - if (current.length() > 0) { - fields.add(current.toString()); - current.setLength(0); - } + : castExecutor.cast(BinaryString.fromString(value)); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java new file mode 100644 index 000000000000..54059675db1d --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.casting; + +import org.apache.paimon.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +/** + * Splits the comma-separated body of an array or row literal into its tokens, honouring quotes, + * escapes and nesting. + * + *

A separator only separates outside quotes and at bracket depth zero, so {@code "a,b"} and + * {@code [a, b]} each stay one token. + * + *

Quotes and backslashes are this level's syntax, so they are removed at depth zero and kept + * verbatim inside a nested literal, where they are the inner level's syntax and the rule for that + * element parses them again. Whether they appeared at depth zero is remembered: quoting or escaping + * is how the literal text {@code null} and the empty string are written, which are otherwise + * unrepresentable. + * + *

Whitespace around a token is dropped; whitespace inside quotes is kept. + */ +class TokenSplitter { + + /** One token of a literal body, plus whether its value was written as a literal. */ + static class Token { + + private final String value; + private final boolean literal; + + Token(String value, boolean literal) { + this.value = value; + this.literal = literal; + } + + String value() { + return value; + } + + /** Whether quotes or an escape made this a literal string rather than a bare word. */ + boolean literal() { + return literal; + } + } + + private TokenSplitter() {} + + static List split(String content) { + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + Stack bracketStack = new Stack<>(); + boolean inQuotes = false; + boolean escaped = false; + boolean literal = false; + // length of current up to the last character that was not unquoted whitespace + int end = 0; + + for (char c : content.toCharArray()) { + boolean nested = !bracketStack.isEmpty(); + if (escaped) { + // the escapee stands for itself and is never read as syntax + escaped = false; + current.append(c); + end = current.length(); + continue; + } + if (c == '\\') { + escaped = true; + if (nested) { + // the inner rule has to see the escape to protect its own separators + current.append(c); + end = current.length(); + } else { + literal = true; + } + continue; + } + if (c == '"') { + inQuotes = !inQuotes; + if (nested) { + current.append(c); + end = current.length(); + } else { + literal = true; + } + continue; + } + if (!inQuotes) { + if (StringUtils.isOpenBracket(c)) { + bracketStack.push(c); + } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { + bracketStack.pop(); + } else if (c == ',' && bracketStack.isEmpty()) { + addToken(tokens, current, end, literal); + current.setLength(0); + end = 0; + literal = false; + continue; + } else if (Character.isWhitespace(c) && end == 0) { + // leading whitespace outside quotes is not part of the token + continue; + } + } + current.append(c); + if (inQuotes || !Character.isWhitespace(c)) { + end = current.length(); + } + } + + addToken(tokens, current, end, literal); + return tokens; + } + + private static void addToken( + List tokens, StringBuilder current, int end, boolean literal) { + // whitespace is not part of a token, so one made only of whitespace was never written; + // quoting is how an empty value is written + if (end > 0 || literal) { + tokens.add(new Token(current.substring(0, end), literal)); + } + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index 31a776c87da3..8325687c21b2 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -975,6 +975,175 @@ public void testRowToString() { BinaryString.fromString("{1, {2025-01-06, {1 -> [1, null, 2]}, null}}")); } + @Test + public void testStringToArrayQuotingAndEscaping() { + ArrayType arrayType = new ArrayType(DataTypes.STRING()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // quotes group a token across the separator and do not survive into the value + compareCastResult( + cast, + BinaryString.fromString("[\"a,b\", c]"), + new GenericArray( + new Object[] { + BinaryString.fromString("a,b"), BinaryString.fromString("c") + })); + + // quoting is how an empty string is written, so the element must be kept + compareCastResult( + cast, + BinaryString.fromString("[\"\", a]"), + new GenericArray( + new Object[] {BinaryString.fromString(""), BinaryString.fromString("a")})); + + // an unquoted null is the null element; a quoted one is the four-character string + compareCastResult( + cast, + BinaryString.fromString("[null, \"null\"]"), + new GenericArray(new Object[] {null, BinaryString.fromString("null")})); + + // a backslash escapes the next character and is itself syntax + compareCastResult( + cast, + BinaryString.fromString("[a\\,b, c]"), + new GenericArray( + new Object[] { + BinaryString.fromString("a,b"), BinaryString.fromString("c") + })); + } + + @Test + public void testStringToRowQuotingAndEscaping() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "f0", DataTypes.STRING()), + DataTypes.FIELD(1, "f1", DataTypes.INT())); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, rowType); + + compareCastResult( + cast, + BinaryString.fromString("{\"a,b\", 2}"), + GenericRow.of(BinaryString.fromString("a,b"), 2)); + + // an empty quoted field stays a field, so the field count still matches + compareCastResult( + cast, + BinaryString.fromString("{\"\", 2}"), + GenericRow.of(BinaryString.fromString(""), 2)); + + // a quoted null is the string, an unquoted one is SQL NULL + compareCastResult( + cast, + BinaryString.fromString("{\"null\", 2}"), + GenericRow.of(BinaryString.fromString("null"), 2)); + compareCastResult(cast, BinaryString.fromString("{null, 2}"), GenericRow.of(null, 2)); + + compareCastResult( + cast, + BinaryString.fromString("{a\\,b, 2}"), + GenericRow.of(BinaryString.fromString("a,b"), 2)); + } + + @Test + public void testStringToNestedArrayKeepsInnerSyntax() { + // quotes and escapes belong to whichever level wrote them: the outer split must leave a + // nested literal's own syntax in place for the element rule to parse again, or the inner + // separator stops being protected and the element count changes + ArrayType nested = new ArrayType(new ArrayType(DataTypes.STRING())); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, nested); + + assertNestedElements(cast, "[[\"a,b\"], [c]]", new String[] {"a,b"}, new String[] {"c"}); + assertNestedElements(cast, "[[a\\,b], [c]]", new String[] {"a,b"}, new String[] {"c"}); + assertNestedElements(cast, "[[\"null\"], [a]]", new String[] {"null"}, new String[] {"a"}); + assertNestedElements(cast, "[[\"\"], [a]]", new String[] {""}, new String[] {"a"}); + assertNestedElements(cast, "[[\" a \"], [b]]", new String[] {" a "}, new String[] {"b"}); + assertNestedElements(cast, "[[1, 2], [3]]", new String[] {"1", "2"}, new String[] {"3"}); + } + + private static void assertNestedElements( + CastExecutor cast, String literal, String[]... expected) { + InternalArray outer = cast.cast(BinaryString.fromString(literal)); + assertThat(outer.size()).as("outer size of %s", literal).isEqualTo(expected.length); + for (int i = 0; i < expected.length; i++) { + InternalArray inner = outer.getArray(i); + assertThat(inner.size()) + .as("inner size of %s at %s", literal, i) + .isEqualTo(expected[i].length); + for (int j = 0; j < expected[i].length; j++) { + assertThat(inner.getString(j).toString()) + .as("element %s.%s of %s", i, j, literal) + .isEqualTo(expected[i][j]); + } + } + } + + @Test + public void testStringToArrayEscapedNullIsALiteral() { + ArrayType arrayType = new ArrayType(DataTypes.STRING()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // escaping, like quoting, says the token is written text rather than the null literal + compareCastResult( + cast, + BinaryString.fromString("[\\null, x]"), + new GenericArray( + new Object[] { + BinaryString.fromString("null"), BinaryString.fromString("x") + })); + } + + @Test + public void testStringToRowWhitespaceOnlyFieldIsNotAField() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "f0", DataTypes.STRING()), + DataTypes.FIELD(1, "f1", DataTypes.STRING()), + DataTypes.FIELD(2, "f2", DataTypes.STRING())); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, rowType); + + // whitespace is not part of a token, so a field made only of whitespace was never + // written, and where it sits does not change that + for (String literal : new String[] {"{a, ,b}", "{ ,a,b}", "{a,b, }"}) { + assertThatThrownBy(() -> cast.cast(BinaryString.fromString(literal))) + .as("%s", literal) + .hasMessageContaining("Row field count mismatch. Expected: 3, Actual: 2"); + } + + // quoting is how an empty field is written + compareCastResult( + cast, + BinaryString.fromString("{a, \"\", b}"), + GenericRow.of( + BinaryString.fromString("a"), + BinaryString.fromString(""), + BinaryString.fromString("b"))); + } + + @Test + public void testStringToArraySkipsAnEmptyElement() { + ArrayType arrayType = new ArrayType(DataTypes.INT()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // an element written as nothing, with or without whitespace, is no element: handing the + // empty string to the int cast instead would fail the whole array + compareCastResult( + cast, BinaryString.fromString("[1,,3]"), new GenericArray(new Integer[] {1, 3})); + compareCastResult( + cast, BinaryString.fromString("[1, , 3]"), new GenericArray(new Integer[] {1, 3})); + } + @Test public void testSplitMapEntriesWithQuotes() { String content = "1, \"abc\"";