Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -3273,7 +3273,7 @@ public Map<Integer, String> statsModePerLevel() {
}

public static String normalizeFileFormat(String fileFormat) {
return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase();
return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase(Locale.ROOT);
}

public String dataFilePrefix() {
Expand Down Expand Up @@ -4238,7 +4238,10 @@ public String partitionMarkDoneCustomClass() {

public Set<PartitionMarkDoneAction> partitionMarkDoneActions() {
return Arrays.stream(options.get(PARTITION_MARK_DONE_ACTION).split(","))
.map(x -> PartitionMarkDoneAction.valueOf(x.replace('-', '_').toUpperCase()))
.map(
x ->
PartitionMarkDoneAction.valueOf(
x.replace('-', '_').toUpperCase(Locale.ROOT)))
.collect(Collectors.toCollection(HashSet::new));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ static Boolean convertToBoolean(Object o) {
return (Boolean) o;
}

switch (o.toString().toUpperCase()) {
switch (o.toString().toUpperCase(Locale.ROOT)) {
case "TRUE":
return true;
case "FALSE":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
Expand Down Expand Up @@ -221,7 +222,9 @@ private static String extractRequestId(ClassicHttpResponse response) {
.filter(
h ->
h.getName() != null
&& h.getName().toLowerCase().contains("request-id"))
&& h.getName()
.toLowerCase(Locale.ROOT)
.contains("request-id"))
.map(Header::getValue)
.filter(Objects::nonNull)
.findFirst()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.paimon.rest.RESTCatalogOptions;
import org.apache.paimon.utils.StringUtils;

import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand Down Expand Up @@ -96,7 +97,7 @@ protected static String parseSigningAlgoFromUri(String uri) {
}

// Check for aliyun openapi endpoints
if (uri.toLowerCase().contains("dlfnext")) {
if (uri.toLowerCase(Locale.ROOT).contains("dlfnext")) {
return DLFOpenApiSigner.IDENTIFIER;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;

Expand Down Expand Up @@ -62,12 +63,12 @@ public class DLFDefaultSigner implements DLFRequestSigner {
private static final String NEW_LINE = "\n";
private static final List<String> SIGNED_HEADERS =
Arrays.asList(
DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(),
DLF_CONTENT_TYPE_KEY.toLowerCase(),
DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(),
DLF_DATE_HEADER_KEY.toLowerCase(),
DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(),
DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase());
DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(Locale.ROOT),
DLF_CONTENT_TYPE_KEY.toLowerCase(Locale.ROOT),
DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(Locale.ROOT),
DLF_DATE_HEADER_KEY.toLowerCase(Locale.ROOT),
DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(Locale.ROOT),
DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase(Locale.ROOT));

private final String region;

Expand Down Expand Up @@ -215,7 +216,7 @@ private static TreeMap<String, String> buildSortedSignedHeadersMap(
TreeMap<String, String> orderMap = new TreeMap<>();
if (headers != null) {
for (Map.Entry<String, String> header : headers.entrySet()) {
String key = header.getKey().toLowerCase();
String key = header.getKey().toLowerCase(Locale.ROOT);
if (SIGNED_HEADERS.contains(key)) {
orderMap.put(key, StringUtils.trim(header.getValue()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ public String identifier() {
private static String buildCanonicalizedHeaders(Map<String, String> headers) {
TreeMap<String, String> sortedHeaders = new TreeMap<>();
for (Map.Entry<String, String> entry : headers.entrySet()) {
String key = entry.getKey().toLowerCase();
String key = entry.getKey().toLowerCase(Locale.ROOT);
if (key.startsWith("x-acs-")) {
sortedHeaders.put(key, StringUtils.trim(entry.getValue()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -213,7 +214,7 @@ private static List<Token> tokenize(String chars) {
builder.setLength(0);
cursor = consumeIdentifier(builder, chars, cursor);
final String token = builder.toString();
final String normalizedToken = token.toUpperCase();
final String normalizedToken = token.toUpperCase(Locale.ROOT);
if (KEYWORDS.contains(normalizedToken)) {
tokens.add(new Token(TokenType.KEYWORD, cursor, normalizedToken));
} else {
Expand Down Expand Up @@ -344,7 +345,7 @@ private enum Keyword {

private static final Set<String> KEYWORDS =
Stream.of(Keyword.values())
.map(k -> k.toString().toUpperCase())
.map(k -> k.toString().toUpperCase(Locale.ROOT))
.collect(Collectors.toSet());

private static class Token {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

import org.apache.paimon.annotation.Public;

import java.util.Locale;

/**
* Lists all kinds of changes that a row can describe in a changelog.
*
Expand Down Expand Up @@ -135,7 +137,7 @@ public static RowKind fromByteValue(byte value) {
* @see #shortString() for mapping of string and {@link RowKind}.
*/
public static RowKind fromShortString(String value) {
switch (value.toUpperCase()) {
switch (value.toUpperCase(Locale.ROOT)) {
case "+I":
return INSERT;
case "-U":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
Expand Down Expand Up @@ -668,7 +669,9 @@ public static String quote(String str) {
}

public static String toLowerCaseIfNeed(String str, boolean caseSensitive) {
return caseSensitive ? str : str.toLowerCase();
// Locale.ROOT: identifier matching must not depend on the JVM default locale
// (e.g. Turkish lowercases 'I' to a dotless glyph and breaks column mapping)
return caseSensitive ? str : str.toLowerCase(Locale.ROOT);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Normalize CDC key lists with the same locale as fields

buildPaimonSchema uses this helper for field names, but CdcActionCommonUtils.listCaseConvert still maps String::toLowerCase for source/configured primary keys and partition keys. With Locale tr-TR and a case-insensitive catalog, a fresh source column ID with primary key ID now becomes field id and key ıd, and Schema rejects the table. In the non-strict database-sync path, fields id/CITY with configured partition CITY instead become fields [id, city] and partitionKeys=[], silently dropping the requested partitioning.

I compiled the exact helper and CDC schema builder and exercised both paths: this head fails/drops the partition as above; the exact-base helper consistently produces [ıd]/[ıd] and [id, cıty]/[cıty]. JDBC metadata supplies source column/key names without an earlier normalization, so these inputs are reachable. Please convert listCaseConvert with the same explicit locale and add Turkish schema tests for inferred/configured keys and non-strict partition handling. The audit must include method references such as String::toLowerCase, which a search for .toLowerCase() misses.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in d741c20.

You are right about the audit gap: I matched .toLowerCase() textually, so every String::toLowerCase method reference stayed on the default locale. listCaseConvert is the one with a correctness consequence, and I reproduced both paths you describe before changing anything.

The same search over main sources turned up six more references in that form, all machine tokens, so they are converted in the same commit: TypeMapping.parse (an upper-case --type-mapping value stops matching a mode whose name contains an i, e.g. TINYINT1-NOT-BOOL), the Kafka offset-reset hint in KafkaActionUtils, Hive partition key names in PaimonMetaHook, predicate-pushdown column names in SearchArgumentToPredicateConverter, and option keys in FileIO. grep -rn '::toLowerCase\|::toUpperCase' over src/main is now empty.

TurkishLocaleSchemaKeyTest covers buildPaimonSchema under tr-TR: a primary key inferred from the source schema, a specified primary key under both strict and non-strict checking, and a specified partition key under non-strict checking, plus the type-mapping case. On the parent commit the four schema cases fail (3 assertion failures, 1 error); on this head all five pass. Also ran TurkishLocaleTypeNameTest, CdcRecordTest, FileIOTest, StringUtilsTest and SearchArgumentToPredicateConverterTest: green.

On the persisted-value half of your review: this patch does not touch UpperTransform or LowerTransform, so no computed column changes value here. I have added your point to the PR body, since it applies to the disclosed migration rather than to a code path this patch changes: where upper/lower output is part of a primary or partition key, resuming a job after this change can address a different key, so those tables need a controlled rewrite or preserved legacy semantics rather than a restart.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Corrections to my numbers above, and two follow-up commits.

The failure counts I posted were from an intermediate state, where only listCaseConvert was reverted and the type-mapping case did not exist yet. Re-measured against the current test:

  • parent-equivalent tree (the conversions reachable from these tests reverted): all five cases fail, 2 assertion failures and 3 exceptions (IllegalStateException from Schema's own allFields.containsAll(primaryKeys) check, IllegalArgumentException from setPrimaryKeys, UnsupportedOperationException from TypeMappingMode.mode);
  • only listCaseConvert reverted: the four schema cases fail, the type-mapping case passes;
  • only TypeMapping.parse reverted: only the type-mapping case fails;
  • head: 5/5 pass.

620f15f came out of reviewing that test. specifiedPrimaryKeyPassesStrictChecking asserted only doesNotThrowAnyException(), which cannot distinguish "the strict path accepted the key" from "it accepted the key and stored a different one"; it now asserts the resulting primary keys. The class is renamed TurkishLocaleCaseFoldingTest, since the fifth case is an option value rather than a schema key.

I also owe you a correction on UpperTransform / LowerTransform. I justified leaving them alone as SQL semantics, which is weaker than the actual reason: they fold a BinaryString, not a java.lang.String. The ASCII paths use Character.toUpperCase(int) / toLowerCase(int) (BinaryString.java:598,632) and the non-ASCII fallbacks are already toString().toUpperCase(Locale.ROOT) / toLowerCase(Locale.ROOT) (:609-611, :643-645). This patch touches neither file, so there is no default-locale dependence there to remove and no computed-column value changes with it. For contrast, Spark's upper under the binary collation reaches UTF8String.toUpperCaseSlow(), which is an unpinned toString().toUpperCase(), but only for non-ASCII input: full-ASCII strings take toUpperCaseAscii(), so upper('istanbul') looks the same either way.

On the completeness question your last paragraph raises: I enumerated the spellings rather than searching for one. Nothing is left in any src/main for no-arg .toLowerCase()/.toUpperCase(), method references on any receiver (zero repo-wide now, tests included), Locale.getDefault(), %S/%T format conversions, Commons/Guava/ICU case helpers, java.text.Collator, Normalizer, Scala's paren-less and .capitalize forms, or valueOf(x.toUpperCase(...))-style enum folding. Six explicit Locale.US conversions remain, in MemorySize, TimeUtils, HadoopFileIO and FlinkFileIO, all on machine tokens; the JDK applies special casing only for the language codes tr, az and lt, so those are byte-identical to ROOT for every input (checked over every defined code point: zero differences for Locale.US, differences under tr-TR). equalsIgnoreCase, CASE_INSENSITIVE_ORDER, regionMatches(true, ...) and Pattern.CASE_INSENSITIVE do not consult the default locale, so they are out of scope.

One judgement call worth naming: paimon-api's StringUtils.toLowerCase is already ROOT-pinned and null-safe, so StringUtils::toLowerCase would have been a literal drop-in for String::toLowerCase. I used inline lambdas instead, because that helper maps null to null and would turn today's NPE on a null key element into a null sitting in a key list. Happy to switch if you prefer the shared helper.

}

public static boolean isNumeric(final CharSequence cs) {
Expand Down Expand Up @@ -733,14 +736,14 @@ public static String toUpperCase(String value) {
if (value == null) {
return null;
}
return value.toUpperCase();
return value.toUpperCase(Locale.ROOT);
}

public static String toLowerCase(String value) {
if (value == null) {
return null;
}
return value.toLowerCase();
return value.toLowerCase(Locale.ROOT);
}

public static boolean isOpenBracket(char c) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.paimon;

import org.apache.paimon.options.Options;
import org.apache.paimon.types.RowKind;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.util.Locale;

import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_ACTION;
import static org.assertj.core.api.Assertions.assertThat;

/**
* Parsing an option value, an enum name or a protocol token uppercases or lowercases it first.
* Under a Turkish default locale 'i' maps to a dotted capital and 'I' to a dotless small letter, so
* those conversions must pin {@link Locale#ROOT} or the token no longer matches what it is compared
* against.
*/
class TurkishLocaleParsingTest {

private Locale original;

@BeforeEach
void setUp() {
original = Locale.getDefault();
Locale.setDefault(new Locale("tr", "TR"));
}

@AfterEach
void tearDown() {
Locale.setDefault(original);
}

@Test
void partitionMarkDoneActionsParse() {
// SUCCESS_FILE and DONE_PARTITION both contain an 'i': a locale-sensitive uppercase
// turns them into names no enum constant has, and valueOf throws
Options options = new Options();
options.set(PARTITION_MARK_DONE_ACTION, "success-file,done-partition");

assertThat(new CoreOptions(options).partitionMarkDoneActions())
.containsExactlyInAnyOrder(
CoreOptions.PartitionMarkDoneAction.SUCCESS_FILE,
CoreOptions.PartitionMarkDoneAction.DONE_PARTITION);
}

@Test
void rowKindFromLowerCaseShortString() {
// "+i" is the only short string this can catch: Turkish differs from ROOT on 'i' and
// 'I' alone, so "-d" or "-u" would pass whichever conversion the code uses
assertThat(RowKind.fromShortString("+i")).isEqualTo(RowKind.INSERT);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.paimon.benchmark.metric.cpu;

import java.util.Locale;

/** An enumeration indicating the operating system that the JVM runs on. */
public enum OperatingSystem {
LINUX,
Expand Down Expand Up @@ -115,7 +117,7 @@ private static OperatingSystem readOSFromSystemProperties() {
if (osName.startsWith(FREEBSD_OS_PREFIX)) {
return FREE_BSD;
}
String osNameLowerCase = osName.toLowerCase();
String osNameLowerCase = osName.toLowerCase(Locale.ROOT);
if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1)
|| osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) {
return SOLARIS;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

import javax.annotation.Nullable;

import java.util.Locale;

/**
* Each compression codec has an implementation of {@link BlockCompressionFactory} to create
* compressors and decompressors.
Expand All @@ -38,7 +40,7 @@ public interface BlockCompressionFactory {
/** Creates {@link BlockCompressionFactory} according to the configuration. */
@Nullable
static BlockCompressionFactory create(CompressOptions compression) {
switch (compression.compress().toUpperCase()) {
switch (compression.compress().toUpperCase(Locale.ROOT)) {
case "NONE":
return null;
case "ZSTD":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;

Expand Down Expand Up @@ -88,16 +89,19 @@ public static FileFormat fromIdentifier(String identifier, Options options) {
/** Create a {@link FileFormat} from format identifier and format options. */
public static FileFormat fromIdentifier(String identifier, FormatContext context) {
return FormatFactoryUtil.discoverFactory(
FileFormat.class.getClassLoader(), identifier.toLowerCase())
FileFormat.class.getClassLoader(), identifier.toLowerCase(Locale.ROOT))
.create(context);
}

protected Options getIdentifierPrefixOptions(Options options) {
Map<String, String> result = new HashMap<>();
String prefix = formatIdentifier.toLowerCase() + ".";
// match against the identifier as written so the suffix is sliced at an offset the key
// actually has: lower-casing can lengthen a string, and U+0130 lower-cases to two chars
String prefix = formatIdentifier + ".";
String lowerCasePrefix = formatIdentifier.toLowerCase(Locale.ROOT) + ".";
for (String key : options.keySet()) {
if (key.toLowerCase().startsWith(prefix)) {
result.put(prefix + key.substring(prefix.length()), options.get(key));
if (key.regionMatches(true, 0, prefix, 0, prefix.length())) {
result.put(lowerCasePrefix + key.substring(prefix.length()), options.get(key));
}
}
return new Options(result);
Expand Down
5 changes: 3 additions & 2 deletions paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Queue;
Expand Down Expand Up @@ -556,13 +557,13 @@ static FileIO get(Path path, CatalogContext config) throws IOException {
if (loader != null) {
Set<String> options =
config.options().keySet().stream()
.map(String::toLowerCase)
.map(s -> s.toLowerCase(Locale.ROOT))
.collect(Collectors.toSet());
Set<String> missOptions = new HashSet<>();
for (String[] keys : loader.requiredOptions()) {
boolean found = false;
for (String key : keys) {
if (options.contains(key.toLowerCase())) {
if (options.contains(key.toLowerCase(Locale.ROOT))) {
found = true;
break;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.PriorityQueue;

Expand Down Expand Up @@ -63,7 +64,7 @@ public static String normalizeRanker(String ranker) {
if (ranker == null || ranker.trim().isEmpty()) {
return RRF_RANKER;
}
String normalized = ranker.trim().toLowerCase();
String normalized = ranker.trim().toLowerCase(Locale.ROOT);
if (!RRF_RANKER.equals(normalized)
&& !WEIGHTED_SCORE_RANKER.equals(normalized)
&& !MRR_RANKER.equals(normalized)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.TreeMap;
Expand Down Expand Up @@ -946,7 +947,8 @@ public void accept(MemorySlice key, byte[] value) throws IOException {

private File newSstFile() {
long sequence = fileSequence.getAndIncrement();
return new File(dataDirectory, String.format("sst-%s-%06d.db", uuid, sequence));
return new File(
dataDirectory, String.format(Locale.ROOT, "sst-%s-%06d.db", uuid, sequence));
}

private void ensureOpen() {
Expand Down
Loading
Loading