From 1721183640beea19f6487d629e2c3a687bf96feb Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 06:05:51 +0800 Subject: [PATCH 1/4] [common] Reject out-of-range range-bitmap chunk-size option RangeBitmapFileIndex parsed the user-facing chunk-size option with (int) MemorySize.parse(...).getBytes(), so a legitimate value such as "2g" silently narrowed to a negative int and the index writer crashed with an unrelated failure deep inside the first chunk. The chunk size also becomes an eagerly allocated per-chunk buffer, so values beyond int range can never work. Validate the option instead: reject anything above Integer.MAX_VALUE with a message naming the option and the offending value. Assisted-by: GLM-5.3 --- .../rangebitmap/RangeBitmapFileIndex.java | 13 +++++++++-- .../rangebitmap/RangeBitmapFileIndexTest.java | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java index 8b3ef92f22eb..6ffc0f41ca04 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java @@ -72,9 +72,18 @@ private static class Writer extends FileIndexWriter { public Writer(DataType dataType, Options options) { KeyFactory factory = KeyFactory.create(dataType); String chunkSize = options.getString(CHUNK_SIZE, factory.defaultChunkSize()); + long bytes = MemorySize.parse(chunkSize).getBytes(); + // the chunk size becomes an eagerly allocated per-chunk buffer, so anything + // beyond int range cannot work; reject it instead of silently truncating + // (e.g. "2g" narrowing to a negative int) and failing deep inside the writer + if (bytes > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + "The '%s' option must not exceed 2147483647 bytes, but was '%s'.", + CHUNK_SIZE, chunkSize)); + } this.converter = factory.createConverter(); - this.appender = - new RangeBitmap.Appender(factory, (int) MemorySize.parse(chunkSize).getBytes()); + this.appender = new RangeBitmap.Appender(factory, (int) bytes); } @Override diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java index 8e94eba87317..874f165456f7 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java @@ -50,6 +50,7 @@ import static org.apache.paimon.predicate.SortValue.SortDirection.ASCENDING; import static org.apache.paimon.predicate.SortValue.SortDirection.DESCENDING; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** test for {@link RangeBitmapFileIndex}. */ public class RangeBitmapFileIndexTest { @@ -57,6 +58,27 @@ public class RangeBitmapFileIndexTest { private static final int ROW_COUNT = 10000; private static final int BOUND = 1000000; + @Test + public void testChunkSizeBeyondIntRangeRejected() { + VarCharType varCharType = new VarCharType(); + + // a chunk size beyond int range must fail with a clear validation message instead of + // silently truncating to a negative int and crashing inside the writer + Options oversized = new Options(); + oversized.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2g"); + assertThatThrownBy(() -> new RangeBitmapFileIndex(varCharType, oversized).createWriter()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("chunk-size"); + + // a large but in-range chunk size still works + Options valid = new Options(); + valid.setString(RangeBitmapFileIndex.CHUNK_SIZE, "16mb"); + FileIndexWriter writer = new RangeBitmapFileIndex(varCharType, valid).createWriter(); + writer.write(BinaryString.fromString("a")); + writer.write(BinaryString.fromString("b")); + assertThat(writer.serializedBytes()).isNotEmpty(); + } + @RepeatedTest(10) public void test() { String prefix = "hello-"; From 8915078ffc37c17c8490995ac8587ba65de40c48 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 12:28:26 +0800 Subject: [PATCH 2/4] test: cover the chunk-size narrowing that does not crash "2g" narrows to a negative int and crashes on the first chunk allocation, but "4g" narrows to 0 and "5g" to 1g: those build a silently wrong index without failing anywhere, which is the case the guard is really there for. Add the "4g" case and say so in the comment. Co-Authored-By: Claude Code --- .../rangebitmap/RangeBitmapFileIndexTest.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java index 874f165456f7..a1ad5e50f9c3 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java @@ -62,11 +62,24 @@ public class RangeBitmapFileIndexTest { public void testChunkSizeBeyondIntRangeRejected() { VarCharType varCharType = new VarCharType(); - // a chunk size beyond int range must fail with a clear validation message instead of - // silently truncating to a negative int and crashing inside the writer - Options oversized = new Options(); - oversized.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2g"); - assertThatThrownBy(() -> new RangeBitmapFileIndex(varCharType, oversized).createWriter()) + // "2g" narrows to a negative int, which crashes on the writer's first chunk allocation + Options negativeAfterNarrowing = new Options(); + negativeAfterNarrowing.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2g"); + assertThatThrownBy( + () -> + new RangeBitmapFileIndex(varCharType, negativeAfterNarrowing) + .createWriter()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("chunk-size"); + + // "4g" narrows to 0, which is the case worth guarding: it does not crash at all, it + // gives every key its own chunk and builds a silently bloated index + Options zeroAfterNarrowing = new Options(); + zeroAfterNarrowing.setString(RangeBitmapFileIndex.CHUNK_SIZE, "4g"); + assertThatThrownBy( + () -> + new RangeBitmapFileIndex(varCharType, zeroAfterNarrowing) + .createWriter()) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("chunk-size"); From 75a27748cfdb1c124af1438a2a3b3c524ab15705 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 12:29:11 +0800 Subject: [PATCH 3/4] docs: name the silent narrowing cases in the chunk-size comment Co-Authored-By: Claude Code --- .../paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java index 6ffc0f41ca04..5d2c957c7f45 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java @@ -73,9 +73,10 @@ public Writer(DataType dataType, Options options) { KeyFactory factory = KeyFactory.create(dataType); String chunkSize = options.getString(CHUNK_SIZE, factory.defaultChunkSize()); long bytes = MemorySize.parse(chunkSize).getBytes(); - // the chunk size becomes an eagerly allocated per-chunk buffer, so anything - // beyond int range cannot work; reject it instead of silently truncating - // (e.g. "2g" narrowing to a negative int) and failing deep inside the writer + // the chunk size becomes an eagerly allocated per-chunk buffer, so anything beyond + // int range cannot work. Truncating is worse than rejecting: "2g" narrows to a + // negative int and crashes deep inside the writer, while "4g" narrows to 0 and + // "5g" to 1g, which build a silently wrong index instead of failing at all. if (bytes > Integer.MAX_VALUE) { throw new IllegalArgumentException( String.format( From db2bb37cf127a663eea44f498333d43a50971bf5 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 15:12:32 +0800 Subject: [PATCH 4/4] test: pin the chunk-size boundary, and say only what the narrowing does The "4g" case re-entered the same comparison as "2g", so it pinned nothing; mutating the guard from > to >= went undetected. Assert the boundary itself instead. The comment also claimed more than the code does. Chunk size 0 is a supported mode (ChunkedDictionaryTest builds one deliberately) and 1g is a value the guard accepts, so neither "4g" nor "5g" produces a wrong index: narrowing substitutes a different size than the one configured, and only the negative case fails. Co-Authored-By: Claude Code --- .../rangebitmap/RangeBitmapFileIndex.java | 7 +++--- .../rangebitmap/RangeBitmapFileIndexTest.java | 24 +++++++------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java index 5d2c957c7f45..0cfa5fa56fbf 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java @@ -73,10 +73,9 @@ public Writer(DataType dataType, Options options) { KeyFactory factory = KeyFactory.create(dataType); String chunkSize = options.getString(CHUNK_SIZE, factory.defaultChunkSize()); long bytes = MemorySize.parse(chunkSize).getBytes(); - // the chunk size becomes an eagerly allocated per-chunk buffer, so anything beyond - // int range cannot work. Truncating is worse than rejecting: "2g" narrows to a - // negative int and crashes deep inside the writer, while "4g" narrows to 0 and - // "5g" to 1g, which build a silently wrong index instead of failing at all. + // the chunk size becomes an eagerly allocated per-chunk buffer, so it has to fit an + // int. Narrowing it silently substitutes a different size: "2g" becomes negative and + // fails only once the writer allocates, "4g" becomes 0 and "5g" becomes 1g. if (bytes > Integer.MAX_VALUE) { throw new IllegalArgumentException( String.format( diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java index a1ad5e50f9c3..5edac83812e3 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java @@ -62,28 +62,22 @@ public class RangeBitmapFileIndexTest { public void testChunkSizeBeyondIntRangeRejected() { VarCharType varCharType = new VarCharType(); - // "2g" narrows to a negative int, which crashes on the writer's first chunk allocation - Options negativeAfterNarrowing = new Options(); - negativeAfterNarrowing.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2g"); + // the boundary is the whole guard: one byte past int range is rejected, int range itself + // is accepted. "2g" and "4g" would both only re-test the same comparison + Options justPastIntRange = new Options(); + justPastIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483648 bytes"); assertThatThrownBy( () -> - new RangeBitmapFileIndex(varCharType, negativeAfterNarrowing) + new RangeBitmapFileIndex(varCharType, justPastIntRange) .createWriter()) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("chunk-size"); - // "4g" narrows to 0, which is the case worth guarding: it does not crash at all, it - // gives every key its own chunk and builds a silently bloated index - Options zeroAfterNarrowing = new Options(); - zeroAfterNarrowing.setString(RangeBitmapFileIndex.CHUNK_SIZE, "4g"); - assertThatThrownBy( - () -> - new RangeBitmapFileIndex(varCharType, zeroAfterNarrowing) - .createWriter()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("chunk-size"); + Options atIntRange = new Options(); + atIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483647 bytes"); + assertThat(new RangeBitmapFileIndex(varCharType, atIntRange).createWriter()).isNotNull(); - // a large but in-range chunk size still works + // a large but in-range chunk size still writes and serializes Options valid = new Options(); valid.setString(RangeBitmapFileIndex.CHUNK_SIZE, "16mb"); FileIndexWriter writer = new RangeBitmapFileIndex(varCharType, valid).createWriter();