diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java index 8b3ef92f22eb..0cfa5fa56fbf 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java @@ -72,9 +72,18 @@ private static class Writer extends FileIndexWriter { public Writer(DataType dataType, Options options) { KeyFactory factory = KeyFactory.create(dataType); String chunkSize = options.getString(CHUNK_SIZE, factory.defaultChunkSize()); + long bytes = MemorySize.parse(chunkSize).getBytes(); + // the chunk size becomes an eagerly allocated per-chunk buffer, so it has to fit an + // int. Narrowing it silently substitutes a different size: "2g" becomes negative and + // fails only once the writer allocates, "4g" becomes 0 and "5g" becomes 1g. + if (bytes > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + "The '%s' option must not exceed 2147483647 bytes, but was '%s'.", + CHUNK_SIZE, chunkSize)); + } this.converter = factory.createConverter(); - this.appender = - new RangeBitmap.Appender(factory, (int) MemorySize.parse(chunkSize).getBytes()); + this.appender = new RangeBitmap.Appender(factory, (int) bytes); } @Override diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java index 8e94eba87317..5edac83812e3 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java @@ -50,6 +50,7 @@ import static org.apache.paimon.predicate.SortValue.SortDirection.ASCENDING; import static org.apache.paimon.predicate.SortValue.SortDirection.DESCENDING; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** test for {@link RangeBitmapFileIndex}. */ public class RangeBitmapFileIndexTest { @@ -57,6 +58,34 @@ public class RangeBitmapFileIndexTest { private static final int ROW_COUNT = 10000; private static final int BOUND = 1000000; + @Test + public void testChunkSizeBeyondIntRangeRejected() { + VarCharType varCharType = new VarCharType(); + + // the boundary is the whole guard: one byte past int range is rejected, int range itself + // is accepted. "2g" and "4g" would both only re-test the same comparison + Options justPastIntRange = new Options(); + justPastIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483648 bytes"); + assertThatThrownBy( + () -> + new RangeBitmapFileIndex(varCharType, justPastIntRange) + .createWriter()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("chunk-size"); + + Options atIntRange = new Options(); + atIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483647 bytes"); + assertThat(new RangeBitmapFileIndex(varCharType, atIntRange).createWriter()).isNotNull(); + + // a large but in-range chunk size still writes and serializes + Options valid = new Options(); + valid.setString(RangeBitmapFileIndex.CHUNK_SIZE, "16mb"); + FileIndexWriter writer = new RangeBitmapFileIndex(varCharType, valid).createWriter(); + writer.write(BinaryString.fromString("a")); + writer.write(BinaryString.fromString("b")); + assertThat(writer.serializedBytes()).isNotEmpty(); + } + @RepeatedTest(10) public void test() { String prefix = "hello-";