From 60d7e528a8c9eb091097c783423ede9c4e630e63 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 07:22:17 +0800 Subject: [PATCH 1/3] [common] Support VECTOR elements in InternalArray accessors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ARRAY is accepted by schema validation and readable via ColumnarArray, but the binary data path rejected it in three places: InternalArray.createElementGetter had no VECTOR case (so InternalArraySerializer, which builds its element getter eagerly, failed construction with "type VECTOR not support"), BinaryArray.calculateFixLengthPartSize threw on VECTOR, and BinaryArray.getVector was an unconditional throw — so a row type with an ARRAY column, e.g. serialized by a primary-key table's local merge, failed no matter how far it got. Route VECTOR through the existing accessors: getVector in the element getter, the 8-byte variable-length slot in calculateFixLengthPartSize (mirroring ARRAY), readVectorData in BinaryArray.getVector (mirroring BinaryRow), and InternalVector in InternalRow.getDataClass for copies. Assisted-by: GLM-5.3 --- .../org/apache/paimon/data/BinaryArray.java | 4 +- .../org/apache/paimon/data/InternalArray.java | 3 + .../org/apache/paimon/data/InternalRow.java | 2 + .../data/InternalArrayVectorGetterTest.java | 83 +++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java index 7a62523711e1..a2f67e6e174c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java @@ -83,6 +83,7 @@ public static int calculateFixLengthPartSize(DataType type) { case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: case ARRAY: + case VECTOR: case MULTISET: case MAP: case ROW: @@ -260,7 +261,8 @@ public InternalArray getArray(int pos) { @Override public InternalVector getVector(int pos) { - throw new IllegalArgumentException("Unsupported type: VectorType"); + assertIndexIsValid(pos); + return MemorySegmentUtils.readVectorData(segments, offset, getLong(pos)); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java index 78382e8bbd23..463f41283bcc 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java @@ -124,6 +124,9 @@ static ElementGetter createElementGetter(DataType elementType) { case ARRAY: elementGetter = InternalArray::getArray; break; + case VECTOR: + elementGetter = InternalArray::getVector; + break; case MULTISET: case MAP: elementGetter = InternalArray::getMap; diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java index 445d8854a19a..ab48c69381d4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java @@ -150,6 +150,8 @@ static Class getDataClass(DataType type) { return Timestamp.class; case ARRAY: return InternalArray.class; + case VECTOR: + return InternalVector.class; case MULTISET: case MAP: return InternalMap.class; diff --git a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java new file mode 100644 index 000000000000..7d81c9929128 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.data.serializer.InternalArraySerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.VectorType; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests that ARRAY<VECTOR> is supported by {@link InternalArray} accessors: the {@link + * InternalArraySerializer} eagerly builds an element getter, so a missing VECTOR case fails + * serializer construction for an accepted DDL type. + */ +class InternalArrayVectorGetterTest { + + private static final VectorType VECTOR_TYPE = new VectorType(3, new FloatType()); + + @Test + void arraySerializerOverVectorConstructs() { + InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); + assertThat(serializer).isNotNull(); + } + + @Test + void elementGetterReadsVector() { + BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {1.0f, 2.0f, 3.0f}); + GenericArray array = new GenericArray(new Object[] {vector}); + + InternalArray.ElementGetter getter = InternalArray.createElementGetter(VECTOR_TYPE); + + assertThat(getter.getElementOrNull(array, 0)).isEqualTo(vector); + } + + @Test + void elementGetterReturnsNullForNullElement() { + GenericArray array = new GenericArray(new Object[] {null}); + + InternalArray.ElementGetter getter = InternalArray.createElementGetter(VECTOR_TYPE); + + assertThat(getter.getElementOrNull(array, 0)).isNull(); + } + + @Test + void vectorArrayRoundTripsThroughSerializer() throws Exception { + InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); + BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {4.0f, 5.0f, 6.0f}); + GenericArray array = new GenericArray(new Object[] {vector}); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serializer.serialize(array, new DataOutputViewStreamWrapper(out)); + InternalArray readBack = + serializer.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(out.toByteArray()))); + + assertThat(readBack.getVector(0).toFloatArray()).isEqualTo(new float[] {4.0f, 5.0f, 6.0f}); + } +} From f61096f52370b4b363b2a75d8d4dff17665cbc7d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 11:37:01 +0800 Subject: [PATCH 2/3] test: pin the getDataClass VECTOR fix via the array copy path The round-trip test copies BinaryArray bytes and never reaches InternalRow.getDataClass, so the VECTOR case that copy() of a GenericArray depends on was unpinned. This copy test throws "Illegal type: VECTOR" on the base and passes on the fix. Co-Authored-By: Claude Code --- .../paimon/data/InternalArrayVectorGetterTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java index 7d81c9929128..f4ed5b2b72f4 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java @@ -80,4 +80,18 @@ void vectorArrayRoundTripsThroughSerializer() throws Exception { assertThat(readBack.getVector(0).toFloatArray()).isEqualTo(new float[] {4.0f, 5.0f, 6.0f}); } + + @Test + void vectorArrayCopyResolvesDataClass() { + // copy() of a GenericArray goes through InternalRow.getDataClass(VECTOR) to allocate the + // element array; without the VECTOR case that throws "Illegal type", so this pins the + // getDataClass fix (the round-trip test above copies raw bytes and never reaches it). + InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); + BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {7.0f, 8.0f, 9.0f}); + GenericArray array = new GenericArray(new Object[] {vector}); + + InternalArray copied = serializer.copy(array); + + assertThat(copied.getVector(0).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + } } From b29ff10b01714e35f82daa9673cbf43a70c6c1b5 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 15:20:01 +0800 Subject: [PATCH 3/3] test: point the VECTOR array test at what only the binary form reaches Three of the five tests pinned the same createElementGetter arm and died together; the one test that touched the binary form used a single non-null element, so the 8-byte variable-length slot, the null bit, toObjectArray and copy-on-binary were pinned by nothing. Fold that coverage into the round trip with a three-element array containing a null, and drop the two tests that only restated the arm elementGetterReadsVector already covers. Co-Authored-By: Claude Code --- .../data/InternalArrayVectorGetterTest.java | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java index f4ed5b2b72f4..3d8416efbfdc 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java @@ -40,12 +40,6 @@ class InternalArrayVectorGetterTest { private static final VectorType VECTOR_TYPE = new VectorType(3, new FloatType()); - @Test - void arraySerializerOverVectorConstructs() { - InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); - assertThat(serializer).isNotNull(); - } - @Test void elementGetterReadsVector() { BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {1.0f, 2.0f, 3.0f}); @@ -56,20 +50,19 @@ void elementGetterReadsVector() { assertThat(getter.getElementOrNull(array, 0)).isEqualTo(vector); } - @Test - void elementGetterReturnsNullForNullElement() { - GenericArray array = new GenericArray(new Object[] {null}); - - InternalArray.ElementGetter getter = InternalArray.createElementGetter(VECTOR_TYPE); - - assertThat(getter.getElementOrNull(array, 0)).isNull(); - } - @Test void vectorArrayRoundTripsThroughSerializer() throws Exception { InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); - BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {4.0f, 5.0f, 6.0f}); - GenericArray array = new GenericArray(new Object[] {vector}); + // more than one element, with a null between them: only the binary form exercises the + // 8-byte variable-length slot and the null bit, and the last vector then starts at an + // offset the first one's payload decided + GenericArray array = + new GenericArray( + new Object[] { + BinaryVector.fromPrimitiveArray(new float[] {4.0f, 5.0f, 6.0f}), + null, + BinaryVector.fromPrimitiveArray(new float[] {7.0f, 8.0f, 9.0f}) + }); ByteArrayOutputStream out = new ByteArrayOutputStream(); serializer.serialize(array, new DataOutputViewStreamWrapper(out)); @@ -78,7 +71,21 @@ void vectorArrayRoundTripsThroughSerializer() throws Exception { new DataInputViewStreamWrapper( new ByteArrayInputStream(out.toByteArray()))); + assertThat(readBack.size()).isEqualTo(3); + assertThat(readBack.isNullAt(1)).isTrue(); assertThat(readBack.getVector(0).toFloatArray()).isEqualTo(new float[] {4.0f, 5.0f, 6.0f}); + assertThat(readBack.getVector(2).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + + // toObjectArray and copy read the binary form through getDataClass and the element + // getter; the projection and manifest paths use them, no test did + InternalVector[] objects = ((BinaryArray) readBack).toObjectArray(VECTOR_TYPE); + assertThat(objects).hasSize(3); + assertThat(objects[1]).isNull(); + assertThat(objects[2].toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + + InternalArray copied = serializer.copy(readBack); + assertThat(copied.isNullAt(1)).isTrue(); + assertThat(copied.getVector(2).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); } @Test