diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java index 7a62523711e1..a2f67e6e174c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java @@ -83,6 +83,7 @@ public static int calculateFixLengthPartSize(DataType type) { case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: case ARRAY: + case VECTOR: case MULTISET: case MAP: case ROW: @@ -260,7 +261,8 @@ public InternalArray getArray(int pos) { @Override public InternalVector getVector(int pos) { - throw new IllegalArgumentException("Unsupported type: VectorType"); + assertIndexIsValid(pos); + return MemorySegmentUtils.readVectorData(segments, offset, getLong(pos)); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java index 78382e8bbd23..463f41283bcc 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java @@ -124,6 +124,9 @@ static ElementGetter createElementGetter(DataType elementType) { case ARRAY: elementGetter = InternalArray::getArray; break; + case VECTOR: + elementGetter = InternalArray::getVector; + break; case MULTISET: case MAP: elementGetter = InternalArray::getMap; diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java index 445d8854a19a..ab48c69381d4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java @@ -150,6 +150,8 @@ static Class getDataClass(DataType type) { return Timestamp.class; case ARRAY: return InternalArray.class; + case VECTOR: + return InternalVector.class; case MULTISET: case MAP: return InternalMap.class; diff --git a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java new file mode 100644 index 000000000000..3d8416efbfdc --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.data.serializer.InternalArraySerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.VectorType; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests that ARRAY<VECTOR> is supported by {@link InternalArray} accessors: the {@link + * InternalArraySerializer} eagerly builds an element getter, so a missing VECTOR case fails + * serializer construction for an accepted DDL type. + */ +class InternalArrayVectorGetterTest { + + private static final VectorType VECTOR_TYPE = new VectorType(3, new FloatType()); + + @Test + void elementGetterReadsVector() { + BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {1.0f, 2.0f, 3.0f}); + GenericArray array = new GenericArray(new Object[] {vector}); + + InternalArray.ElementGetter getter = InternalArray.createElementGetter(VECTOR_TYPE); + + assertThat(getter.getElementOrNull(array, 0)).isEqualTo(vector); + } + + @Test + void vectorArrayRoundTripsThroughSerializer() throws Exception { + InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); + // more than one element, with a null between them: only the binary form exercises the + // 8-byte variable-length slot and the null bit, and the last vector then starts at an + // offset the first one's payload decided + GenericArray array = + new GenericArray( + new Object[] { + BinaryVector.fromPrimitiveArray(new float[] {4.0f, 5.0f, 6.0f}), + null, + BinaryVector.fromPrimitiveArray(new float[] {7.0f, 8.0f, 9.0f}) + }); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serializer.serialize(array, new DataOutputViewStreamWrapper(out)); + InternalArray readBack = + serializer.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(out.toByteArray()))); + + assertThat(readBack.size()).isEqualTo(3); + assertThat(readBack.isNullAt(1)).isTrue(); + assertThat(readBack.getVector(0).toFloatArray()).isEqualTo(new float[] {4.0f, 5.0f, 6.0f}); + assertThat(readBack.getVector(2).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + + // toObjectArray and copy read the binary form through getDataClass and the element + // getter; the projection and manifest paths use them, no test did + InternalVector[] objects = ((BinaryArray) readBack).toObjectArray(VECTOR_TYPE); + assertThat(objects).hasSize(3); + assertThat(objects[1]).isNull(); + assertThat(objects[2].toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + + InternalArray copied = serializer.copy(readBack); + assertThat(copied.isNullAt(1)).isTrue(); + assertThat(copied.getVector(2).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + } + + @Test + void vectorArrayCopyResolvesDataClass() { + // copy() of a GenericArray goes through InternalRow.getDataClass(VECTOR) to allocate the + // element array; without the VECTOR case that throws "Illegal type", so this pins the + // getDataClass fix (the round-trip test above copies raw bytes and never reaches it). + InternalArraySerializer serializer = new InternalArraySerializer(VECTOR_TYPE); + BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[] {7.0f, 8.0f, 9.0f}); + GenericArray array = new GenericArray(new Object[] {vector}); + + InternalArray copied = serializer.copy(array); + + assertThat(copied.getVector(0).toFloatArray()).isEqualTo(new float[] {7.0f, 8.0f, 9.0f}); + } +}