From c1852731a77ae7bffc408afd1f912e34fb033972 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 15 Sep 2026 18:45:12 -0400 Subject: [PATCH 1/3] ZSTDArray::append_to_builder falls back to canonical for non utf8/binary dtypes Signed-off-by: Robert Kruszewski --- encodings/zstd/src/array.rs | 17 +++++++++-------- encodings/zstd/src/test.rs | 19 +++++++++++++++++-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index dafe8e820a9..ce9d5d38b28 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -289,14 +289,15 @@ impl VTable for Zstd { { return result; } - // The two arms here are every builder a `Utf8`/`Binary` dtype has: all four - // `VarBinBuilder` widths above, and `VarBinViewBuilder` below. There is deliberately no - // canonicalize-then-append fallback — it would decompress to a `VarBinView` only for - // `VarBinView::append_to_builder` to reject the same remainder. - let Some(builder) = builder.as_any_mut().downcast_mut::() else { - vortex_bail!("append_to_builder for Zstd requires a variable-binary builder") - }; - append_to_varbinview(array, builder, ctx) + if let Some(builder) = builder.as_any_mut().downcast_mut::() { + return append_to_varbinview(array, builder, ctx); + } + array + .array() + .clone() + .execute::(ctx)? + .into_array() + .append_to_builder(builder, ctx) } fn reduce_parent( diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index fafbfcaf159..cef78f736ed 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -14,6 +14,7 @@ use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::builders::VarBinBuilder; use vortex_array::builders::VarBinViewBuilder; +use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; @@ -217,7 +218,21 @@ fn test_zstd_var_bin_view() { } #[test] -fn test_zstd_append_to_offset_builder() { +fn test_zstd_append_to_primitive_builder() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); + let array = PrimitiveArray::from_option_iter([Some(1_i32), None, Some(3), Some(4), Some(5)]); + let compressed = Zstd::from_primitive(&array, 0, 3, &mut ctx)?.slice(1..4)?; + let mut builder = builder_with_capacity(compressed.dtype(), compressed.len()); + compressed.append_to_builder(builder.as_mut(), &mut ctx)?; + assert_arrays_eq!(builder.finish(), array.into_array().slice(1..4)?, &mut ctx); + Ok(()) +} + +#[rstest] +fn test_zstd_append_to_offset_builder( + #[values(DType::Utf8(Nullability::Nullable), DType::Binary(Nullability::Nullable))] + dtype: DType, +) { let mut ctx = array_session().create_execution_ctx(); let array = VarBinViewArray::from_iter( [ @@ -227,7 +242,7 @@ fn test_zstd_append_to_offset_builder() { Some(b"Lorem ipsum dolor sit amet".as_slice()), Some(b"baz".as_slice()), ], - DType::Utf8(Nullability::Nullable), + dtype, ); let compressed = Zstd::from_var_bin_view(&array, 0, 3, &mut ctx) .unwrap() From dae38311a0a91b01017e6b8991a2374594d99521 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 15 Sep 2026 21:36:56 -0400 Subject: [PATCH 2/3] fixes Signed-off-by: Robert Kruszewski --- encodings/zstd/src/test.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index cef78f736ed..1ead58dc89c 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -14,13 +14,14 @@ use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::builders::VarBinBuilder; use vortex_array::builders::VarBinViewBuilder; -use vortex_array::builders::builder_with_capacity; +use vortex_array::builders::builder_with_capacity_in; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::validity::Validity; use vortex_buffer::Alignment; use vortex_buffer::Buffer; +use vortex_buffer::BufferAllocatorRef; use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -222,7 +223,11 @@ fn test_zstd_append_to_primitive_builder() -> VortexResult<()> { let mut ctx = array_session().create_execution_ctx(); let array = PrimitiveArray::from_option_iter([Some(1_i32), None, Some(3), Some(4), Some(5)]); let compressed = Zstd::from_primitive(&array, 0, 3, &mut ctx)?.slice(1..4)?; - let mut builder = builder_with_capacity(compressed.dtype(), compressed.len()); + let mut builder = builder_with_capacity_in( + compressed.dtype(), + compressed.len(), + BufferAllocatorRef::static_ref(), + ); compressed.append_to_builder(builder.as_mut(), &mut ctx)?; assert_arrays_eq!(builder.finish(), array.into_array().slice(1..4)?, &mut ctx); Ok(()) @@ -230,7 +235,10 @@ fn test_zstd_append_to_primitive_builder() -> VortexResult<()> { #[rstest] fn test_zstd_append_to_offset_builder( - #[values(DType::Utf8(Nullability::Nullable), DType::Binary(Nullability::Nullable))] + #[values( + DType::Utf8(Nullability::Nullable), + DType::Binary(Nullability::Nullable) + )] dtype: DType, ) { let mut ctx = array_session().create_execution_ctx(); From 2b8d3e13e18ed6164d70cb91563215b761a5d949 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 15 Sep 2026 22:21:30 -0400 Subject: [PATCH 3/3] fixes Signed-off-by: Robert Kruszewski --- encodings/zstd/src/test.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 1ead58dc89c..f01d2654e6a 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -259,7 +259,7 @@ fn test_zstd_append_to_offset_builder( let mut builder = VarBinBuilder::::with_capacity_in( compressed.dtype().clone(), compressed.len(), - vortex_buffer::BufferAllocatorRef::static_ref(), + BufferAllocatorRef::static_ref(), ); compressed .append_to_builder(&mut builder, &mut ctx) @@ -290,7 +290,7 @@ fn test_zstd_append_to_view_builder_keeps_only_the_sliced_bytes() -> VortexResul let mut builder = VarBinViewBuilder::with_capacity_in( compressed.dtype().clone(), 9, - vortex_buffer::BufferAllocatorRef::statically_allocated(), + BufferAllocatorRef::statically_allocated(), ); builder.append_value(&values[0]); compressed.append_to_builder(&mut builder, &mut ctx)?; @@ -406,7 +406,7 @@ fn test_zstd_rejects_corrupt_frame_metadata( let mut builder = VarBinBuilder::::with_capacity_in( compressed.dtype().clone(), compressed.len(), - vortex_buffer::BufferAllocatorRef::static_ref(), + BufferAllocatorRef::static_ref(), ); assert!( compressed @@ -447,17 +447,11 @@ fn test_zstd_rejects_a_frame_ending_in_a_dangling_length_prefix() -> VortexResul )?; assert!(Zstd::decompress(&compressed, &mut ctx).is_err()); - let mut varbin = VarBinBuilder::::with_capacity_in( - dtype.clone(), - 2, - vortex_buffer::BufferAllocatorRef::static_ref(), - ); + let mut varbin = + VarBinBuilder::::with_capacity_in(dtype.clone(), 2, BufferAllocatorRef::static_ref()); assert!(compressed.append_to_builder(&mut varbin, &mut ctx).is_err()); - let mut views = VarBinViewBuilder::with_capacity_in( - dtype, - 2, - vortex_buffer::BufferAllocatorRef::statically_allocated(), - ); + let mut views = + VarBinViewBuilder::with_capacity_in(dtype, 2, BufferAllocatorRef::statically_allocated()); assert!(compressed.append_to_builder(&mut views, &mut ctx).is_err()); Ok(()) }