diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs index bf87b9a7888e6..ec863dd7fa5ff 100644 --- a/datafusion/common/src/utils/mod.rs +++ b/datafusion/common/src/utils/mod.rs @@ -35,13 +35,13 @@ use arrow::array::{ }; use arrow::array::{ ArrowPrimitiveType, BooleanArray, Datum, GenericListArray, Int32Array, Int64Array, - MutableArrayData, PrimitiveArray, make_array, + MutableArrayData, PrimitiveArray, layout, make_array, }; use arrow::array::{LargeListViewArray, ListViewArray}; -use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::kernels::cmp::eq; use arrow::compute::kernels::length::length; -use arrow::compute::{SortColumn, SortOptions, partition}; +use arrow::compute::{SortColumn, SortOptions, nullif, partition}; use arrow::datatypes::{ ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef, }; @@ -1521,6 +1521,49 @@ pub fn normalize_float_zero_scalar(scalar: ScalarValue) -> ScalarValue { } } +/// Apply a struct's nulls to one of its fields. +/// +/// Arrays with validity bitmaps share their value buffers; unions and +/// run-end encoded arrays are rebuilt. +pub fn apply_parent_nulls( + col: &ArrayRef, + parent_nulls: Option<&NullBuffer>, +) -> Result { + let Some(parent_nulls) = parent_nulls else { + // If there are no parent nulls to apply, we can just return + return Ok(Arc::clone(col)); + }; + + // NullArray is already entirely null and cannot have a validity bitmap. + // If we have 0 parent nulls, we can also avoid extra work. + if col.data_type().is_null() || parent_nulls.null_count() == 0 { + return Ok(Arc::clone(col)); + } + + if layout(col.data_type()).can_contain_null_mask { + // `nullif` marks a row null where the mask is true and keeps the + // field's own nulls. Only the validity bitmap is rebuilt; the value + // buffers and child arrays are shared with `col`. + let null_parents = BooleanArray::new(!parent_nulls.inner(), None); + return Ok(nullif(col.as_ref(), &null_parents)?); + } + + // Unions and run-end encoded arrays have no validity bitmap of their own + // and represent nulls in their children. Rebuild the array so null parents + // become null values in those children. + let data = col.to_data(); + let mut mutable = MutableArrayData::new(vec![&data], true, data.len()); + let mut end = 0; + for (start, valid_end) in parent_nulls.valid_slices() { + mutable.try_extend_nulls(start - end)?; + mutable.try_extend(0, start, valid_end)?; + end = valid_end; + } + mutable.try_extend_nulls(data.len() - end)?; + + Ok(make_array(mutable.freeze())) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -1528,13 +1571,50 @@ mod tests { use super::*; use crate::ScalarValue::Null; use arrow::{ - array::{Float64Array, Int32Array}, - buffer::NullBuffer, + array::{Float64Array, Int32Array, NullArray}, datatypes::Int32Type, }; #[cfg(feature = "sql")] use sqlparser::ast::Ident; + #[test] + fn test_apply_parent_nulls_sliced() -> Result<()> { + let child = Arc::new( + Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]).slice(1, 3), + ) as ArrayRef; + let parent_nulls = + NullBuffer::from(vec![true, true, true, false, true]).slice(1, 3); + let result = apply_parent_nulls(&child, Some(&parent_nulls))?; + assert_eq!( + result.as_ref(), + &Int32Array::from(vec![None, Some(3), None]) + ); + assert!(result.to_data().buffers()[0].ptr_eq(&child.to_data().buffers()[0])); + Ok(()) + } + + #[test] + fn test_apply_parent_nulls_noop() -> Result<()> { + let child = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])) as ArrayRef; + for parent_nulls in [None, Some(NullBuffer::new_valid(3))] { + assert!(Arc::ptr_eq( + &apply_parent_nulls(&child, parent_nulls.as_ref())?, + &child, + )); + } + let child = Arc::new(NullArray::new(3)) as ArrayRef; + assert!(Arc::ptr_eq( + &apply_parent_nulls(&child, Some(&NullBuffer::new_null(3)))?, + &child, + )); + let empty = child.slice(0, 0); + assert!(Arc::ptr_eq( + &apply_parent_nulls(&empty, Some(&NullBuffer::new_valid(0)))?, + &empty, + )); + Ok(()) + } + #[test] fn test_offset_span() { let offsets = OffsetBuffer::new(vec![0_i32, 5, 8, 8, 12].into()); diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 96e7feebf6f45..1807c12ba1f89 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -5125,7 +5125,7 @@ impl Unnest { )); Ok(get_unnested_columns( &r.output_column.name, - original_field.data_type(), + original_field, r.depth, )? .into_iter() @@ -5136,7 +5136,7 @@ impl Unnest { if transformed_columns.is_empty() { transformed_columns = get_unnested_columns( &column_to_unnest.name, - original_field.data_type(), + original_field, 1, )?; match original_field.data_type() { @@ -5216,9 +5216,10 @@ impl Unnest { // the recursion level fn get_unnested_columns( col_name: &String, - data_type: &DataType, + field: &Field, depth: usize, ) -> Result)>> { + let data_type = field.data_type(); let mut qualified_columns = Vec::with_capacity(1); match data_type { @@ -5242,7 +5243,11 @@ fn get_unnested_columns( qualified_columns.extend(fields.iter().map(|f| { let new_name = format!("{}.{}", col_name, f.name()); let column = Column::from_name(&new_name); - let new_field = f.as_ref().clone().with_name(new_name); + let new_field = f + .as_ref() + .clone() + .with_name(new_name) + .with_nullable(field.is_nullable() || f.is_nullable()); // let column = Column::from((None, &f)); (column, Arc::new(new_field)) })) diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs index 7ee20a01edb3c..2b75623daac74 100644 --- a/datafusion/functions/src/core/getfield.rs +++ b/datafusion/functions/src/core/getfield.rs @@ -17,14 +17,12 @@ use std::sync::{Arc, OnceLock}; -use arrow::array::{ - Array, ArrayRef, BooleanArray, MutableArrayData, cast::AsArray, layout, make_array, -}; -use arrow::buffer::NullBuffer; -use arrow::compute::{nullif, take}; +use arrow::array::{Array, cast::AsArray}; +use arrow::compute::take; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cast::{as_map_array, as_struct_array}; +use datafusion_common::utils::apply_parent_nulls; use datafusion_common::{ Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, plan_datafusion_err, }; @@ -100,46 +98,6 @@ impl Default for GetFieldFunc { } } -/// Apply a struct's nulls to one of its fields. -fn apply_parent_nulls( - col: &ArrayRef, - parent_nulls: Option<&NullBuffer>, -) -> Result { - let Some(parent_nulls) = parent_nulls else { - // If there are no parent nulls to apply, we can just return - return Ok(Arc::clone(col)); - }; - - // NullArray is already entirely null and cannot have a validity bitmap. - // If we have 0 parent nulls, we can also avoid extra work. - if col.data_type().is_null() || parent_nulls.null_count() == 0 { - return Ok(Arc::clone(col)); - } - - if layout(col.data_type()).can_contain_null_mask { - // `nullif` marks a row null where the mask is true and keeps the - // field's own nulls. Only the validity bitmap is rebuilt; the value - // buffers and child arrays are shared with `col`. - let null_parents = BooleanArray::new(!parent_nulls.inner(), None); - return Ok(nullif(col.as_ref(), &null_parents)?); - } - - // Unions and run-end encoded arrays have no validity bitmap of their own - // and represent nulls in their children. Rebuild the array so null parents - // become null values in those children. - let data = col.to_data(); - let mut mutable = MutableArrayData::new(vec![&data], true, data.len()); - let mut end = 0; - for (start, valid_end) in parent_nulls.valid_slices() { - mutable.try_extend_nulls(start - end)?; - mutable.try_extend(0, start, valid_end)?; - end = valid_end; - } - mutable.try_extend_nulls(data.len() - end)?; - - Ok(make_array(mutable.freeze())) -} - /// Extract a single field from a struct or map array fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result { let arrays = ColumnarValue::values_to_arrays(&[base])?; @@ -617,6 +575,7 @@ mod tests { ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder, RunArray, StructArray, UnionArray, }; + use arrow::buffer::NullBuffer; use arrow::datatypes::{Fields, Int32Type, UnionFields}; #[test] diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 6a717defb95fa..85c64fe5276ce 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -193,3 +193,7 @@ name = "window_filter" [[bench]] harness = false name = "range_repartition" + +[[bench]] +harness = false +name = "unnest" diff --git a/datafusion/physical-plan/benches/unnest.rs b/datafusion/physical-plan/benches/unnest.rs new file mode 100644 index 0000000000000..1897a9856aca5 --- /dev/null +++ b/datafusion/physical-plan/benches/unnest.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hint::black_box; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array, StringArray, StructArray}; +use arrow::buffer::NullBuffer; +use arrow::datatypes::{Field, Schema}; +use arrow::record_batch::RecordBatch; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use datafusion_common::UnnestOptions; +use datafusion_execution::{TaskContext, config::SessionConfig}; +use datafusion_physical_plan::test::TestMemoryExec; +use datafusion_physical_plan::unnest::UnnestExec; +use datafusion_physical_plan::{ExecutionPlan, collect}; +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand::seq::SliceRandom; + +fn unnest_benchmark(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("unnest_struct"); + for num_rows in [1024, 8192, 65536] { + let integers = Arc::new(Int64Array::from_iter( + (0..num_rows).map(|i| (i % 10 != 0).then_some(i as i64)), + )) as ArrayRef; + let strings = Arc::new(StringArray::from_iter( + (0..num_rows).map(|i| (i % 10 != 0).then(|| format!("value-{i:0122}"))), + )) as ArrayRef; + let context = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(num_rows)), + ); + group.throughput(Throughput::Elements(num_rows as u64)); + for (name, child) in [("int64", integers), ("utf8_128_bytes", strings)] { + for null_percent in [0, 1, 50, 100] { + let nulls = (null_percent != 0).then(|| { + let mut validity = vec![true; num_rows]; + validity[..num_rows * null_percent / 100].fill(false); + validity.shuffle(&mut StdRng::seed_from_u64(42)); + NullBuffer::from(validity) + }); + let fields = vec![Field::new("value", child.data_type().clone(), true)]; + let parent = StructArray::new( + fields.clone().into(), + vec![Arc::clone(&child)], + nulls, + ); + let batch = + RecordBatch::try_from_iter(vec![("s", Arc::new(parent) as ArrayRef)]) + .unwrap(); + let source = TestMemoryExec::try_new_exec( + &[vec![batch.clone()]], + batch.schema(), + None, + ) + .unwrap(); + let plan: Arc = Arc::new( + UnnestExec::new( + source, + vec![], + vec![0], + Arc::new(Schema::new(fields)), + UnnestOptions::default(), + ) + .unwrap(), + ); + group.bench_function( + BenchmarkId::new(format!("{name}_nulls_{null_percent}"), num_rows), + |b| { + b.iter(|| { + black_box( + runtime + .block_on(collect( + Arc::clone(&plan), + Arc::clone(&context), + )) + .unwrap(), + ) + }); + }, + ); + } + } + } + group.finish(); +} + +criterion_group!(benches, unnest_benchmark); +criterion_main!(benches); diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index d93e0280515c6..cac675d590d31 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -46,6 +46,7 @@ use arrow::record_batch::RecordBatch; use arrow_ord::cmp::lt; use async_trait::async_trait; use datafusion_common::tree_node::TreeNodeRecursion; +use datafusion_common::utils::apply_parent_nulls; use datafusion_common::{ Constraints, HashMap, HashSet, Result, UnnestOptions, exec_datafusion_err, exec_err, internal_err, @@ -767,7 +768,15 @@ fn flatten_struct_cols( DataType::Struct(_) => { let struct_arr = column_data.as_any().downcast_ref::().unwrap(); - Ok(struct_arr.columns().to_vec()) + if struct_arr.null_count() == 0 { + Ok(struct_arr.columns().to_vec()) + } else { + struct_arr + .columns() + .iter() + .map(|column| apply_parent_nulls(column, struct_arr.nulls())) + .collect() + } } data_type => internal_err!( "expecting column {idx} from input plan to be a struct, got {data_type}" @@ -1435,15 +1444,272 @@ fn repeat_arrs_from_indices( mod tests { use super::*; use arrow::array::{ - GenericListArray, Int32Array, NullBufferBuilder, OffsetSizeTrait, StringArray, + BooleanArray, DictionaryArray, GenericListArray, Int32Array, MapArray, NullArray, + NullBufferBuilder, OffsetSizeTrait, RunArray, StringArray, StringViewArray, + UnionArray, layout, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; - use arrow::datatypes::{Field, Int32Type}; + use arrow::datatypes::{Field, Int32Type, UnionFields}; use datafusion_common::NullHandling; use datafusion_common::test_util::batches_to_string; use datafusion_physical_expr_common::metrics::MetricValue; use insta::assert_snapshot; + #[test] + fn test_flatten_struct_parent_nulls() -> Result<()> { + let values = Int32Array::from(vec![Some(1), Some(2), None]); + let mut children: Vec = vec![ + Arc::new(values.clone()), + Arc::new(StringArray::from(vec![Some("x"), Some("y"), None])), + Arc::new(NullArray::new(3)), + Arc::new(RunArray::::try_new( + &Int32Array::from(vec![2, 3]), + &Int32Array::from(vec![Some(1), None]), + )?), + ]; + for offsets in [None, Some(vec![0, 1, 2].into())] { + children.push(Arc::new(UnionArray::try_new( + UnionFields::try_new([0], [Field::new("v", DataType::Int32, true)])?, + vec![0, 0, 0].into(), + offsets, + vec![Arc::new(values.clone())], + )?)); + } + + for child in children { + let fields = vec![Field::new("v", child.data_type().clone(), true)]; + let schema = Arc::new(Schema::new(vec![fields[0].clone().with_name("s.v")])); + for nulls in [None, Some(NullBuffer::from(vec![true, false, true]))] { + let parent = StructArray::new( + fields.clone().into(), + vec![Arc::clone(&child)], + nulls.clone(), + ); + let output = flatten_struct_cols( + &[Arc::new(parent)], + &schema, + &HashSet::from_iter([0]), + )?; + let result = output.column(0); + result.to_data().validate_full()?; + assert_eq!(result.data_type(), child.data_type()); + if nulls.is_none() { + assert!(Arc::ptr_eq(result, &child)); + } else { + assert_eq!( + result.logical_nulls(), + Some(NullBuffer::from(vec![ + !child.data_type().is_null(), + false, + false, + ])) + ); + assert_eq!(result.slice(0, 1).as_ref(), child.slice(0, 1).as_ref()); + } + } + } + Ok(()) + } + + #[rstest::rstest] + #[case::values(false)] + #[case::buffers(true)] + #[tokio::test] + async fn test_unnest_struct_parent_nulls(#[case] check_buffers: bool) -> Result<()> { + let values = + Int32Array::from(vec![Some(10), Some(20), None, Some(40), Some(50), None]); + let child_nulls = values.nulls().cloned(); + let strings = vec![ + Some("first value longer than twelve bytes"), + Some("second value longer than twelve bytes"), + None, + Some("fourth value longer than twelve bytes"), + Some("fifth value longer than twelve bytes"), + None, + ]; + let list_values = vec![ + Some(vec![Some(1), None]), + Some(vec![]), + None, + Some(vec![Some(4)]), + Some(vec![Some(5), Some(6)]), + None, + ]; + let list = ListArray::from_iter_primitive::(list_values.clone()); + let large_list = + LargeListArray::from_iter_primitive::(list_values); + let nested = StructArray::new( + vec![Field::new("v", DataType::Int32, true)].into(), + vec![Arc::new(values.clone())], + child_nulls.clone(), + ); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", DataType::Int32, true), + ] + .into(), + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])), + Arc::new(values.clone()), + ], + None, + ); + let mut children: Vec = vec![ + Arc::new(values.clone()), + Arc::new(BooleanArray::from(vec![ + Some(true), + Some(false), + None, + Some(true), + Some(false), + None, + ])), + Arc::new(StringArray::from(strings.clone())), + Arc::new(StringViewArray::from(strings)), + Arc::new(NullArray::new(6)), + Arc::new(list.clone()), + Arc::new(large_list.clone()), + Arc::new(ListViewArray::from(list)), + Arc::new(LargeListViewArray::from(large_list)), + Arc::new(FixedSizeListArray::new( + Arc::new(Field::new_list_field(DataType::Int32, true)), + 1, + Arc::new(values.clone()), + child_nulls.clone(), + )), + Arc::new(nested), + Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 1, 2, 3, 4, 5, 6].into()), + entries, + child_nulls, + false, + )), + Arc::new(DictionaryArray::::try_new( + Int32Array::from(vec![Some(0), Some(1), None, Some(0), Some(2), None]), + Arc::new(StringArray::from(vec![Some("first"), None, Some("last")])), + )?), + Arc::new(RunArray::::try_new( + &Int32Array::from(vec![2, 3, 5, 6]), + &Int32Array::from(vec![Some(10), None, Some(40), None]), + )?), + ]; + for offsets in [None, Some(vec![0, 0, 2, 3, 4, 5].into())] { + children.push(Arc::new(UnionArray::try_new( + UnionFields::try_new( + [3, 7], + [ + Field::new("int", DataType::Int32, true), + Field::new("string", DataType::Utf8, true), + ], + )?, + vec![3, 3, 3, 7, 7, 3].into(), + offsets, + vec![ + Arc::new(values.clone()), + Arc::new(StringArray::from(vec!["text"; 6])), + ], + )?)); + } + let fields: Vec = children + .iter() + .enumerate() + .map(|(i, child)| { + Field::new(format!("v{i}"), child.data_type().clone(), true) + }) + .collect(); + let schema = Arc::new(Schema::new(fields.clone())); + for nulls in [ + None, + Some(NullBuffer::new_valid(6)), + Some(NullBuffer::from(vec![true, false, true, true, false, true])), + Some(NullBuffer::new_null(6)), + ] { + let parent = StructArray::new(fields.clone().into(), children.clone(), nulls); + for (offset, len) in [(0, 6), (1, 4)] { + let parent = parent.slice(offset, len); + let batch = RecordBatch::try_from_iter(vec![( + "s", + Arc::new(parent.clone()) as ArrayRef, + )])?; + let source = crate::test::TestMemoryExec::try_new_exec( + &[vec![batch.clone()]], + batch.schema(), + None, + )?; + let unnest = UnnestExec::new( + source, + vec![], + vec![0], + Arc::clone(&schema), + UnnestOptions::default(), + )?; + let batches = crate::common::collect( + unnest.execute(0, Arc::new(TaskContext::default()))?, + ) + .await?; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].schema(), schema); + assert_eq!(batches[0].num_rows(), len); + for (result, child) in batches[0].columns().iter().zip(parent.columns()) { + let actual = result.to_data(); + actual.validate_full()?; + assert_eq!(result.data_type(), child.data_type()); + let child_nulls = child.logical_nulls(); + let result_nulls = result.logical_nulls(); + for row in 0..len { + assert_eq!( + result_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(row)), + parent.is_null(row) + || child_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(row)), + "{:?}, row {row}", + child.data_type(), + ); + if parent.is_valid(row) { + assert_eq!( + result.slice(row, 1).as_ref(), + child.slice(row, 1).as_ref() + ); + } + } + if check_buffers && layout(child.data_type()).can_contain_null_mask { + let expected = child.to_data(); + assert_eq!(actual.offset(), expected.offset()); + assert_eq!(actual.buffers().len(), expected.buffers().len()); + for (actual, expected) in + actual.buffers().iter().zip(expected.buffers()) + { + assert!( + actual.ptr_eq(expected), + "value buffer copied for {:?}", + child.data_type() + ); + } + assert_eq!( + actual.child_data().len(), + expected.child_data().len() + ); + for (actual, expected) in + actual.child_data().iter().zip(expected.child_data()) + { + assert!( + actual.ptr_eq(expected), + "child buffers copied for {:?}", + child.data_type() + ); + } + } + } + } + } + Ok(()) + } + // Create a GenericListArray with the following list values: // [A, B, C], [], NULL, [D], NULL, [NULL, F] fn make_generic_array() -> GenericListArray diff --git a/datafusion/sqllogictest/test_files/unnest_struct_nulls.slt b/datafusion/sqllogictest/test_files/unnest_struct_nulls.slt new file mode 100644 index 0000000000000..03baeea39bd7b --- /dev/null +++ b/datafusion/sqllogictest/test_files/unnest_struct_nulls.slt @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +statement ok +SET datafusion.catalog.information_schema = true; + +statement ok +CREATE TABLE struct_values AS SELECT named_struct('a', 1, 'b', 'x') AS s; + +statement ok +CREATE TABLE null_struct AS SELECT nullif(s, s) AS s FROM struct_values; + +query B +SELECT s IS NULL FROM null_struct; +---- +true + +query IT +SELECT unnest(s) FROM null_struct; +---- +NULL NULL + +query I +SELECT unnest(arrow_cast(NULL, 'Struct("a": non-null Int32)')); +---- +NULL + +statement ok +CREATE VIEW required_fields AS +SELECT unnest(arrow_cast(named_struct('a', 1), 'Struct("a": non-null Int32)')); + +query T +SELECT is_nullable FROM information_schema.columns WHERE table_name = 'required_fields'; +---- +NO + +statement ok +CREATE VIEW nullable_fields AS +SELECT unnest(arrow_cast(NULL, 'Struct("a": non-null Int32)')); + +query T +SELECT is_nullable FROM information_schema.columns WHERE table_name = 'nullable_fields'; +---- +YES + +statement ok +CREATE TABLE mixed_structs AS +SELECT column1 AS id, named_struct('a', column2, 'b', column3) AS s +FROM (VALUES (1, 10, 'x'), (2, 20, 'y'), (3, NULL, 'z')); + +query IIT +SELECT id, unnest(nullif(s, named_struct('a', 20, 'b', 'y'))) +FROM mixed_structs ORDER BY id; +---- +1 10 x +2 NULL NULL +3 NULL z + +statement ok +CREATE TABLE nested_struct AS +SELECT named_struct('items', [1, 2], 'inner', named_struct('v', 3)) AS s; + +query ?? +SELECT unnest(nullif(s, s)) FROM nested_struct; +---- +NULL NULL + +query I +SELECT unnest(nullif(s, s)['inner']) FROM nested_struct; +---- +NULL + +query I +WITH unnested(items, inner_struct) AS ( + SELECT unnest(nullif(s, s)) FROM nested_struct +) +SELECT unnest(inner_struct) FROM unnested; +---- +NULL + +query ITI +SELECT unnest(nullif(s, s)), unnest([1, 2]) FROM struct_values; +---- +NULL NULL 1 +NULL NULL 2 + +query ? +SELECT unnest(arrow_cast(NULL, 'Struct("a": Null)')); +---- +NULL + +query IT +SELECT unnest(s) FROM struct_values WHERE false; +---- + +statement ok +SET datafusion.catalog.information_schema = false;