Merge main into gatekeeper/fix-4260-1

Align the vendored Arrow crate and JSONB pin with Lance 14.0.0-beta.11. Preserve the sliced-validity repair and carry the release's Null-child fix.

python/vendor/lance-arrow/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lance-arrow"
-version = "14.0.0-beta.10"
+version = "14.0.0-beta.11"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"

python/vendor/lance-arrow/README.lancedb.md

@@ -1,5 +1,5 @@
-This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.10`
-(commit `fa3638fd816ab2ff9a2ecbba7e51465d8879958f`), licensed under Apache-2.0.
+This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.11`
+(commit `f9c563fe1516a48b5c40d255c1d884af2f3c3d6c`), licensed under Apache-2.0.
 
 LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
 until a Lance release includes the fix. The patch is selected in the root

python/vendor/lance-arrow/src/struct.rs

@@ -7,7 +7,7 @@
 use arrow_array::{Array, StructArray, cast::AsArray, make_array};
 use arrow_buffer::NullBuffer;
 use arrow_data::{ArrayData, ArrayDataBuilder};
-use arrow_schema::ArrowError;
+use arrow_schema::{ArrowError, DataType};
 
 pub trait StructArrayExt {
     /// Removes the offset / length of the struct array by pushing it into the children
@@ -104,7 +104,11 @@ impl StructArrayExt for StructArray {
             .child_data()
             .iter()
             .map(|c| {
-                if let Some(child_validity) = c.nulls() {
+                if c.data_type() == &DataType::Null {
+                    // A Null array is already null in every slot, and arrow-rs rejects
+                    // a validity buffer on it.
+                    Ok(c.clone())
+                } else if let Some(child_validity) = c.nulls() {
                     let new_validity = child_validity.inner() & validity.inner();
                     c.clone()
                         .into_builder()
@@ -125,7 +129,10 @@ impl StructArrayExt for StructArray {
 
 #[cfg(test)]
 mod tests {
-    use arrow_array::{Array, Int32Array, StructArray, cast::AsArray, make_array};
+    use arrow_array::{
+        Array, Int32Array, NullArray, StructArray, cast::AsArray, make_array, types::Int32Type,
+    };
+    use arrow_buffer::NullBuffer;
     use arrow_schema::{DataType, Field, Fields};
     use std::sync::Arc;
 
@@ -189,4 +196,32 @@ mod tests {
 
         assert_eq!(normalized, struct_array.slice(1, 2));
     }
+
+    #[test]
+    fn test_pushdown_nulls_with_null_typed_child() {
+        // `pa.Table.from_pylist([{"s": {"a": 1, "b": None}}, {"s": None}])` infers a
+        // struct with a Null-typed child. Arrow forbids a validity buffer on a Null
+        // array, so the struct's nulls must not be pushed into it.
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Null, true),
+            ]),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(NullArray::new(3)),
+            ],
+            Some(NullBuffer::from(vec![true, false, true])),
+        );
+
+        let pushed = struct_array.pushdown_nulls().unwrap();
+
+        assert_eq!(pushed.nulls(), struct_array.nulls());
+        assert_eq!(
+            pushed.column(0).as_primitive::<Int32Type>(),
+            &Int32Array::from(vec![Some(1), None, Some(3)])
+        );
+        assert_eq!(pushed.column(1).data_type(), &DataType::Null);
+        assert_eq!(pushed.column(1).logical_null_count(), 3);
+    }
 }