Merge main into gatekeeper/fix-4260-1

Align the vendored Arrow crate and JSONB pin with Lance 14.0.0-beta.11. Preserve the sliced-validity repair and carry the release's Null-child fix.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.41.0-beta.2"
+version = "0.41.0-beta.4"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/util.py

@@ -10,6 +10,7 @@ import os
 import pathlib
 import warnings
 from datetime import date, datetime
+from decimal import Decimal
 from functools import singledispatch
 from typing import Tuple, Union, Optional, Any, List
 from urllib.parse import urlparse
@@ -373,6 +374,14 @@ def _(value: float):
     return str(value)
 
 
+@value_to_sql.register(Decimal)
+def _(value: Decimal):
+    if not value.is_finite():
+        raise ValueError("Non-finite Decimal values cannot be converted to SQL")
+    # Quote exact decimal text so the target-column cast avoids float rounding.
+    return value_to_sql(format(value, "f"))
+
+
 @value_to_sql.register(bool)
 def _(value: bool):
     return str(value).upper()

python/python/tests/test_table.py

@@ -12,7 +12,7 @@ import warnings
 import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
-from decimal import Decimal
+from decimal import Decimal, localcontext
 from time import sleep
 from typing import List
 from unittest.mock import patch
@@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
     assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
 
 
+@pytest.mark.parametrize("use_arrow_scalar", [False, True])
+@pytest.mark.parametrize(
+    "decimal_type, value",
+    [
+        (pa.decimal128(10, 2), "9.99"),
+        (pa.decimal128(10, 2), "-9.99"),
+        (pa.decimal128(10, 2), "0.00"),
+        (pa.decimal128(10, 2), "1.23E+5"),
+        (pa.decimal128(38, 18), "1E-18"),
+        (pa.decimal128(38, 18), "12345678901234567.123456789012345678"),
+        (pa.decimal128(38, 18), "12345678901234567890.123456789012345678"),
+        (pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"),
+        (
+            pa.decimal256(76, 38),
+            "12345678901234567890123456789012345678."
+            "12345678901234567890123456789012345678",
+        ),
+        (
+            pa.decimal256(76, 38),
+            "-12345678901234567890123456789012345678."
+            "12345678901234567890123456789012345678",
+        ),
+    ],
+)
+def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar):
+    expected = Decimal(value)
+    table = mem_db.create_table(
+        "my_table",
+        pa.table(
+            {
+                "id": [1, 2],
+                "price": pa.array([Decimal("1"), expected], type=decimal_type),
+            }
+        ),
+    )
+    update = table.to_arrow()["price"][1] if use_arrow_scalar else expected
+    with localcontext() as context:
+        context.prec = 6
+        result = table.update(where="id = 1", values={"price": update})
+    assert result.rows_updated == 1
+    actual = table.to_arrow()
+    assert actual.schema.field("price").type == decimal_type
+    assert actual["price"].to_pylist() == [expected, expected]
+
+
 def test_update_types(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",

python/python/tests/test_util.py

@@ -4,6 +4,7 @@
 
 import os
 import pathlib
+from decimal import Decimal, localcontext
 from typing import Optional
 
 import lance
@@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path):
         assert table.to_pandas().query("search == @value")["replace"].item() == value
 
 
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        ("9.99", "'9.99'"),
+        ("-9.99", "'-9.99'"),
+        ("0", "'0'"),
+        ("-0.00", "'-0.00'"),
+        ("1.23E+5", "'123000'"),
+        ("1E-18", "'0.000000000000000001'"),
+        (
+            "12345678901234567890.123456789012345678",
+            "'12345678901234567890.123456789012345678'",
+        ),
+        (
+            "12345678901234567890123456789012345678."
+            "12345678901234567890123456789012345678",
+            "'12345678901234567890123456789012345678."
+            "12345678901234567890123456789012345678'",
+        ),
+    ],
+)
+def test_value_to_sql_decimal(value, expected):
+    with localcontext() as context:
+        context.prec = 6
+        assert value_to_sql(Decimal(value)) == expected
+
+
+@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"])
+def test_value_to_sql_decimal_non_finite(value):
+    with pytest.raises(
+        ValueError, match="^Non-finite Decimal values cannot be converted to SQL$"
+    ):
+        value_to_sql(Decimal(value))
+
+
 def test_value_to_sql_dict():
     # Simple flat struct
     assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"

python/vendor/lance-arrow/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lance-arrow"
-version = "14.0.0-beta.10"
+version = "14.0.0-beta.11"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"

python/vendor/lance-arrow/README.lancedb.md

@@ -1,5 +1,5 @@
-This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.10`
-(commit `fa3638fd816ab2ff9a2ecbba7e51465d8879958f`), licensed under Apache-2.0.
+This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.11`
+(commit `f9c563fe1516a48b5c40d255c1d884af2f3c3d6c`), licensed under Apache-2.0.
 
 LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
 until a Lance release includes the fix. The patch is selected in the root

python/vendor/lance-arrow/src/struct.rs

@@ -7,7 +7,7 @@
 use arrow_array::{Array, StructArray, cast::AsArray, make_array};
 use arrow_buffer::NullBuffer;
 use arrow_data::{ArrayData, ArrayDataBuilder};
-use arrow_schema::ArrowError;
+use arrow_schema::{ArrowError, DataType};
 
 pub trait StructArrayExt {
     /// Removes the offset / length of the struct array by pushing it into the children
@@ -104,7 +104,11 @@ impl StructArrayExt for StructArray {
             .child_data()
             .iter()
             .map(|c| {
-                if let Some(child_validity) = c.nulls() {
+                if c.data_type() == &DataType::Null {
+                    // A Null array is already null in every slot, and arrow-rs rejects
+                    // a validity buffer on it.
+                    Ok(c.clone())
+                } else if let Some(child_validity) = c.nulls() {
                     let new_validity = child_validity.inner() & validity.inner();
                     c.clone()
                         .into_builder()
@@ -125,7 +129,10 @@ impl StructArrayExt for StructArray {
 
 #[cfg(test)]
 mod tests {
-    use arrow_array::{Array, Int32Array, StructArray, cast::AsArray, make_array};
+    use arrow_array::{
+        Array, Int32Array, NullArray, StructArray, cast::AsArray, make_array, types::Int32Type,
+    };
+    use arrow_buffer::NullBuffer;
     use arrow_schema::{DataType, Field, Fields};
     use std::sync::Arc;
 
@@ -189,4 +196,32 @@ mod tests {
 
         assert_eq!(normalized, struct_array.slice(1, 2));
     }
+
+    #[test]
+    fn test_pushdown_nulls_with_null_typed_child() {
+        // `pa.Table.from_pylist([{"s": {"a": 1, "b": None}}, {"s": None}])` infers a
+        // struct with a Null-typed child. Arrow forbids a validity buffer on a Null
+        // array, so the struct's nulls must not be pushed into it.
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Null, true),
+            ]),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(NullArray::new(3)),
+            ],
+            Some(NullBuffer::from(vec![true, false, true])),
+        );
+
+        let pushed = struct_array.pushdown_nulls().unwrap();
+
+        assert_eq!(pushed.nulls(), struct_array.nulls());
+        assert_eq!(
+            pushed.column(0).as_primitive::<Int32Type>(),
+            &Int32Array::from(vec![Some(1), None, Some(3)])
+        );
+        assert_eq!(pushed.column(1).data_type(), &DataType::Null);
+        assert_eq!(pushed.column(1).logical_null_count(), 3);
+    }
 }