From 1f378c1b4999d753facc82f5b829c664069948d1 Mon Sep 17 00:00:00 2001 From: Kevin Dong <2970752254@qq.com> Date: Fri, 9 Oct 2026 16:51:56 -0700 Subject: [PATCH] fix(python): preserve Decimal precision in table updates (#4431) Python `Decimal` values and Arrow decimal scalars raise `NotImplementedError` in `table.update(values=...)`. Register a Decimal converter that emits quoted fixed-point text so the native cast to the target column preserves every digit, including decimal128 and decimal256 values. Non-finite Decimal values raise `ValueError` with a static message. Closes #4360. Validation with the native extension built from the current source at 7f593359, on macOS arm64 and Python 3.12.14: - All 33 new regression cases failed before the fix and passed afterward. They cover Python and Arrow scalar updates, signs, zero, exponents, full 38/76-digit precision, and a Decimal context precision of 6. - 16 existing converter, update, and Decimal expression tests passed. - Repository Ruff formatting and lint passed; formatting left all 156 files unchanged. - A native round-trip check confirmed that an unquoted SQL number rounds a high-precision value while the quoted Decimal update preserves it exactly. The full suite, other Python versions, and remote API were not exercised. AI assistance: Codex helped implement and validate the Decimal conversion and its regression tests. --- python/python/lancedb/util.py | 9 ++++++ python/python/tests/test_table.py | 47 ++++++++++++++++++++++++++++++- python/python/tests/test_util.py | 36 +++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index 3a6a8a102..fefc234ce 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -10,6 +10,7 @@ import os import pathlib import warnings from datetime import date, datetime +from decimal import Decimal from functools import singledispatch from typing import Tuple, Union, Optional, Any, List from urllib.parse import urlparse @@ -373,6 +374,14 @@ def _(value: float): return str(value) +@value_to_sql.register(Decimal) +def _(value: Decimal): + if not value.is_finite(): + raise ValueError("Non-finite Decimal values cannot be converted to SQL") + # Quote exact decimal text so the target-column cast avoids float rounding. + return value_to_sql(format(value, "f")) + + @value_to_sql.register(bool) def _(value: bool): return str(value).upper() diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index c579f3037..a963a0d19 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -12,7 +12,7 @@ import warnings import weakref from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, timedelta -from decimal import Decimal +from decimal import Decimal, localcontext from time import sleep from typing import List from unittest.mock import patch @@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection): assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]] +@pytest.mark.parametrize("use_arrow_scalar", [False, True]) +@pytest.mark.parametrize( + "decimal_type, value", + [ + (pa.decimal128(10, 2), "9.99"), + (pa.decimal128(10, 2), "-9.99"), + (pa.decimal128(10, 2), "0.00"), + (pa.decimal128(10, 2), "1.23E+5"), + (pa.decimal128(38, 18), "1E-18"), + (pa.decimal128(38, 18), "12345678901234567.123456789012345678"), + (pa.decimal128(38, 18), "12345678901234567890.123456789012345678"), + (pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"), + ( + pa.decimal256(76, 38), + "12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + ), + ( + pa.decimal256(76, 38), + "-12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + ), + ], +) +def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar): + expected = Decimal(value) + table = mem_db.create_table( + "my_table", + pa.table( + { + "id": [1, 2], + "price": pa.array([Decimal("1"), expected], type=decimal_type), + } + ), + ) + update = table.to_arrow()["price"][1] if use_arrow_scalar else expected + with localcontext() as context: + context.prec = 6 + result = table.update(where="id = 1", values={"price": update}) + assert result.rows_updated == 1 + actual = table.to_arrow() + assert actual.schema.field("price").type == decimal_type + assert actual["price"].to_pylist() == [expected, expected] + + def test_update_types(mem_db: DBConnection): table = mem_db.create_table( "my_table", diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index d9c574eb9..fe5187604 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -4,6 +4,7 @@ import os import pathlib +from decimal import Decimal, localcontext from typing import Optional import lance @@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path): assert table.to_pandas().query("search == @value")["replace"].item() == value +@pytest.mark.parametrize( + "value, expected", + [ + ("9.99", "'9.99'"), + ("-9.99", "'-9.99'"), + ("0", "'0'"), + ("-0.00", "'-0.00'"), + ("1.23E+5", "'123000'"), + ("1E-18", "'0.000000000000000001'"), + ( + "12345678901234567890.123456789012345678", + "'12345678901234567890.123456789012345678'", + ), + ( + "12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + "'12345678901234567890123456789012345678." + "12345678901234567890123456789012345678'", + ), + ], +) +def test_value_to_sql_decimal(value, expected): + with localcontext() as context: + context.prec = 6 + assert value_to_sql(Decimal(value)) == expected + + +@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"]) +def test_value_to_sql_decimal_non_finite(value): + with pytest.raises( + ValueError, match="^Non-finite Decimal values cannot be converted to SQL$" + ): + value_to_sql(Decimal(value)) + + def test_value_to_sql_dict(): # Simple flat struct assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"