fix(python): parse decimal exponent literals exactly (#4449)

Decimal literals such as Decimal("1E-7") and Decimal("1E+3") currently
raise ValueError while building an expression. Parse the coefficient and
exponent exactly, and normalize representable negative scales to zero so
both typed filters and predicate.to_sql() queries preserve the value.
Checked arithmetic bounds the Decimal128 coefficient to 38 digits,
preserves existing scale limits, and rejects nonfinite or overflowing
values. This does not add Decimal256 literal support.

Added 21 cases covering actual Arrow-backed table queries, SQL queries,
zero, signed exponents, precision boundaries, and errors. Both Python
3.11 and 3.13 expression suites pass 123 tests; the baseline fails 10 of
the new cases. Python binding Rust tests pass 13; official make doctest
passes 77 with 6 skips; Ruff, Rustfmt, hooks, Pyright and
workspace/tests/all-features Clippy with -D warnings pass. A fresh
environment restored from the original dependency freeze passes the full
required suite: 1528 passed, 37 skipped, 68 deselected (143.51s); while
optional Torch suites have the same baseline remote mock-server ordering
failure.

Tested against f4544628; observed official main d864e719 contains one
Catalog API commit without expr or Cargo.lock changes. AI assistance:
gpt-6.1-sol assisted implementation and local validation; all reported
checks were actually executed.
This commit is contained in:
LeonSGP
2026-10-23 02:43:50 +08:00
committed by GitHub
parent 13df747f18
commit 1e313d3c0d
2 changed files with 109 additions and 10 deletions
+61
View File
@@ -80,6 +80,67 @@ class TestExprConstruction:
e = lit(d)
assert isinstance(e, Expr)
@pytest.mark.parametrize(
"value, decimal_type",
[
("1E-7", pa.decimal128(12, 7)),
("-1.23E-7", pa.decimal128(12, 9)),
("1E+3", pa.decimal128(12, 0)),
("-1.23E+5", pa.decimal128(12, 0)),
("0E-7", pa.decimal128(12, 7)),
("-0E+3", pa.decimal128(12, 0)),
("1E-38", pa.decimal128(38, 38)),
("1E+37", pa.decimal128(38, 0)),
("1.234567890123456789012345678901234567E+37", pa.decimal128(38, 0)),
("0E+128", pa.decimal128(1, 0)),
],
)
def test_lit_decimal_exponent_query(self, tmp_path, value, decimal_type):
expected = Decimal(value)
table = lancedb.connect(tmp_path).create_table(
"decimals",
pa.table(
{
"id": [1, 2],
# Arrow's Python ingestion rejects 0E+128 even though its
# numeric value is representable. Store canonical zero to
# exercise the expression literal independently of ingestion.
"price": pa.array(
[Decimal(0) if expected.is_zero() else expected, None],
type=decimal_type,
),
}
),
)
predicate = col("price") == lit(expected)
result = table.search().where(predicate).to_arrow()
assert result["id"].to_pylist() == [1]
assert result["price"].to_pylist() == [expected]
# Exercise the SQL serializer and parser as well as the typed query path.
sql_result = table.search().where(predicate.to_sql()).to_arrow()
assert sql_result["id"].to_pylist() == [1]
assert sql_result["price"].to_pylist() == [expected]
@pytest.mark.parametrize(
"value",
[
"NaN",
"sNaN",
"Infinity",
"-Infinity",
"1E-39",
"1E+38",
"1E+129",
"1E+1000000000",
"0E+129",
"12345678901234567890123456789012345678E+1",
"9" * 39,
],
)
def test_lit_decimal_unrepresentable_raises(self, value):
with pytest.raises(ValueError):
lit(Decimal(value))
class TestExprOperators:
def test_eq_operator(self):