mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-23 04:02:27 +00:00
feat(python): accept expressions in update filters (#3876)
## Summary - allow Python sync, async, and remote table updates to accept type-safe `Expr` filters - serialize expression filters before invoking the existing update implementation - cover numeric-looking text and apostrophe-containing text in sync and async regression tests ## Root cause `Table.update` was the remaining Python write path that required callers to construct a raw SQL predicate. Dynamic text interpolated without SQL literal encoding could therefore be parsed as an integer, float, or unterminated string instead of Utf8. The expression API already encodes literals safely for query and delete filters. ## Validation - `cd python && .venv/bin/pytest python/tests/test_table.py::test_update_async python/tests/test_table.py::test_update_expr_filter_literals_async python/tests/test_table.py::test_update python/tests/test_table.py::test_update_expr_filter_literals -q` - `cd python && .venv/bin/pytest python/tests/test_expr.py -q` - `cd python && .venv/bin/ruff format --check .` - `cd python && .venv/bin/ruff check .` Fixes #1869 <!-- lance-gatekeeper-fix:v1 agent=01f1e7b69c65e8b6d3b3c1e1a7918179 generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
lancedb-gatefixer[bot]
co-authored by
Xuanwo
Xuanwo
parent
0dd9dfdfc7
commit
25645d82d4
@@ -11,6 +11,7 @@ import warnings
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from time import sleep
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
@@ -336,6 +337,21 @@ async def test_update_async(mem_db_async: AsyncConnection):
|
||||
assert await table.count_rows("id == 10") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
|
||||
values = ["5", "4.66e-84", "it's"]
|
||||
table = await mem_db_async.create_table(
|
||||
"update_expr_literals",
|
||||
data=[{"field": value, "result": "original"} for value in values],
|
||||
)
|
||||
|
||||
for value in values:
|
||||
update_res = await table.update({"result": value}, where=col("field") == value)
|
||||
assert update_res.rows_updated == 1
|
||||
|
||||
assert (await table.to_arrow())["result"].to_pylist() == values
|
||||
|
||||
|
||||
def test_create_table(mem_db: DBConnection):
|
||||
schema = pa.schema(
|
||||
{
|
||||
@@ -2343,6 +2359,148 @@ def test_update(mem_db: DBConnection):
|
||||
assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
|
||||
|
||||
|
||||
def test_update_expr_filter_literals(mem_db: DBConnection):
|
||||
values = ["5", "4.66e-84", "it's"]
|
||||
table = mem_db.create_table(
|
||||
"update_expr_literals",
|
||||
data=[{"field": value, "result": "original"} for value in values],
|
||||
)
|
||||
|
||||
for value in values:
|
||||
update_res = table.update(where=col("field") == value, values={"result": value})
|
||||
assert update_res.rows_updated == 1
|
||||
|
||||
assert table.to_arrow()["result"].to_pylist() == values
|
||||
|
||||
|
||||
def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
|
||||
low = Decimal("1.234567890123456789")
|
||||
high = Decimal("1.234567890123456790")
|
||||
decimal_schema = pa.schema(
|
||||
[("val", pa.decimal128(19, 18)), ("result", pa.string())]
|
||||
)
|
||||
decimal_table = mem_db.create_table(
|
||||
"update_expr_decimal",
|
||||
pa.table(
|
||||
{"val": [low, high], "result": ["old", "old"]},
|
||||
schema=decimal_schema,
|
||||
),
|
||||
)
|
||||
predicate = col("val") < lit(high)
|
||||
assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = decimal_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
keyword_table = mem_db.create_table(
|
||||
"update_expr_keyword", [{"null": 1, "result": "old"}]
|
||||
)
|
||||
predicate = col("null") == 1
|
||||
assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = keyword_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
empty_in_table = mem_db.create_table(
|
||||
"update_expr_empty_in", [{"id": 1, "result": "old"}]
|
||||
)
|
||||
predicate = col("id").isin([])
|
||||
assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
|
||||
result = empty_in_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 0
|
||||
|
||||
marker = "__lancedb_binary_placeholder_0__"
|
||||
binary_schema = pa.schema(
|
||||
[("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
|
||||
)
|
||||
binary_table = mem_db.create_table(
|
||||
"update_expr_binary",
|
||||
pa.table(
|
||||
{
|
||||
"payload": [b"\x01", b"\x02"],
|
||||
"text": ["other", marker],
|
||||
"result": ["old", "old"],
|
||||
},
|
||||
schema=binary_schema,
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
|
||||
assert binary_table.search().where(predicate).to_arrow().num_rows == 2
|
||||
result = binary_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 2
|
||||
|
||||
nonfinite_table = mem_db.create_table(
|
||||
"update_expr_nonfinite",
|
||||
[{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
|
||||
)
|
||||
predicate = col("x") < float("inf")
|
||||
assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
|
||||
result = nonfinite_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 2
|
||||
|
||||
float16_table = mem_db.create_table(
|
||||
"update_expr_float16",
|
||||
[{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
|
||||
)
|
||||
predicate = col("x").cast(pa.float16()) < 2.0
|
||||
assert float16_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = float16_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
string_cast_table = mem_db.create_table(
|
||||
"update_expr_string_cast",
|
||||
[{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
|
||||
)
|
||||
predicate = col("x").cast("string") == "1"
|
||||
assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = string_cast_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
quoted_identifier_schema = pa.schema(
|
||||
[("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
|
||||
)
|
||||
quoted_identifier_table = mem_db.create_table(
|
||||
"update_expr_quoted_identifier",
|
||||
pa.table(
|
||||
{"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
|
||||
schema=quoted_identifier_schema,
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
|
||||
assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
decimal256_schema = pa.schema(
|
||||
[("val", pa.decimal256(40, 2)), ("result", pa.string())]
|
||||
)
|
||||
decimal256_table = mem_db.create_table(
|
||||
"update_expr_decimal256",
|
||||
pa.table(
|
||||
{
|
||||
"val": [Decimal("1.00"), Decimal("3.00")],
|
||||
"result": ["old", "old"],
|
||||
},
|
||||
schema=decimal256_schema,
|
||||
),
|
||||
)
|
||||
predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
|
||||
assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = decimal256_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
binary_empty_table = mem_db.create_table(
|
||||
"update_expr_binary_empty",
|
||||
pa.table(
|
||||
{"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
|
||||
schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")).isin([])
|
||||
assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
|
||||
assert predicate.to_sql() == "false"
|
||||
result = binary_empty_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 0
|
||||
|
||||
|
||||
def test_update_with_arrow_scalar(mem_db: DBConnection):
|
||||
schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
|
||||
table = mem_db.create_table("my_table", schema=schema)
|
||||
|
||||
Reference in New Issue
Block a user