diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index 926769f48..5b4c0c343 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -270,7 +270,8 @@ def _iter_projection_pairs( if isinstance(expr, str): yield name, expr elif isinstance(expr, Expr): - yield name, expr.to_sql() + source = expr._column_name() + yield name, source if source is not None else expr.to_sql() return for column in projection: if isinstance(column, str): @@ -280,7 +281,8 @@ def _iter_projection_pairs( if isinstance(expr, str): yield name, expr elif isinstance(expr, Expr): - yield name, expr.to_sql() + source = expr._column_name() + yield name, source if source is not None else expr.to_sql() def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table: diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 593bceffa..7d7ca7f2a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -87,6 +87,7 @@ class PyExpr: def contains(self, substr: "PyExpr") -> "PyExpr": ... def isin(self, values: List["PyExpr"]) -> "PyExpr": ... def cast(self, data_type: pa.DataType) -> "PyExpr": ... + def column_name(self) -> Optional[str]: ... def to_sql(self) -> str: ... def expr_col(name: str) -> PyExpr: ... @@ -608,6 +609,7 @@ class PyQueryRequest: filter: Optional[Union[str, bytes]] full_text_search: Optional[FullTextQuery] select: Optional[Union[str, List[str]]] + select_source_columns: Optional[Dict[str, str]] fast_search: Optional[bool] with_row_id: Optional[bool] use_lsm: Optional[bool] diff --git a/python/python/lancedb/expr.py b/python/python/lancedb/expr.py index d16ba95d7..80d01e29a 100644 --- a/python/python/lancedb/expr.py +++ b/python/python/lancedb/expr.py @@ -249,6 +249,10 @@ class Expr: # ── utilities ──────────────────────────────────────────────────────────── + def _column_name(self) -> str | None: + """Return the source name when this is a bare column expression.""" + return self._inner.column_name() + def to_sql(self) -> str: """Render the expression as a SQL string (useful for debugging).""" return self._inner.to_sql() @@ -312,7 +316,7 @@ def func(name: str, *args: ExprLike) -> Expr: -------- >>> from lancedb.expr import col, func >>> func("lower", col("name")) - Expr(lower(name)) + Expr(lower(`name`)) """ inner_args = [_coerce(a)._inner for a in args] return Expr(expr_func(name, inner_args)) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 9301d7df8..451384ad1 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -167,6 +167,12 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]: return {"columns": projection} +def _query_request_projection(req: "PyQueryRequest") -> QueryProjection: + if req.select_source_columns is not None: + return req.select_source_columns + return req.select + + def _scanner_kwargs_for_query( query: Query, blob_mode: BlobMode, @@ -2799,15 +2805,16 @@ class AsyncQueryBase(object): req = self._inner.to_query_request() schema = await self._table.schema() + projection = _query_request_projection(req) self._blob_auto_row_id = blob_auto_row_id_for_scan( schema, - req.select, + projection, with_row_id=self._with_row_id, ) if not self._blob_auto_row_id: self._blob_paths = () return - self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys()) + self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys()) self._inner.with_row_id() def select(self, columns: Union[List[str], dict[str, str]]) -> Self: @@ -3894,14 +3901,15 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase): blob_paths: tuple[str, ...] = () if self._table is not None: schema = await self._table.schema() + projection = _query_request_projection(req) blob_auto_row_id = blob_auto_row_id_for_scan( schema, - req.select, + projection, with_row_id=self._with_row_id, ) if blob_auto_row_id: blob_paths = tuple( - blob_v2_projection_sources(schema, req.select).keys() + blob_v2_projection_sources(schema, projection).keys() ) self._blob_auto_row_id = blob_auto_row_id self._blob_paths = blob_paths diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index d9139396b..02748b9bc 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -36,6 +36,7 @@ from lancedb._lancedb import ( UpdateResult, ) from lancedb.embeddings.base import EmbeddingFunctionConfig +from lancedb.expr import Expr from lancedb.index import ( FTS, BTree, @@ -863,7 +864,7 @@ class RemoteTable(Table): def update( self, - where: Optional[str] = None, + where: Optional[Union[str, Expr]] = None, values: Optional[dict] = None, *, values_sql: Optional[Dict[str, str]] = None, @@ -874,9 +875,11 @@ class RemoteTable(Table): Parameters ---------- - where: str, optional - The SQL where clause to use when updating rows. For example, 'x = 2' - or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. + where: str or [Expr][lancedb.expr.Expr], optional + The filter condition. Can be a SQL string or a type-safe + [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and + [lit][lancedb.expr.lit]. The filter must not be empty, or it will + error. values: dict, optional The values to update. The keys are the column names and the values are the values to set. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index c354a944e..765b7fa14 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1744,7 +1744,7 @@ class Table(ABC): @abstractmethod def update( self, - where: Optional[str] = None, + where: Optional[Union[str, Expr]] = None, values: Optional[dict] = None, *, values_sql: Optional[Dict[str, str]] = None, @@ -1759,9 +1759,11 @@ class Table(ABC): Parameters ---------- - where: str, optional - The SQL where clause to use when updating rows. For example, 'x = 2' - or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. + where: str or [Expr][lancedb.expr.Expr], optional + The filter condition. Can be a SQL string or a type-safe + [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and + [lit][lancedb.expr.lit]. The filter must not be empty, or it will + error. values: dict, optional The values to update. The keys are the column names and the values are the values to set. @@ -1779,6 +1781,7 @@ class Table(ABC): Examples -------- >>> import lancedb + >>> from lancedb.expr import col >>> import pandas as pd >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]}) >>> db = lancedb.connect("./.lancedb") @@ -1788,7 +1791,7 @@ class Table(ABC): 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] - >>> table.update(where="x = 2", values={"vector": [10.0, 10]}) + >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]}) UpdateResult(rows_updated=1, version=2) >>> table.to_pandas() x vector @@ -3841,7 +3844,7 @@ class LanceTable(Table): def update( self, - where: Optional[str] = None, + where: Optional[Union[str, Expr]] = None, values: Optional[dict] = None, *, values_sql: Optional[Dict[str, str]] = None, @@ -3852,9 +3855,11 @@ class LanceTable(Table): Parameters ---------- - where: str, optional - The SQL where clause to use when updating rows. For example, 'x = 2' - or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. + where: str or [Expr][lancedb.expr.Expr], optional + The filter condition. Can be a SQL string or a type-safe + [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and + [lit][lancedb.expr.lit]. The filter must not be empty, or it will + error. values: dict, optional The values to update. The keys are the column names and the values are the values to set. @@ -3872,6 +3877,7 @@ class LanceTable(Table): Examples -------- >>> import lancedb + >>> from lancedb.expr import col >>> import pandas as pd >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]}) >>> db = lancedb.connect("./.lancedb") @@ -3881,7 +3887,7 @@ class LanceTable(Table): 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] - >>> table.update(where="x = 2", values={"vector": [10.0, 10]}) + >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]}) UpdateResult(rows_updated=1, version=2) >>> table.to_pandas() x vector @@ -5995,7 +6001,7 @@ class AsyncTable: self, updates: Optional[Dict[str, Any]] = None, *, - where: Optional[str] = None, + where: Optional[Union[str, Expr]] = None, updates_sql: Optional[Dict[str, str]] = None, ) -> UpdateResult: """ @@ -6010,9 +6016,11 @@ class AsyncTable: The updates to apply. The keys should be the name of the column to update. The values should be the new values to assign. This is required unless updates_sql is supplied. - where: str, optional - An SQL filter that controls which rows are updated. For example, 'x = 2' - or 'x IN (1, 2, 3)'. Only rows that satisfy this filter will be udpated. + where: str or [Expr][lancedb.expr.Expr], optional + The filter condition. Can be a SQL string or a type-safe + [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and + [lit][lancedb.expr.lit]. Only rows that satisfy this filter will + be updated. updates_sql: dict, optional The updates to apply, expressed as SQL expression strings. The keys should be column names. The values should be SQL expressions. These can be SQL @@ -6030,13 +6038,14 @@ class AsyncTable: -------- >>> import asyncio >>> import lancedb + >>> from lancedb.expr import col >>> import pandas as pd >>> async def demo_update(): ... data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]}) ... db = await lancedb.connect_async("./.lancedb") ... table = await db.create_table("my_table", data) ... # x is [1, 2], vector is [[1, 2], [3, 4]] - ... await table.update({"vector": [10, 10]}, where="x = 2") + ... await table.update({"vector": [10, 10]}, where=col("x") == 2) ... # x is [1, 2], vector is [[1, 2], [10, 10]] ... await table.update(updates_sql={"x": "x + 1"}) ... # x is [2, 3], vector is [[1, 2], [10, 10]] @@ -6050,7 +6059,8 @@ class AsyncTable: if updates is not None: updates_sql = {k: value_to_sql(v) for k, v in updates.items()} - return await self._inner.update(updates_sql, where) + predicate = where.to_sql() if isinstance(where, Expr) else where + return await self._inner.update(updates_sql, predicate) async def add_columns( self, diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 5d7682f24..351769ff6 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -8,7 +8,12 @@ import pyarrow.compute as pc import pytest import lancedb -from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids +from lancedb._blob import ( + blob_v2_projection_sources, + read_row_ids_from_hits, + stash_auto_row_ids, +) +from lancedb.expr import col from lancedb.index import FTS from lancedb.schema import blob_column_paths, blob_v2_column_paths @@ -70,6 +75,14 @@ def test_blob_v2_column_paths_include_list_children(): ] +def test_blob_v2_projection_sources_use_typed_column_name(): + schema = pa.schema([lancedb.blob("blob")]) + + assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == { + "blob_alias": "blob" + } + + def _legacy_v1_table(name): db = lancedb.connect("memory:///") schema = pa.schema( @@ -166,6 +179,20 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id(): assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"} +@pytest.mark.asyncio +async def test_async_typed_blob_projection_preserves_source_column(): + db = await lancedb.connect_async("memory:///typed_blob_projection") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")]) + table = await db.create_table("typed_blob_projection", schema=schema) + await table.add([{"id": 1, "blob": b"alpha"}]) + + hits = await table.query().select({"blob_alias": col("blob")}).to_arrow() + + assert "_lance_row_id" in hits.schema.field("blob_alias").type.names + blobs = await table.fetch_blobs("blob", hits) + assert blobs.to_pylist() == [b"alpha"] + + def test_fetch_blobs_round_trip(): table = _blob_table( "round_trip", @@ -403,6 +430,50 @@ async def test_blob_v2_hybrid_fetch_blobs_async(): assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"} +@pytest.mark.asyncio +async def test_async_hybrid_typed_blob_projection_preserves_source_column(): + db = await lancedb.connect_async("memory:///hybrid_typed_blob") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("text", pa.utf8()), + pa.field("vector", pa.list_(pa.float32(), list_size=2)), + lancedb.blob("blob"), + ] + ) + table = await db.create_table("hybrid_typed_blob", schema=schema) + await table.add( + [ + { + "id": 1, + "text": "hello alpha", + "vector": [1.0, 0.0], + "blob": b"alpha", + }, + { + "id": 2, + "text": "hello beta", + "vector": [0.9, 0.1], + "blob": b"beta", + }, + ] + ) + await table.create_index("text", config=FTS(with_position=False)) + + hits = await ( + table.query() + .nearest_to([1.0, 0.0]) + .nearest_to_text("hello") + .select({"blob_alias": col("blob")}) + .limit(2) + .to_arrow() + ) + + assert "_lance_row_id" in hits.schema.field("blob_alias").type.names + blobs = await table.fetch_blobs("blob", hits) + assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"} + + def test_blob_file_seek_read_and_read_range(): payload = _identifiable_payload(1024) table = _blob_table("seek_read", [{"id": 1, "image": payload}]) diff --git a/python/python/tests/test_expr.py b/python/python/tests/test_expr.py index 0eb6f8929..0f49231f1 100644 --- a/python/python/tests/test_expr.py +++ b/python/python/tests/test_expr.py @@ -52,7 +52,7 @@ class TestExprConstruction: def test_func(self): e = func("lower", col("name")) assert isinstance(e, Expr) - assert e.to_sql() == "lower(name)" + assert e.to_sql() == "lower(`name`)" def test_func_unknown_raises(self): with pytest.raises(Exception): @@ -115,7 +115,7 @@ class TestExprOperators: def test_and_operator(self): e = (col("age") > lit(18)) & (col("status") == lit("active")) assert isinstance(e, Expr) - assert e.to_sql() == "((age > 18) AND (status = 'active'))" + assert e.to_sql() == "((age > 18) AND (`status` = 'active'))" def test_or_operator(self): e = (col("a") == lit(1)) | (col("b") == lit(2)) @@ -166,7 +166,7 @@ class TestExprOperators: def test_coerce_plain_str(self): e = col("name") == "alice" assert isinstance(e, Expr) - assert e.to_sql() == "(name = 'alice')" + assert e.to_sql() == "(`name` = 'alice')" def test_reflexive_comparisons(self): # 10 < col("age") swaps to col("age") > 10 @@ -198,85 +198,85 @@ class TestExprBytesLiteral: def test_bytes_equality_expr_sql(self): e = col("data") == lit(b"\xca\xfe") - assert e.to_sql() == "(data = X'CAFE')" + assert e.to_sql() == "(`data` = X'CAFE')" def test_bytes_ne_expr_sql(self): e = col("data") != lit(b"\xff") - assert e.to_sql() == "(data <> X'FF')" + assert e.to_sql() == "(`data` <> X'FF')" def test_bytes_compound_expr_sql(self): e = (col("data") == lit(b"\x01")) & (col("id") > lit(5)) - assert e.to_sql() == "((data = X'01') AND (id > 5))" + assert e.to_sql() == "((`data` = X'01') AND (id > 5))" def test_bytes_in_function_call(self): # Regression test: binary literals inside scalar function calls # used to fail because DataFusion's unparser does not support Binary # scalars. Now handled via a placeholder-substitution rewrite. e = func("contains", col("data"), lit(b"\xff")) - assert e.to_sql() == "contains(data, X'FF')" + assert e.to_sql() == "contains(`data`, X'FF')" def test_bytes_in_not(self): e = ~(col("data") == lit(b"\xff")) - assert e.to_sql() == "NOT (data = X'FF')" + assert e.to_sql() == "NOT (`data` = X'FF')" class TestExprStringMethods: def test_lower(self): e = col("name").lower() assert isinstance(e, Expr) - assert e.to_sql() == "lower(name)" + assert e.to_sql() == "lower(`name`)" def test_upper(self): e = col("name").upper() assert isinstance(e, Expr) - assert e.to_sql() == "upper(name)" + assert e.to_sql() == "upper(`name`)" def test_contains(self): e = col("text").contains(lit("hello")) assert isinstance(e, Expr) - assert e.to_sql() == "contains(text, 'hello')" + assert e.to_sql() == "contains(`text`, 'hello')" def test_contains_with_str_coerce(self): e = col("text").contains("hello") assert isinstance(e, Expr) - assert e.to_sql() == "contains(text, 'hello')" + assert e.to_sql() == "contains(`text`, 'hello')" def test_chained_lower_eq(self): e = col("name").lower() == lit("alice") assert isinstance(e, Expr) - assert e.to_sql() == "(lower(name) = 'alice')" + assert e.to_sql() == "(lower(`name`) = 'alice')" class TestExprCast: def test_cast_string(self): e = col("id").cast("string") assert isinstance(e, Expr) - assert e.to_sql() == "CAST(id AS VARCHAR)" + assert e.to_sql() == "arrow_cast(id, 'Utf8')" def test_cast_int32(self): e = col("score").cast("int32") assert isinstance(e, Expr) - assert e.to_sql() == "CAST(score AS INTEGER)" + assert e.to_sql() == "arrow_cast(score, 'Int32')" def test_cast_float64(self): e = col("val").cast("float64") assert isinstance(e, Expr) - assert e.to_sql() == "CAST(val AS DOUBLE)" + assert e.to_sql() == "arrow_cast(val, 'Float64')" def test_cast_pyarrow_type(self): e = col("score").cast(pa.int32()) assert isinstance(e, Expr) - assert e.to_sql() == "CAST(score AS INTEGER)" + assert e.to_sql() == "arrow_cast(score, 'Int32')" def test_cast_pyarrow_float64(self): e = col("val").cast(pa.float64()) assert isinstance(e, Expr) - assert e.to_sql() == "CAST(val AS DOUBLE)" + assert e.to_sql() == "arrow_cast(val, 'Float64')" def test_cast_pyarrow_string(self): e = col("id").cast(pa.string()) assert isinstance(e, Expr) - assert e.to_sql() == "CAST(id AS VARCHAR)" + assert e.to_sql() == "arrow_cast(id, 'Utf8')" def test_cast_pyarrow_and_string_equivalent(self): # pa.int32() and "int32" should produce equivalent SQL @@ -597,14 +597,14 @@ class TestExprIsin: def test_isin_strs(self): assert ( col("status").isin(["active", "pending"]).to_sql() - == "status IN ('active', 'pending')" + == "`status` IN ('active', 'pending')" ) def test_isin_coerces_and_mixes(self): assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)" def test_isin_empty(self): - assert col("id").isin([]).to_sql() == "id IN ()" + assert col("id").isin([]).to_sql() == "false" def test_isin_filter(self, simple_table): result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow() diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 56e0eacfd..0be9e139d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -11,6 +11,7 @@ import warnings import weakref from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, timedelta +from decimal import Decimal from time import sleep from typing import List from unittest.mock import patch @@ -336,6 +337,21 @@ async def test_update_async(mem_db_async: AsyncConnection): assert await table.count_rows("id == 10") == 1 +@pytest.mark.asyncio +async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection): + values = ["5", "4.66e-84", "it's"] + table = await mem_db_async.create_table( + "update_expr_literals", + data=[{"field": value, "result": "original"} for value in values], + ) + + for value in values: + update_res = await table.update({"result": value}, where=col("field") == value) + assert update_res.rows_updated == 1 + + assert (await table.to_arrow())["result"].to_pylist() == values + + def test_create_table(mem_db: DBConnection): schema = pa.schema( { @@ -2343,6 +2359,148 @@ def test_update(mem_db: DBConnection): assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]])) +def test_update_expr_filter_literals(mem_db: DBConnection): + values = ["5", "4.66e-84", "it's"] + table = mem_db.create_table( + "update_expr_literals", + data=[{"field": value, "result": "original"} for value in values], + ) + + for value in values: + update_res = table.update(where=col("field") == value, values={"result": value}) + assert update_res.rows_updated == 1 + + assert table.to_arrow()["result"].to_pylist() == values + + +def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection): + low = Decimal("1.234567890123456789") + high = Decimal("1.234567890123456790") + decimal_schema = pa.schema( + [("val", pa.decimal128(19, 18)), ("result", pa.string())] + ) + decimal_table = mem_db.create_table( + "update_expr_decimal", + pa.table( + {"val": [low, high], "result": ["old", "old"]}, + schema=decimal_schema, + ), + ) + predicate = col("val") < lit(high) + assert decimal_table.search().where(predicate).to_arrow().num_rows == 1 + result = decimal_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + keyword_table = mem_db.create_table( + "update_expr_keyword", [{"null": 1, "result": "old"}] + ) + predicate = col("null") == 1 + assert keyword_table.search().where(predicate).to_arrow().num_rows == 1 + result = keyword_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + empty_in_table = mem_db.create_table( + "update_expr_empty_in", [{"id": 1, "result": "old"}] + ) + predicate = col("id").isin([]) + assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0 + result = empty_in_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 0 + + marker = "__lancedb_binary_placeholder_0__" + binary_schema = pa.schema( + [("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())] + ) + binary_table = mem_db.create_table( + "update_expr_binary", + pa.table( + { + "payload": [b"\x01", b"\x02"], + "text": ["other", marker], + "result": ["old", "old"], + }, + schema=binary_schema, + ), + ) + predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker) + assert binary_table.search().where(predicate).to_arrow().num_rows == 2 + result = binary_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 2 + + nonfinite_table = mem_db.create_table( + "update_expr_nonfinite", + [{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}], + ) + predicate = col("x") < float("inf") + assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2 + result = nonfinite_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 2 + + float16_table = mem_db.create_table( + "update_expr_float16", + [{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}], + ) + predicate = col("x").cast(pa.float16()) < 2.0 + assert float16_table.search().where(predicate).to_arrow().num_rows == 1 + result = float16_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + string_cast_table = mem_db.create_table( + "update_expr_string_cast", + [{"x": 1, "result": "old"}, {"x": 2, "result": "old"}], + ) + predicate = col("x").cast("string") == "1" + assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1 + result = string_cast_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + quoted_identifier_schema = pa.schema( + [("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())] + ) + quoted_identifier_table = mem_db.create_table( + "update_expr_quoted_identifier", + pa.table( + {"payload": [b"\x01"], "odd'name": [1], "result": ["old"]}, + schema=quoted_identifier_schema, + ), + ) + predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1) + assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1 + result = quoted_identifier_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + decimal256_schema = pa.schema( + [("val", pa.decimal256(40, 2)), ("result", pa.string())] + ) + decimal256_table = mem_db.create_table( + "update_expr_decimal256", + pa.table( + { + "val": [Decimal("1.00"), Decimal("3.00")], + "result": ["old", "old"], + }, + schema=decimal256_schema, + ), + ) + predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2)) + assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1 + result = decimal256_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 1 + + binary_empty_table = mem_db.create_table( + "update_expr_binary_empty", + pa.table( + {"payload": [b"\x01", b"\x02"], "result": ["old", "old"]}, + schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]), + ), + ) + predicate = (col("payload") == lit(b"\x01")).isin([]) + assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0 + assert predicate.to_sql() == "false" + result = binary_empty_table.update(where=predicate, values={"result": "new"}) + assert result.rows_updated == 0 + + def test_update_with_arrow_scalar(mem_db: DBConnection): schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)}) table = mem_db.create_table("my_table", schema=schema) diff --git a/python/src/expr.rs b/python/src/expr.rs index eae1d96ec..79b448fdf 100644 --- a/python/src/expr.rs +++ b/python/src/expr.rs @@ -130,6 +130,14 @@ impl PyExpr { // ── utilities ──────────────────────────────────────────────────────────── + /// Return the referenced column name for a bare column expression. + fn column_name(&self) -> Option { + match &self.0 { + DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()), + _ => None, + } + } + /// Render the expression as a SQL string (useful for debugging). fn to_sql(&self) -> PyResult { lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string())) diff --git a/python/src/query.rs b/python/src/query.rs index 014e79e2d..38153729f 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -325,6 +326,7 @@ pub struct PyQueryRequest { pub filter: Option, pub full_text_search: Option>, pub select: PySelect, + pub select_source_columns: Option>, pub fast_search: Option, pub with_row_id: Option, pub use_lsm: Option, @@ -355,6 +357,7 @@ impl From for PyQueryRequest { full_text_search: query_request .full_text_search .map(|fts| PyLanceDB(fts.query)), + select_source_columns: PySelect::source_columns(&query_request.select), select: PySelect(query_request.select), fast_search: Some(query_request.fast_search), with_row_id: Some(query_request.with_row_id), @@ -380,6 +383,7 @@ impl From for PyQueryRequest { offset: vector_query.base.offset, filter: vector_query.base.filter.map(PyQueryFilter), full_text_search: None, + select_source_columns: PySelect::source_columns(&vector_query.base.select), select: PySelect(vector_query.base.select), fast_search: Some(vector_query.base.fast_search), with_row_id: Some(vector_query.base.with_row_id), @@ -412,6 +416,25 @@ impl From for PyQueryRequest { #[derive(Clone)] pub struct PySelect(Select); +impl PySelect { + fn source_columns(select: &Select) -> Option> { + match select { + Select::Expr(pairs) => Some( + pairs + .iter() + .filter_map(|(output, expr)| match expr { + lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => { + Some((output.clone(), column.name.clone())) + } + _ => None, + }) + .collect(), + ), + _ => None, + } + } +} + impl<'py> IntoPyObject<'py> for PySelect { type Target = PyAny; type Output = Bound<'py, Self::Target>; diff --git a/rust/lancedb/src/expr.rs b/rust/lancedb/src/expr.rs index da69914e3..1625d9632 100644 --- a/rust/lancedb/src/expr.rs +++ b/rust/lancedb/src/expr.rs @@ -157,7 +157,7 @@ mod tests { use datafusion_common::ScalarValue; let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe])))); let sql = expr_to_sql_string(&expr).unwrap(); - assert_eq!(sql, "(data = X'CAFE')"); + assert_eq!(sql, "(`data` = X'CAFE')"); } #[test] @@ -167,7 +167,7 @@ mod tests { let int_expr = col("id").gt(lit(5i64)); let combined = bin_expr.and(int_expr); let sql = expr_to_sql_string(&combined).unwrap(); - assert_eq!(sql, "((data = X'01') AND (id > 5))"); + assert_eq!(sql, "((`data` = X'01') AND (id > 5))"); } #[test] @@ -185,7 +185,7 @@ mod tests { // serialized correctly (regression test for placeholder rewrite path). let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff])))); let sql = expr_to_sql_string(&expr).unwrap(); - assert_eq!(sql, "contains(data, X'FF')"); + assert_eq!(sql, "contains(`data`, X'FF')"); } #[test] @@ -196,7 +196,7 @@ mod tests { .eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd])))) .not(); let sql = expr_to_sql_string(&expr).unwrap(); - assert_eq!(sql, "NOT (data = X'ABCD')"); + assert_eq!(sql, "NOT (`data` = X'ABCD')"); } #[test] @@ -206,6 +206,122 @@ mod tests { assert!(sql.contains("IN"), "expected IN in: {}", sql); } + #[test] + fn test_empty_is_in() { + let expr = is_in(col("id"), vec![]); + assert_eq!(expr_to_sql_string(&expr).unwrap(), "false"); + } + + #[test] + fn test_empty_is_in_discards_binary_children() { + use datafusion_common::ScalarValue; + + let expr = is_in( + col("payload").eq(lit(ScalarValue::Binary(Some(vec![0x01])))), + vec![], + ); + assert_eq!(expr_to_sql_string(&expr).unwrap(), "false"); + } + + #[test] + fn test_keyword_identifier() { + let expr = col("null").eq(lit(1i64)); + assert_eq!(expr_to_sql_string(&expr).unwrap(), "(`null` = 1)"); + } + + #[test] + fn test_decimal_literal_preserves_type() { + use datafusion_common::ScalarValue; + + let expr = col("val").lt(lit(ScalarValue::Decimal128( + Some(1_234_567_890_123_456_790), + 19, + 18, + ))); + let sql = expr_to_sql_string(&expr).unwrap(); + assert_eq!( + sql, + "(val < arrow_cast('1.234567890123456790', 'Decimal128(19, 18)'))" + ); + } + + #[test] + fn test_non_finite_float_literal_preserves_type() { + let expr = col("x").lt(lit(f64::INFINITY)); + assert_eq!( + expr_to_sql_string(&expr).unwrap(), + "(x < arrow_cast('inf', 'Float64'))" + ); + } + + #[test] + fn test_cast_uses_arrow_type_name() { + let string = expr_cast(col("x"), DataType::Utf8); + assert_eq!( + expr_to_sql_string(&string).unwrap(), + "arrow_cast(x, 'Utf8')" + ); + + let int32 = expr_cast(col("x"), DataType::Int32); + assert_eq!( + expr_to_sql_string(&int32).unwrap(), + "arrow_cast(x, 'Int32')" + ); + + let expr = expr_cast(col("x"), DataType::Float16).lt(lit(2.0)); + assert_eq!( + expr_to_sql_string(&expr).unwrap(), + "(arrow_cast(x, 'Float16') < 2.0)" + ); + + let decimal = expr_cast(lit("2.00"), DataType::Decimal256(40, 2)); + assert_eq!( + expr_to_sql_string(&decimal).unwrap(), + "arrow_cast('2.00', 'Decimal256(40, 2)')" + ); + } + + #[test] + fn test_binary_placeholder_does_not_rewrite_user_string() { + use datafusion_common::ScalarValue; + + let marker = "__lancedb_binary_placeholder_0__"; + let expr = col("payload") + .eq(lit(ScalarValue::Binary(Some(vec![0x01])))) + .or(col("text").eq(lit(marker))); + assert_eq!( + expr_to_sql_string(&expr).unwrap(), + "((payload = X'01') OR (`text` = '__lancedb_binary_placeholder_0__'))" + ); + } + + #[test] + fn test_binary_binding_skips_quoted_identifiers() { + use datafusion_common::ScalarValue; + + let expr = col("payload") + .eq(lit(ScalarValue::Binary(Some(vec![0x01])))) + .and(col("odd'name").eq(lit(1i64))) + .and(col("odd`'name").eq(lit(2i64))); + assert_eq!( + expr_to_sql_string(&expr).unwrap(), + "(((payload = X'01') AND (`odd'name` = 1)) AND (`odd``'name` = 2))" + ); + } + + #[test] + fn test_binary_placeholder_collision_search_is_linear() { + use datafusion_common::ScalarValue; + + let collision_shaped = format!("__lancedb_binary_placeholder_0__{}", "_".repeat(64_000)); + let expr = col("payload") + .eq(lit(ScalarValue::Binary(Some(vec![0x01])))) + .and(col("text").eq(lit(collision_shaped.clone()))); + let sql = expr_to_sql_string(&expr).unwrap(); + assert!(sql.contains("X'01'")); + assert!(sql.contains(&format!("'{collision_shaped}'"))); + } + #[test] fn test_multiple_binary_literals() { use datafusion_common::ScalarValue; diff --git a/rust/lancedb/src/expr/sql.rs b/rust/lancedb/src/expr/sql.rs index 24a676485..2a1ca201d 100644 --- a/rust/lancedb/src/expr/sql.rs +++ b/rust/lancedb/src/expr/sql.rs @@ -1,13 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::any::TypeId; +use std::{ + any::TypeId, + collections::{HashMap, HashSet}, +}; +use arrow_array::types::{ + Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, +}; +use arrow_schema::DataType; use datafusion_common::ScalarValue; use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion_expr::Expr; +use datafusion_functions::core::expr_fn::{ + arrow_cast as datafusion_arrow_cast, arrow_try_cast as datafusion_arrow_try_cast, +}; use datafusion_sql::sqlparser::{ dialect::{Dialect as SqlParserDialect, GenericDialect}, + keywords::ALL_KEYWORDS, tokenizer::{Token, Tokenizer}, }; use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect}; @@ -27,11 +38,13 @@ struct LanceSqlDialect; impl UnparserDialect for LanceSqlDialect { fn identifier_quote_style(&self, identifier: &str) -> Option { - let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase()) - || !identifier - .chars() - .enumerate() - .all(|(i, c)| c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())); + let identifier_upper = identifier.to_ascii_uppercase(); + let needs_quote = + (identifier_upper != "ID" && ALL_KEYWORDS.contains(&identifier_upper.as_str())) + || identifier.chars().any(|c| c.is_ascii_uppercase()) + || !identifier.chars().enumerate().all(|(i, c)| { + c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()) + }); if needs_quote { Some('`') } else { None } } } @@ -100,24 +113,128 @@ fn bytes_to_hex_sql(bytes: &[u8]) -> String { format!("X'{hex}'") } -/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal -/// anywhere in its subtree. DataFusion's SQL unparser cannot serialize those -/// variants, so we route such expressions through a placeholder-substitution -/// path that emits SQL `X'...'` byte-string literals. -fn has_binary_literal(expr: &Expr) -> bool { - let mut found = false; +fn string_literals(expr: &Expr) -> HashSet { + let mut literals = HashSet::new(); let _ = expr.apply(&mut |e: &Expr| { - if matches!( - e, - Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _) - ) { - found = true; - Ok(TreeNodeRecursion::Stop) - } else { - Ok(TreeNodeRecursion::Continue) + if let Expr::Literal( + ScalarValue::Utf8(Some(value)) + | ScalarValue::LargeUtf8(Some(value)) + | ScalarValue::Utf8View(Some(value)), + _, + ) = e + { + literals.insert(value.clone()); } + Ok(TreeNodeRecursion::Continue) }); - found + literals +} + +fn typed_string_literal(value: String, data_type: DataType) -> Expr { + datafusion_arrow_cast( + Expr::Literal(ScalarValue::Utf8(Some(value)), None), + Expr::Literal(ScalarValue::Utf8(Some(data_type.to_string())), None), + ) +} + +fn next_binary_placeholder(user_strings: &HashSet, next_id: &mut usize) -> String { + loop { + let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id); + *next_id += 1; + if !user_strings.contains(&placeholder) { + return placeholder; + } + } +} + +fn bind_binary_literals( + sql: &str, + mut bindings: HashMap>, +) -> crate::Result { + let bytes = sql.as_bytes(); + let mut output = Vec::with_capacity(bytes.len()); + let mut index = 0; + + // Walk SQL string tokens once. Placeholders are plain, unescaped string + // literals, so this remains linear even when user strings are large or + // deliberately resemble the placeholder prefix. + while index < bytes.len() { + if bytes[index] == b'`' { + let identifier_start = index; + index += 1; + let mut identifier_end = None; + while index < bytes.len() { + if bytes[index] == b'`' { + if index + 1 < bytes.len() && bytes[index + 1] == b'`' { + index += 2; + } else { + index += 1; + identifier_end = Some(index); + break; + } + } else { + index += 1; + } + } + + let Some(identifier_end) = identifier_end else { + return Err(crate::Error::InvalidInput { + message: "unterminated identifier while binding binary literal".to_string(), + }); + }; + output.extend_from_slice(&bytes[identifier_start..identifier_end]); + continue; + } + + if bytes[index] != b'\'' { + output.push(bytes[index]); + index += 1; + continue; + } + + let literal_start = index; + index += 1; + let content_start = index; + let mut escaped = false; + let mut content_end = None; + while index < bytes.len() { + if bytes[index] == b'\'' { + if index + 1 < bytes.len() && bytes[index + 1] == b'\'' { + escaped = true; + index += 2; + } else { + content_end = Some(index); + index += 1; + break; + } + } else { + index += 1; + } + } + + let Some(content_end) = content_end else { + return Err(crate::Error::InvalidInput { + message: "unterminated string while binding binary literal".to_string(), + }); + }; + + let placeholder = &sql[content_start..content_end]; + if !escaped && let Some(value) = bindings.remove(placeholder) { + output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes()); + } else { + output.extend_from_slice(&bytes[literal_start..index]); + } + } + + if !bindings.is_empty() { + return Err(crate::Error::InvalidInput { + message: "failed to bind binary literal while serializing expression".to_string(), + }); + } + + String::from_utf8(output).map_err(|e| crate::Error::InvalidInput { + message: format!("failed to bind binary literal: {e}"), + }) } fn run_unparser(expr: &Expr) -> crate::Result { @@ -130,25 +247,37 @@ fn run_unparser(expr: &Expr) -> crate::Result { } pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { - // Fast path: no binary literals — DataFusion's unparser handles everything. - if !has_binary_literal(expr) { - return run_unparser(expr); - } - - // Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary` - // scalars, so we rewrite each one to a unique string-literal placeholder, - // let the unparser do the rest of the work, then substitute the SQL - // `X'...'` byte-string literal back in. This keeps the operator/function - // serialization logic centralized in DataFusion and works for every - // expression node type the unparser supports. - let mut bindings: Vec> = Vec::new(); + // DataFusion's unparser needs a few adaptations before its SQL can be + // reparsed by Lance without changing the typed expression's semantics: + // + // * decimal literals need an explicit cast to preserve precision and scale; + // * casts need exact Arrow type names rather than SQL type aliases; + // * an empty IN list is valid in DataFusion but invalid SQL; + // * binary literals are unsupported by the unparser and need placeholders. + // Eliminate empty membership expressions before visiting their children. + // Otherwise a discarded binary child could leave behind a stale binding. let rewritten = expr .clone() + .transform(|e: Expr| match e { + Expr::InList(in_list) if in_list.list.is_empty() => Ok(Transformed::yes( + Expr::Literal(ScalarValue::Boolean(Some(in_list.negated)), None), + )), + other => Ok(Transformed::no(other)), + }) + .map_err(|e| crate::Error::InvalidInput { + message: format!("failed to rewrite expression: {e}"), + })? + .data; + + let user_strings = string_literals(&rewritten); + let mut next_placeholder_id = 0; + let mut binary_bindings = HashMap::new(); + let rewritten = rewritten .transform(|e: Expr| match e { Expr::Literal(ScalarValue::Binary(Some(bytes)), m) | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => { - let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len()); - bindings.push(bytes); + let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id); + binary_bindings.insert(placeholder.clone(), bytes); Ok(Transformed::yes(Expr::Literal( ScalarValue::Utf8(Some(placeholder)), m, @@ -158,6 +287,57 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { | Expr::Literal(ScalarValue::LargeBinary(None), m) => { Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m))) } + Expr::Literal(ScalarValue::Decimal32(Some(value), precision, scale), _m) => { + let value = Decimal32Type::format_decimal(value, precision, scale); + Ok(Transformed::yes(typed_string_literal( + value, + DataType::Decimal32(precision, scale), + ))) + } + Expr::Literal(ScalarValue::Decimal64(Some(value), precision, scale), _m) => { + let value = Decimal64Type::format_decimal(value, precision, scale); + Ok(Transformed::yes(typed_string_literal( + value, + DataType::Decimal64(precision, scale), + ))) + } + Expr::Literal(ScalarValue::Decimal128(Some(value), precision, scale), _m) => { + let value = Decimal128Type::format_decimal(value, precision, scale); + Ok(Transformed::yes(typed_string_literal( + value, + DataType::Decimal128(precision, scale), + ))) + } + Expr::Literal(ScalarValue::Decimal256(Some(value), precision, scale), _m) => { + let value = Decimal256Type::format_decimal(value, precision, scale); + Ok(Transformed::yes(typed_string_literal( + value, + DataType::Decimal256(precision, scale), + ))) + } + Expr::Literal(ScalarValue::Float16(Some(value)), _m) if !value.is_finite() => Ok( + Transformed::yes(typed_string_literal(value.to_string(), DataType::Float16)), + ), + Expr::Literal(ScalarValue::Float32(Some(value)), _m) if !value.is_finite() => Ok( + Transformed::yes(typed_string_literal(value.to_string(), DataType::Float32)), + ), + Expr::Literal(ScalarValue::Float64(Some(value)), _m) if !value.is_finite() => Ok( + Transformed::yes(typed_string_literal(value.to_string(), DataType::Float64)), + ), + Expr::Cast(cast) => Ok(Transformed::yes(datafusion_arrow_cast( + *cast.expr, + Expr::Literal( + ScalarValue::Utf8(Some(cast.field.data_type().to_string())), + None, + ), + ))), + Expr::TryCast(cast) => Ok(Transformed::yes(datafusion_arrow_try_cast( + *cast.expr, + Expr::Literal( + ScalarValue::Utf8(Some(cast.field.data_type().to_string())), + None, + ), + ))), other => Ok(Transformed::no(other)), }) .map_err(|e| crate::Error::InvalidInput { @@ -165,14 +345,12 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { })? .data; - let mut sql = run_unparser(&rewritten)?; - for (i, bytes) in bindings.iter().enumerate() { - // The unparser quotes string literals with single quotes, so the - // placeholder appears as `'__lancedb_binary_placeholder___'`. - let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i); - sql = sql.replace("ed, &bytes_to_hex_sql(bytes)); + let sql = run_unparser(&rewritten)?; + if binary_bindings.is_empty() { + Ok(sql) + } else { + bind_binary_literals(&sql, binary_bindings) } - Ok(sql) } #[cfg(test)]