mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-03 20:18:54 +00:00
feat: support blob computed column refresh (#4081)
Computed-column planning currently sees Blob v2 storage descriptors, so
expressions cannot consume payload bytes or preserve Blob semantics in
their outputs.
A computed declaration now derives its output field from its expression.
A direct projection of a Blob v2 field inherits the source field's Blob
metadata; other expressions retain their ordinary Arrow-inferred type.
Declarations remain ordered, so the same rule applies across chained
projections.
Refresh materializes referenced Blob inputs as `LargeBinary` payload
bytes and publishes inherited Blob outputs through Lance's Blob
conversion path. Remote requests remain within the shared namespace
contract as `{name, computed}`; the server planner is being updated in
tandem to implement the same Blob-aware planning semantics, and remote
enablement must be aligned with that server rollout.
The existing null-as-unfilled contract remains unchanged. Row-level
freshness and cell flags remain follow-up work.
This commit is contained in:
@@ -67,7 +67,15 @@ from ..query import (
|
||||
LanceTakeQueryBuilder,
|
||||
LanceVectorQueryBuilder,
|
||||
)
|
||||
from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
|
||||
from ..table import (
|
||||
AsyncTable,
|
||||
BlobMode,
|
||||
Branches,
|
||||
IndexStatistics,
|
||||
Query,
|
||||
Table,
|
||||
Tags,
|
||||
)
|
||||
from ..types import BaseTokenizerType
|
||||
|
||||
|
||||
|
||||
@@ -2165,9 +2165,11 @@ class Table(ABC):
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
A map of column name to a SQL expression defining the column. The
|
||||
column's type and inputs are derived from the expression, so no
|
||||
data type is supplied.
|
||||
A mapping from output column names to SQL expressions derives each
|
||||
output field from its expression. A direct projection of a Blob v2
|
||||
field inherits Blob v2 semantics; other expressions derive their
|
||||
ordinary Arrow type. Mapping order is declaration and dependency
|
||||
order.
|
||||
|
||||
Unlike ``transforms``, the expression is stored rather than
|
||||
evaluated now: the column is committed with no values, and rows get
|
||||
@@ -6268,8 +6270,11 @@ class AsyncTable:
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
A map of column name to a SQL expression defining the column. The
|
||||
column's type and inputs are derived from the expression.
|
||||
A mapping from output column names to SQL expressions derives each
|
||||
output field from its expression. A direct projection of a Blob v2
|
||||
field inherits Blob v2 semantics; other expressions derive their
|
||||
ordinary Arrow type. Mapping order is declaration and dependency
|
||||
order.
|
||||
|
||||
Unlike ``transforms``, the expression is stored rather than
|
||||
evaluated now: the column is committed with no values, and rows get
|
||||
|
||||
@@ -4087,6 +4087,29 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
|
||||
table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
|
||||
|
||||
|
||||
def test_computed_column_blob_projection_inherits_semantics(tmp_path):
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("computed_column_blob", schema=schema)
|
||||
table.add(
|
||||
[
|
||||
{"id": 1, "image": b"hello"},
|
||||
{"id": 2, "image": b""},
|
||||
{"id": 3, "image": None},
|
||||
]
|
||||
)
|
||||
|
||||
table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
|
||||
assert table.refresh_column("image_copy").rows_filled == 2
|
||||
assert table.refresh_column("second_copy").rows_filled == 2
|
||||
assert table.blob_columns() == ["image", "image_copy", "second_copy"]
|
||||
|
||||
hits = table.search().with_row_id(True).limit(10).to_arrow()
|
||||
rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
|
||||
copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
|
||||
assert copied.to_pylist() == [b"hello", b"", None]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_computed_column_async(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
|
||||
Reference in New Issue
Block a user