rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-03 20:18:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: inherit blob metadata for computed projections

Browse Source
This commit is contained in:
Xuanwo
2026-08-28 16:44:17 +08:00
parent 2702cdb219
commit 6c8e5e9d98
11 changed files with 158 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/_lancedb.pyi
+1 -2
View File
@@ -352,8 +352,7 @@ class Table:
async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
async def add_computed_columns(
self,
columns: list[tuple[str | pa.Field, str]],
self, columns: list[tuple[str, str]]
) -> AddColumnsResult: ...
async def add_function_columns(
self, application_json: str, output_name: Optional[str]
python/python/lancedb/remote/table.py
+1 -2
View File
@@ -13,7 +13,6 @@ from typing import (
Iterable,
List,
Optional,
Sequence,
Union,
Literal,
overload,
@@ -984,7 +983,7 @@ class RemoteTable(Table):
| FunctionApplication
| None = None,
*,
computed: Dict[str, str] | Sequence[tuple[str | pa.Field, str]] | None = None,
computed: Dict[str, str] | None = None,
) -> AddColumnsResult:
return LOOP.run(self._table.add_columns(transforms, computed=computed))
python/python/lancedb/table.py
+14 -42
View File
@@ -913,26 +913,6 @@ def _normalize_progress(progress):
return progress, False
def _normalize_computed_columns(
computed: Dict[str, str] | Sequence[tuple[str | pa.Field, str]],
) -> list[tuple[str | pa.Field, str]]:
columns: list[tuple[str | pa.Field, str]] = []
declarations = computed.items() if isinstance(computed, dict) else computed
for declaration in declarations:
if not isinstance(declaration, (tuple, list)) or len(declaration) != 2:
raise TypeError(
"computed sequences must contain (column name or pyarrow Field, "
"SQL expression) pairs"
)
field, expression = declaration
if not isinstance(field, (str, pa.Field)):
raise TypeError("computed targets must be column names or pyarrow Fields")
if not isinstance(expression, str):
raise TypeError("computed values must be SQL expression strings")
columns.append((field, expression))
return columns
class Table(ABC):
"""
A Table is a collection of Records in a LanceDB Database.
@@ -2162,7 +2142,7 @@ class Table(ABC):
| pa.Schema
| None = None,
*,
computed: Dict[str, str] | Sequence[tuple[str | pa.Field, str]] | None = None,
computed: Dict[str, str] | None = None,
):
"""
Add new columns with defined values.
@@ -2184,15 +2164,12 @@ class Table(ABC):
atomic binding; aliases come from ``rename(columns=...)``.
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str] or Sequence[Tuple[str | pa.Field, str]], optional
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. An ordered sequence may instead
use a pyarrow Field as a target, supplying its name, type,
nullability, and extension metadata; use
``(lancedb.blob("name"), expression)`` for a Blob v2 output.
Explicit fields must be nullable and their expression result type
must be compatible. Mapping or sequence order is declaration and
dependency order.
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -4324,7 +4301,7 @@ class LanceTable(Table):
| pa.Schema
| None = None,
*,
computed: Dict[str, str] | Sequence[tuple[str | pa.Field, str]] | None = None,
computed: Dict[str, str] | None = None,
) -> AddColumnsResult:
return LOOP.run(self._table.add_columns(transforms, computed=computed))
@@ -6272,7 +6249,7 @@ class AsyncTable:
| pa.Schema
| None = None,
*,
computed: dict[str, str] | Sequence[tuple[str | pa.Field, str]] | None = None,
computed: dict[str, str] | None = None,
) -> AddColumnsResult:
"""
Add new columns with defined values.
@@ -6292,15 +6269,12 @@ class AsyncTable:
atomic binding; aliases come from ``rename(columns=...)``.
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str] or Sequence[Tuple[str | pa.Field, str]], optional
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. An ordered sequence may instead
use a pyarrow Field as a target, supplying its name, type,
nullability, and extension metadata; use
``(lancedb.blob("name"), expression)`` for a Blob v2 output.
Explicit fields must be nullable and their expression result type
must be compatible. Mapping or sequence order is declaration and
dependency order.
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -6358,9 +6332,7 @@ class AsyncTable:
raise ValueError(
"add_columns cannot take both transforms and computed columns"
)
return await self._inner.add_computed_columns(
_normalize_computed_columns(computed)
)
return await self._inner.add_computed_columns(list(computed.items()))
if transforms is None:
raise ValueError("add_columns requires transforms or computed columns")
if isinstance(transforms, pa.Schema):
python/python/tests/test_table.py
+5 -37
View File
@@ -4087,7 +4087,7 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
def test_computed_column_blob_input_and_explicit_output(tmp_path):
def test_computed_column_blob_projection_inherits_semantics(tmp_path):
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
db = lancedb.connect(tmp_path)
table = db.create_table("computed_column_blob", schema=schema)
@@ -4099,49 +4099,17 @@ def test_computed_column_blob_input_and_explicit_output(tmp_path):
]
)
table.add_columns(
computed=[
(lancedb.blob("image_copy"), "image"),
("payload_copy", "image_copy"),
]
)
table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
assert table.refresh_column("image_copy").rows_filled == 2
assert table.refresh_column("payload_copy").rows_filled == 2
values = table.to_arrow()["payload_copy"].combine_chunks().to_pylist()
assert values == [b"hello", b"", None]
assert table.blob_columns() == ["image", "image_copy"]
assert table.refresh_column("second_copy").rows_filled == 2
assert table.blob_columns() == ["image", "image_copy", "second_copy"]
hits = table.search().with_row_id(True).limit(10).to_arrow()
rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
copied = table.fetch_blobs("image_copy", [row_id for _, row_id in rows])
copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
assert copied.to_pylist() == [b"hello", b"", None]
def test_blob_output_declaration_rejects_eager_transforms(tmp_path):
db = lancedb.connect(tmp_path)
table = db.create_table("computed_column_blob_mixed", [{"x": 1}])
with pytest.raises(ValueError):
table.add_columns(
{"a": "x + 1"},
computed=[(lancedb.blob("b"), "x")],
)
def test_computed_column_validates_declaration_mapping(tmp_path):
db = lancedb.connect(tmp_path)
table = db.create_table("computed_column_mapping", [{"x": 1}])
with pytest.raises(
TypeError, match="targets must be column names or pyarrow Fields"
):
table.add_columns(computed={42: "x"}) # type: ignore[dict-item]
with pytest.raises(TypeError, match="values must be SQL expression strings"):
table.add_columns(computed={"copy": 42}) # type: ignore[dict-item]
with pytest.raises(TypeError, match="sequences must contain"):
table.add_columns(computed=[("copy", "x", "extra")]) # type: ignore[list-item]
@pytest.mark.asyncio
async def test_computed_column_async(tmp_path):
db = await lancedb.connect_async(tmp_path)
python/src/table.rs
+6 -21
View File
@@ -13,16 +13,15 @@ use crate::{
};
use arrow::{
array::{Array, LargeBinaryArray},
datatypes::{DataType, Field, Schema},
datatypes::{DataType, Schema},
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
};
use lancedb::blob::{BlobFile, BlobRangeRequest};
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::table::{
AddDataMode, ColumnAlteration, ComputedColumnDeclaration, Duration, FieldMetadataUpdate,
FtsToken as LanceDbFtsToken, NewColumnTransform, OptimizeAction, OptimizeOptions, Ref,
Table as LanceDbTable,
AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, FtsToken as LanceDbFtsToken,
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
};
use lancedb::tokenize as lancedb_tokenize;
use pyo3::{
@@ -101,12 +100,6 @@ enum PredicateArg {
Sql(String),
}
#[derive(FromPyObject)]
pub enum ComputedColumnFieldArg {
Name(String),
Field(PyArrowType<Field>),
}
/// Statistics about a compaction operation.
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
@@ -1584,21 +1577,13 @@ impl Table {
pub fn add_computed_columns(
self_: PyRef<'_, Self>,
columns: Vec<(ComputedColumnFieldArg, String)>,
columns: Vec<(String, String)>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.add_columns();
for (field, expression) in columns {
let declaration = match field {
ComputedColumnFieldArg::Name(name) => {
ComputedColumnDeclaration::inferred(name, expression)
}
ComputedColumnFieldArg::Field(PyArrowType(field)) => {
ComputedColumnDeclaration::with_field(field, expression)
}
};
builder = builder.computed_column(declaration);
for (name, expression) in columns {
builder = builder.computed(name, expression);
}
let result = builder.execute().await.infer_error()?;
Ok(AddColumnsResult::from(result))