feat: refresh computed columns (#3938)

table.refresh_column("doubled") fills the rows of a declared column that
hold no value, in two passes per fragment: the first scans only the
unfilled
live rows to count exact gains and decide staging, the second streams
the
fragment's physical rows into a standalone column file published in one
DataReplacement -- committed under the dataset's own session -- so peak
memory is bounded by a scan batch. A row that holds a value keeps it;
deleted and already-filled rows never reach the expression, so a poison
value in them cannot fail the refresh. Refresh refuses under an LSM
write
spec, including the mem-wal catch-up flag that outlives unset and marks
retained SSTable rows.

---

<sub>Stack created with <a
href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a
href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

python/python/lancedb/_lancedb.pyi

@@ -341,6 +341,7 @@ class Table:
     async def add_computed_columns(
         self, columns: list[tuple[str, str]]
     ) -> AddColumnsResult: ...
+    async def refresh_column(self, column: str) -> RefreshColumnResult: ...
     async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
     async def alter_columns(
         self, columns: list[dict[str, Any]]
@@ -686,6 +687,10 @@ class LsmWriteSpec:
 class AddColumnsResult:
     version: int
 
+class RefreshColumnResult:
+    rows_filled: int
+    version: int
+
 class AlterColumnsResult:
     version: int
 

python/python/lancedb/remote/table.py

@@ -970,6 +970,9 @@ class RemoteTable(Table):
             )
         return LOOP.run(self._table.add_columns(transforms))
 
+    def refresh_column(self, column: str):
+        raise NotImplementedError("computed columns are supported only on local tables")
+
     def alter_columns(
         self, *alterations: Iterable[Dict[str, str]]
     ) -> AlterColumnsResult:

python/python/lancedb/table.py

@@ -176,6 +176,7 @@ if TYPE_CHECKING:
         CompactionStats,
         Tag,
         AddColumnsResult,
+        RefreshColumnResult,
         AddResult,
         AlterColumnsResult,
         UpdateFieldMetadataResult,
@@ -1943,9 +1944,10 @@ class Table(ABC):
             data type is supplied.
 
             Unlike ``transforms``, the expression is stored rather than
-            evaluated now: the column is committed with no values, and a
-            later refresh fills the rows. Declaring one therefore costs the
-            same on a large table as on an empty one.
+            evaluated now: the column is committed with no values, and rows get
+            them from [`refresh_column`][lancedb.table.Table.refresh_column].
+            Declaring one therefore costs the same on a large table as on an
+            empty one.
 
             A refresh does not revisit rows it has already filled, so mutating
             an input leaves the value computed at fill time; recomputing means
@@ -1967,8 +1969,37 @@ class Table(ABC):
         >>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}])
         >>> table.add_columns(computed={"doubled": "x * 2"})
         AddColumnsResult(version=2)
-        >>> table.to_arrow()["doubled"].to_pylist()
-        [None, None]
+        >>> table.refresh_column("doubled")
+        RefreshColumnResult(rows_filled=2, version=3)
+        >>> table.to_arrow().sort_by("x").to_pandas()
+           x  doubled
+        0  1        2
+        1  2        4
+        """
+
+    @abstractmethod
+    def refresh_column(self, column: str) -> "RefreshColumnResult":
+        """
+        Fill the rows of a computed column that hold no value yet.
+
+        Declared with ``add_columns(computed=...)``, a column starts empty and
+        gets its values here. Rows appended since the last refresh are filled
+        by the next one; rows already filled are left as they are, so the call
+        is idempotent and does not observe a mutated input.
+
+        Local tables only; LanceDB Cloud and Enterprise raise
+        ``NotImplementedError``.
+
+        Parameters
+        ----------
+        column: str
+            The name of the computed column to fill.
+
+        Returns
+        -------
+        RefreshColumnResult
+            rows_filled: the number of rows given a value.
+            version: the new version number of the table.
         """
 
     @abstractmethod
@@ -3984,6 +4015,11 @@ class LanceTable(Table):
     ) -> AddColumnsResult:
         return LOOP.run(self._table.add_columns(transforms, computed=computed))
 
+    def refresh_column(self, column: str) -> "RefreshColumnResult":
+        """Fill a computed column's unfilled rows. See
+        [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
+        return LOOP.run(self._table.refresh_column(column))
+
     def alter_columns(
         self, *alterations: Iterable[Dict[str, str]]
     ) -> AlterColumnsResult:
@@ -5922,8 +5958,9 @@ class AsyncTable:
             column's type and inputs are derived from the expression.
 
             Unlike ``transforms``, the expression is stored rather than
-            evaluated now: the column is committed with no values, and a
-            later refresh fills the rows.
+            evaluated now: the column is committed with no values, and rows get
+            them from
+            [`refresh_column`][lancedb.table.AsyncTable.refresh_column].
 
             A refresh does not revisit rows it has already filled, so mutating
             an input leaves the value computed at fill time. While a
@@ -5957,6 +5994,30 @@ class AsyncTable:
         else:
             return await self._inner.add_columns(list(transforms.items()))
 
+    async def refresh_column(self, column: str) -> RefreshColumnResult:
+        """
+        Fill the rows of a computed column that hold no value yet.
+
+        Declared with ``add_columns(computed=...)``, a column starts empty and
+        gets its values here. Rows appended since the last refresh are filled
+        by the next one; rows already filled are left as they are, so the call
+        is idempotent and does not observe a mutated input.
+
+        Local tables only; LanceDB Cloud and Enterprise raise
+        ``NotImplementedError``.
+
+        Parameters
+        ----------
+        column: str
+            The name of the computed column to fill.
+
+        Returns
+        -------
+        RefreshColumnResult
+            The number of rows filled and the new version of the table.
+        """
+        return await self._inner.refresh_column(column)
+
     async def alter_columns(
         self, *alterations: Iterable[dict[str, Any]]
     ) -> AlterColumnsResult:

python/python/tests/test_table.py

@@ -3856,16 +3856,20 @@ async def test_async_search_runs_embedding_on_dedicated_executor(
     )
 
 
-def test_computed_column_declares_all_null(tmp_path):
+def test_computed_column_declare_and_refresh(tmp_path):
     db = lancedb.connect(tmp_path)
     table = db.create_table("computed", [{"x": 1}, {"x": 2}])
 
     table.add_columns(computed={"doubled": "x * 2"})
     assert table.to_arrow()["doubled"].to_pylist() == [None, None]
 
-    # The declaration is durable field metadata.
-    field = table.schema.field("doubled")
-    assert field.metadata[b"computed_column.expression"] == b"x * 2"
+    result = table.refresh_column("doubled")
+    assert result.rows_filled == 2
+    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
+
+    table.add([{"x": 5}])
+    assert table.refresh_column("doubled").rows_filled == 1
+    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4, 10]
 
 
 def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
@@ -3873,3 +3877,14 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
     table = db.create_table("computed_mixed", [{"x": 1}])
     with pytest.raises(ValueError):
         table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
+
+
+@pytest.mark.asyncio
+async def test_computed_column_async(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("computed_async", [{"x": 3}])
+
+    await table.add_columns(computed={"tripled": "x * 3"})
+    await table.refresh_column("tripled")
+
+    assert (await table.to_arrow())["tripled"].to_pylist() == [9]

python/src/lib.rs

@@ -16,7 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
 use table::{
     AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
-    LsmWriteSpec, MergeResult, PyBlobFile, Table, UpdateFieldMetadataResult, UpdateResult,
+    LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult,
+    UpdateResult,
 };
 
 pub mod arrow;
@@ -57,6 +58,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<VectorQuery>()?;
     m.add_class::<RecordBatchStream>()?;
     m.add_class::<AddColumnsResult>()?;
+    m.add_class::<RefreshColumnResult>()?;
     m.add_class::<AlterColumnsResult>()?;
     m.add_class::<UpdateFieldMetadataResult>()?;
     m.add_class::<AddResult>()?;

python/src/table.rs

@@ -415,6 +415,32 @@ pub struct AddColumnsResult {
     pub version: u64,
 }
 
+#[pyclass(get_all, from_py_object)]
+#[derive(Clone, Debug)]
+pub struct RefreshColumnResult {
+    pub rows_filled: u64,
+    pub version: u64,
+}
+
+#[pymethods]
+impl RefreshColumnResult {
+    pub fn __repr__(&self) -> String {
+        format!(
+            "RefreshColumnResult(rows_filled={}, version={})",
+            self.rows_filled, self.version
+        )
+    }
+}
+
+impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
+    fn from(result: lancedb::table::RefreshColumnResult) -> Self {
+        Self {
+            rows_filled: result.rows_filled,
+            version: result.version,
+        }
+    }
+}
+
 #[pymethods]
 impl AddColumnsResult {
     pub fn __repr__(&self) -> String {
@@ -1525,6 +1551,14 @@ impl Table {
         })
     }
 
+    pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let result = inner.refresh_column(column).await.infer_error()?;
+            Ok(RefreshColumnResult::from(result))
+        })
+    }
+
     pub fn add_columns_with_schema(
         self_: PyRef<'_, Self>,
         schema: PyArrowType<Schema>,