feat: refresh computed columns (#3938)

table.refresh_column("doubled") fills the rows of a declared column that
hold no value, in two passes per fragment: the first scans only the
unfilled
live rows to count exact gains and decide staging, the second streams
the
fragment's physical rows into a standalone column file published in one
DataReplacement -- committed under the dataset's own session -- so peak
memory is bounded by a scan batch. A row that holds a value keeps it;
deleted and already-filled rows never reach the expression, so a poison
value in them cannot fail the refresh. Refresh refuses under an LSM
write
spec, including the mem-wal catch-up flag that outlives unset and marks
retained SSTable rows.

---

<sub>Stack created with <a
href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a
href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
This commit is contained in:
Wyatt Alt
2026-08-14 14:43:41 -07:00
committed by GitHub
parent def869bb78
commit fc0d917d32
18 changed files with 1042 additions and 29 deletions
+34
View File
@@ -415,6 +415,32 @@ pub struct AddColumnsResult {
pub version: u64,
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RefreshColumnResult {
pub rows_filled: u64,
pub version: u64,
}
#[pymethods]
impl RefreshColumnResult {
pub fn __repr__(&self) -> String {
format!(
"RefreshColumnResult(rows_filled={}, version={})",
self.rows_filled, self.version
)
}
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(result: lancedb::table::RefreshColumnResult) -> Self {
Self {
rows_filled: result.rows_filled,
version: result.version,
}
}
}
#[pymethods]
impl AddColumnsResult {
pub fn __repr__(&self) -> String {
@@ -1525,6 +1551,14 @@ impl Table {
})
}
pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner.refresh_column(column).await.infer_error()?;
Ok(RefreshColumnResult::from(result))
})
}
pub fn add_columns_with_schema(
self_: PyRef<'_, Self>,
schema: PyArrowType<Schema>,