feat: expose the remaining refresh options on the SDKs (#4890)

The server's backfill request takes a row filter, failure limits,
per-call timeouts and worker sizing, and SQL exposes all of them through
`REFRESH COLUMN ... WHERE ... WITH (...)`, but the typed SDKs could send
only batch_size, GPU placement and the retry and error policies. Anyone
needing a filtered refresh or a bounded skip policy had to drop to SQL.

This adds `where`, `max_failed_fraction`, `max_failed_rows`,
`row_timeout_seconds`, `init_timeout_seconds`, `num_workers` and
`max_executor_processes_per_worker` to Python, Rust and Node, passed
through unchanged. `RefreshColumnOptions` now serializes straight into
the request body, so the three SDKs share one producer. Node gets a
`RefreshColumnOptions` object, which also carries the options it lacked
before. A LanceDB Embedded refresh rejects every option other than
batch_size and GPU placement, naming the option.
This commit is contained in:
Wyatt Alt
2026-11-06 09:19:27 -08:00
committed by GitHub
parent d379bbb722
commit 2af85bea89
18 changed files with 974 additions and 53 deletions
+84 -4
View File
@@ -421,20 +421,30 @@ impl Table {
}
#[napi(catch_unwind)]
pub async fn refresh_column(&self, column: String) -> napi::Result<RefreshColumnResult> {
pub async fn refresh_column(
&self,
column: String,
options: Option<RefreshColumnOptions>,
) -> napi::Result<RefreshColumnResult> {
let options = options.unwrap_or_default().try_into()?;
let res = self
.inner_ref()?
.refresh_column(column)
.refresh_column_with_options(column, options)
.await
.default_error()?;
Ok(res.into())
}
#[napi(catch_unwind)]
pub async fn refresh_column_async(&self, column: String) -> napi::Result<crate::job::Job> {
pub async fn refresh_column_async(
&self,
column: String,
options: Option<RefreshColumnOptions>,
) -> napi::Result<crate::job::Job> {
let options = options.unwrap_or_default().try_into()?;
let job = self
.inner_ref()?
.refresh_column_async(column)
.refresh_column_async_with_options(column, options)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
@@ -1515,6 +1525,76 @@ pub struct FunctionErrorsOptions {
pub limit: Option<i64>,
}
/// Options for `Table.refreshColumn` and `Table.refreshColumnAsync`. Every
/// option is LanceDB Enterprise only; a LanceDB Embedded refresh ignores
/// `batchSize`, `gpuClass` and `gpuCount` and rejects the rest.
#[napi(object)]
#[derive(Clone, Debug, Default)]
pub struct RefreshColumnOptions {
/// Only refresh the rows matching this SQL filter.
#[napi(js_name = "where")]
pub where_clause: Option<String>,
/// Maximum rows per Function call; must be greater than zero.
pub batch_size: Option<u32>,
/// GPU class for Function workers.
pub gpu_class: Option<String>,
/// Positive aggregate GPU budget for Function workers.
pub gpu_count: Option<u32>,
/// `"fail"` (default) stops on a row error; `"skip"` writes null and
/// records the error, subject to the failure limits.
pub error_policy: Option<String>,
/// Under a skip policy, fail the job once more than this share of rows
/// (0 to 1) has failed.
pub max_failed_fraction: Option<f64>,
/// Under a skip policy, fail the job once more than this many rows have
/// failed.
pub max_failed_rows: Option<i64>,
/// Deadline for one Function call, in seconds.
pub row_timeout_seconds: Option<f64>,
/// Deadline for Function initialization, in seconds.
pub init_timeout_seconds: Option<f64>,
/// Worker count for an expression column; Function columns reject it.
pub num_workers: Option<u32>,
/// Cap on executor processes per Function worker.
pub max_executor_processes_per_worker: Option<u32>,
}
impl TryFrom<RefreshColumnOptions> for lancedb::table::RefreshColumnOptions {
type Error = napi::Error;
fn try_from(value: RefreshColumnOptions) -> napi::Result<Self> {
let mut options = Self::default();
options.where_clause = value.where_clause;
options.batch_size = value.batch_size.map(|size| size as usize);
options.gpu_class = value.gpu_class;
options.gpu_count = value.gpu_count;
options.error_policy = match value.error_policy.as_deref() {
None => None,
Some("fail") => Some(lancedb::function::FunctionErrorPolicy::Fail),
Some("skip") => Some(lancedb::function::FunctionErrorPolicy::Skip),
Some(other) => {
return Err(napi::Error::from_reason(format!(
"errorPolicy must be \"fail\" or \"skip\", got \"{other}\""
)));
}
};
options.max_failed_fraction = value.max_failed_fraction;
options.max_failed_rows = value
.max_failed_rows
.map(|rows| {
u64::try_from(rows).map_err(|_| {
napi::Error::from_reason("maxFailedRows must be a non-negative integer")
})
})
.transpose()?;
options.row_timeout_seconds = value.row_timeout_seconds;
options.init_timeout_seconds = value.init_timeout_seconds;
options.num_workers = value.num_workers.map(|workers| workers as usize);
options.max_executor_processes_per_worker = value.max_executor_processes_per_worker;
Ok(options)
}
}
/// One row a Function refresh skipped, as the server recorded it.
#[napi(object)]
#[derive(Clone, Debug)]