mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat: expose the remaining refresh options on the SDKs (#4890)
The server's backfill request takes a row filter, failure limits, per-call timeouts and worker sizing, and SQL exposes all of them through `REFRESH COLUMN ... WHERE ... WITH (...)`, but the typed SDKs could send only batch_size, GPU placement and the retry and error policies. Anyone needing a filtered refresh or a bounded skip policy had to drop to SQL. This adds `where`, `max_failed_fraction`, `max_failed_rows`, `row_timeout_seconds`, `init_timeout_seconds`, `num_workers` and `max_executor_processes_per_worker` to Python, Rust and Node, passed through unchanged. `RefreshColumnOptions` now serializes straight into the request body, so the three SDKs share one producer. Node gets a `RefreshColumnOptions` object, which also carries the options it lacked before. A LanceDB Embedded refresh rejects every option other than batch_size and GPU placement, naming the option.
This commit is contained in:
+84
-4
@@ -421,20 +421,30 @@ impl Table {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh_column(&self, column: String) -> napi::Result<RefreshColumnResult> {
|
||||
pub async fn refresh_column(
|
||||
&self,
|
||||
column: String,
|
||||
options: Option<RefreshColumnOptions>,
|
||||
) -> napi::Result<RefreshColumnResult> {
|
||||
let options = options.unwrap_or_default().try_into()?;
|
||||
let res = self
|
||||
.inner_ref()?
|
||||
.refresh_column(column)
|
||||
.refresh_column_with_options(column, options)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(res.into())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh_column_async(&self, column: String) -> napi::Result<crate::job::Job> {
|
||||
pub async fn refresh_column_async(
|
||||
&self,
|
||||
column: String,
|
||||
options: Option<RefreshColumnOptions>,
|
||||
) -> napi::Result<crate::job::Job> {
|
||||
let options = options.unwrap_or_default().try_into()?;
|
||||
let job = self
|
||||
.inner_ref()?
|
||||
.refresh_column_async(column)
|
||||
.refresh_column_async_with_options(column, options)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
@@ -1515,6 +1525,76 @@ pub struct FunctionErrorsOptions {
|
||||
pub limit: Option<i64>,
|
||||
}
|
||||
|
||||
/// Options for `Table.refreshColumn` and `Table.refreshColumnAsync`. Every
|
||||
/// option is LanceDB Enterprise only; a LanceDB Embedded refresh ignores
|
||||
/// `batchSize`, `gpuClass` and `gpuCount` and rejects the rest.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct RefreshColumnOptions {
|
||||
/// Only refresh the rows matching this SQL filter.
|
||||
#[napi(js_name = "where")]
|
||||
pub where_clause: Option<String>,
|
||||
/// Maximum rows per Function call; must be greater than zero.
|
||||
pub batch_size: Option<u32>,
|
||||
/// GPU class for Function workers.
|
||||
pub gpu_class: Option<String>,
|
||||
/// Positive aggregate GPU budget for Function workers.
|
||||
pub gpu_count: Option<u32>,
|
||||
/// `"fail"` (default) stops on a row error; `"skip"` writes null and
|
||||
/// records the error, subject to the failure limits.
|
||||
pub error_policy: Option<String>,
|
||||
/// Under a skip policy, fail the job once more than this share of rows
|
||||
/// (0 to 1) has failed.
|
||||
pub max_failed_fraction: Option<f64>,
|
||||
/// Under a skip policy, fail the job once more than this many rows have
|
||||
/// failed.
|
||||
pub max_failed_rows: Option<i64>,
|
||||
/// Deadline for one Function call, in seconds.
|
||||
pub row_timeout_seconds: Option<f64>,
|
||||
/// Deadline for Function initialization, in seconds.
|
||||
pub init_timeout_seconds: Option<f64>,
|
||||
/// Worker count for an expression column; Function columns reject it.
|
||||
pub num_workers: Option<u32>,
|
||||
/// Cap on executor processes per Function worker.
|
||||
pub max_executor_processes_per_worker: Option<u32>,
|
||||
}
|
||||
|
||||
impl TryFrom<RefreshColumnOptions> for lancedb::table::RefreshColumnOptions {
|
||||
type Error = napi::Error;
|
||||
|
||||
fn try_from(value: RefreshColumnOptions) -> napi::Result<Self> {
|
||||
let mut options = Self::default();
|
||||
options.where_clause = value.where_clause;
|
||||
options.batch_size = value.batch_size.map(|size| size as usize);
|
||||
options.gpu_class = value.gpu_class;
|
||||
options.gpu_count = value.gpu_count;
|
||||
options.error_policy = match value.error_policy.as_deref() {
|
||||
None => None,
|
||||
Some("fail") => Some(lancedb::function::FunctionErrorPolicy::Fail),
|
||||
Some("skip") => Some(lancedb::function::FunctionErrorPolicy::Skip),
|
||||
Some(other) => {
|
||||
return Err(napi::Error::from_reason(format!(
|
||||
"errorPolicy must be \"fail\" or \"skip\", got \"{other}\""
|
||||
)));
|
||||
}
|
||||
};
|
||||
options.max_failed_fraction = value.max_failed_fraction;
|
||||
options.max_failed_rows = value
|
||||
.max_failed_rows
|
||||
.map(|rows| {
|
||||
u64::try_from(rows).map_err(|_| {
|
||||
napi::Error::from_reason("maxFailedRows must be a non-negative integer")
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
options.row_timeout_seconds = value.row_timeout_seconds;
|
||||
options.init_timeout_seconds = value.init_timeout_seconds;
|
||||
options.num_workers = value.num_workers.map(|workers| workers as usize);
|
||||
options.max_executor_processes_per_worker = value.max_executor_processes_per_worker;
|
||||
Ok(options)
|
||||
}
|
||||
}
|
||||
|
||||
/// One row a Function refresh skipped, as the server recorded it.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
|
||||
Reference in New Issue
Block a user