feat: expose structured Function refresh options in SDKs (#5047)

## Problem

ENT-3267 still lacks typed refresh controls for `supervisor_resources`
and `planning_workers` in Python, Rust, and Node, and Node has no
`retry_policy` refresh option. Callers must drop to SQL for those
controls.

## Change

- Add canonical Rust `FunctionRefreshPodResources` and
`FunctionRefreshPlanningConfig`, and serialize them through
`RefreshColumnOptions`.
- Expose both options in the Python sync/async wrappers and PyO3
binding.
- Expose both options in the Node NAPI object and add a Node
`retryPolicy` object backed by the existing Rust `FunctionRetryPolicy`.
- Omitted options stay omitted; branch targeting continues through the
table handle. No server, dependency, or pin changes are included.

Client-side checks only reject non-positive structural values.
Deployment resource limits and column-kind validation remain
server-side.

## Validation

`cargo fmt --all -- --check`, `python3 -m py_compile`, and `git diff
--check` pass. No native build was run locally; CI compiles and tests
the change. Generated JS API pages are unchanged.
This commit is contained in:
Xuanwo
2026-11-14 20:59:44 +08:00
committed by GitHub
parent 4843ac4c7e
commit f6f8119ba4
21 changed files with 1164 additions and 10 deletions
+114 -4
View File
@@ -30,8 +30,9 @@ use lancedb::connection::LanceFileVersion;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::table::{
AddDataMode, ColumnAlteration, CompactionMode, CompactionOptions, Duration,
FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionWorkerResourceBudget, IndexRemapMode,
NewColumnTransform, Ref, RefreshColumnOptions, Table as LanceDbTable,
FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionRefreshPlanningConfig,
FunctionRefreshPodResources, FunctionWorkerResourceBudget, IndexRemapMode, NewColumnTransform,
Ref, RefreshColumnOptions, Table as LanceDbTable,
};
use lancedb::tokenize as lancedb_tokenize;
use pyo3::{
@@ -52,6 +53,8 @@ struct RefreshColumnArgs {
max_concurrency: Option<u32>,
publication_interval_rows: Option<i64>,
resource_budget: Option<FunctionWorkerResourceBudget>,
supervisor_resources: Option<FunctionRefreshPodResources>,
planning_workers: Option<FunctionRefreshPlanningConfig>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -159,6 +162,101 @@ fn parse_resource_budget(
))
}
fn structured_integer(value: &Bound<'_, PyAny>, name: &str) -> PyResult<u64> {
if value.is_none() || value.is_instance_of::<PyBool>() {
return Err(PyValueError::new_err(format!("{name} must be an integer")));
}
let parsed = value
.extract::<u64>()
.map_err(|_| PyValueError::new_err(format!("{name} must be an integer")))?;
if parsed == 0 {
return Err(PyValueError::new_err(format!(
"{name} must be greater than zero"
)));
}
Ok(parsed)
}
fn parse_pod_resources(
resources: &Bound<'_, PyDict>,
prefix: &str,
) -> PyResult<FunctionRefreshPodResources> {
for key in resources.keys() {
let key = key
.extract::<String>()
.map_err(|_| PyValueError::new_err(format!("{prefix} keys must be strings")))?;
if !matches!(
key.as_str(),
"cpu_millis" | "memory_bytes" | "ephemeral_storage_bytes"
) {
return Err(PyValueError::new_err(format!(
"{prefix} contains unsupported key '{key}'"
)));
}
}
let get = |name: &str| -> PyResult<u64> {
let value = resources
.get_item(name)?
.ok_or_else(|| PyValueError::new_err(format!("{prefix}.{name} is required")))?;
structured_integer(&value, &format!("{prefix}.{name}"))
};
Ok(FunctionRefreshPodResources::new(
get("cpu_millis")?,
get("memory_bytes")?,
get("ephemeral_storage_bytes")?,
))
}
fn parse_supervisor_resources(
resources: Option<&Bound<'_, PyDict>>,
) -> PyResult<Option<FunctionRefreshPodResources>> {
resources
.map(|resources| parse_pod_resources(resources, "supervisor_resources"))
.transpose()
}
fn parse_planning_workers(
planning: Option<&Bound<'_, PyDict>>,
) -> PyResult<Option<FunctionRefreshPlanningConfig>> {
let Some(planning) = planning else {
return Ok(None);
};
for key in planning.keys() {
let key = key
.extract::<String>()
.map_err(|_| PyValueError::new_err("planning_workers keys must be strings"))?;
if !matches!(
key.as_str(),
"resources" | "max_workers" | "rows_per_worker" | "fragments_per_worker"
) {
return Err(PyValueError::new_err(format!(
"planning_workers contains unsupported key '{key}'"
)));
}
}
let resources = planning
.get_item("resources")?
.ok_or_else(|| PyValueError::new_err("planning_workers.resources is required"))?;
let resources = resources
.cast::<PyDict>()
.map_err(|_| PyValueError::new_err("planning_workers.resources must be a dict"))?;
let resources = parse_pod_resources(resources, "planning_workers.resources")?;
let get = |name: &str| -> PyResult<u64> {
let value = planning
.get_item(name)?
.ok_or_else(|| PyValueError::new_err(format!("planning_workers.{name} is required")))?;
structured_integer(&value, &format!("planning_workers.{name}"))
};
let max_workers = usize::try_from(get("max_workers")?)
.map_err(|_| PyValueError::new_err("planning_workers.max_workers is too large"))?;
Ok(Some(FunctionRefreshPlanningConfig::new(
resources,
max_workers,
get("rows_per_worker")?,
get("fragments_per_worker")?,
)))
}
fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOptions> {
let RefreshColumnArgs {
where_clause,
@@ -166,6 +264,8 @@ fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOpti
max_concurrency,
publication_interval_rows,
resource_budget,
supervisor_resources,
planning_workers,
gpu_class,
gpu_count,
retry_policy,
@@ -211,6 +311,8 @@ fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOpti
options.publication_interval_rows =
positive_count(publication_interval_rows, "publication_interval_rows")?;
options.resource_budget = resource_budget;
options.supervisor_resources = supervisor_resources;
options.planning_workers = planning_workers;
options.gpu_class = gpu_class;
options.gpu_count = gpu_count
.map(|count| {
@@ -2492,7 +2594,7 @@ impl Table {
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, supervisor_resources=None, planning_workers=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column<'py>(
self_: PyRef<'py, Self>,
@@ -2502,6 +2604,8 @@ impl Table {
max_concurrency: Option<Bound<'py, PyAny>>,
publication_interval_rows: Option<i64>,
resource_budget: Option<Bound<'py, PyDict>>,
supervisor_resources: Option<Bound<'py, PyDict>>,
planning_workers: Option<Bound<'py, PyDict>>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -2519,6 +2623,8 @@ impl Table {
max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?,
publication_interval_rows,
resource_budget: parse_resource_budget(resource_budget.as_ref())?,
supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?,
planning_workers: parse_planning_workers(planning_workers.as_ref())?,
gpu_class,
gpu_count,
retry_policy,
@@ -2540,7 +2646,7 @@ impl Table {
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, supervisor_resources=None, planning_workers=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column_async<'py>(
self_: PyRef<'py, Self>,
@@ -2550,6 +2656,8 @@ impl Table {
max_concurrency: Option<Bound<'py, PyAny>>,
publication_interval_rows: Option<i64>,
resource_budget: Option<Bound<'py, PyDict>>,
supervisor_resources: Option<Bound<'py, PyDict>>,
planning_workers: Option<Bound<'py, PyDict>>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -2567,6 +2675,8 @@ impl Table {
max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?,
publication_interval_rows,
resource_budget: parse_resource_budget(resource_budget.as_ref())?,
supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?,
planning_workers: parse_planning_workers(planning_workers.as_ref())?,
gpu_class,
gpu_count,
retry_policy,