mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: expose structured Function refresh options in SDKs (#5047)
## Problem ENT-3267 still lacks typed refresh controls for `supervisor_resources` and `planning_workers` in Python, Rust, and Node, and Node has no `retry_policy` refresh option. Callers must drop to SQL for those controls. ## Change - Add canonical Rust `FunctionRefreshPodResources` and `FunctionRefreshPlanningConfig`, and serialize them through `RefreshColumnOptions`. - Expose both options in the Python sync/async wrappers and PyO3 binding. - Expose both options in the Node NAPI object and add a Node `retryPolicy` object backed by the existing Rust `FunctionRetryPolicy`. - Omitted options stay omitted; branch targeting continues through the table handle. No server, dependency, or pin changes are included. Client-side checks only reject non-positive structural values. Deployment resource limits and column-kind validation remain server-side. ## Validation `cargo fmt --all -- --check`, `python3 -m py_compile`, and `git diff --check` pass. No native build was run locally; CI compiles and tests the change. Generated JS API pages are unchanged.
This commit is contained in:
@@ -979,6 +979,42 @@ describe("remote connection", () => {
|
||||
},
|
||||
"resourceBudget.gpuCount",
|
||||
],
|
||||
[
|
||||
{
|
||||
supervisorResources: {
|
||||
cpuMillis: 0,
|
||||
memoryBytes: 1_024,
|
||||
ephemeralStorageBytes: 2_048,
|
||||
},
|
||||
},
|
||||
"supervisorResources.cpuMillis",
|
||||
],
|
||||
[
|
||||
{
|
||||
supervisorResources: {
|
||||
cpuMillis: 1_000,
|
||||
memoryBytes: 1_024,
|
||||
ephemeralStorageBytes: 0,
|
||||
},
|
||||
},
|
||||
"supervisorResources.ephemeralStorageBytes",
|
||||
],
|
||||
[
|
||||
{
|
||||
planningWorkers: {
|
||||
resources: {
|
||||
cpuMillis: 1_000,
|
||||
memoryBytes: 1_024,
|
||||
ephemeralStorageBytes: 2_048,
|
||||
},
|
||||
maxWorkers: 0,
|
||||
rowsPerWorker: 100,
|
||||
fragmentsPerWorker: 1,
|
||||
},
|
||||
},
|
||||
"planningWorkers.maxWorkers",
|
||||
],
|
||||
[{ retryPolicy: { maxAttempts: 0 } }, "retry_policy.max_attempts"],
|
||||
] as const) {
|
||||
await expect(
|
||||
table.refreshColumnAsync("embedding", options),
|
||||
@@ -1016,6 +1052,91 @@ describe("remote connection", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("forwards structured Function refresh options and Node retry policy", async () => {
|
||||
const bodies: unknown[] = [];
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const path = req.url ?? "";
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(
|
||||
JSON.stringify({
|
||||
name: "docs",
|
||||
version: 1,
|
||||
schema: { fields: [] },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (path === "/v1/table/docs/backfill_column") {
|
||||
let body = "";
|
||||
req.on("data", (chunk) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
bodies.push(JSON.parse(body));
|
||||
res
|
||||
.writeHead(202, { "Content-Type": "application/json" })
|
||||
.end('{"job_id": "j-structured"}');
|
||||
});
|
||||
return;
|
||||
}
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("docs");
|
||||
const job = await table.refreshColumnAsync("embedding", {
|
||||
supervisorResources: {
|
||||
cpuMillis: 2_000,
|
||||
memoryBytes: 4 * 1024 ** 3,
|
||||
ephemeralStorageBytes: 10 * 1024 ** 3,
|
||||
},
|
||||
planningWorkers: {
|
||||
resources: {
|
||||
cpuMillis: 1_000,
|
||||
memoryBytes: 2 * 1024 ** 3,
|
||||
ephemeralStorageBytes: 5 * 1024 ** 3,
|
||||
},
|
||||
maxWorkers: 4,
|
||||
rowsPerWorker: 100_000,
|
||||
fragmentsPerWorker: 8,
|
||||
},
|
||||
retryPolicy: { transient: true },
|
||||
});
|
||||
expect(job.id).toBe("j-structured");
|
||||
},
|
||||
);
|
||||
expect(bodies).toEqual([
|
||||
{
|
||||
column: "embedding",
|
||||
["supervisor_resources"]: {
|
||||
["cpu_millis"]: 2_000,
|
||||
["memory_bytes"]: 4 * 1024 ** 3,
|
||||
["ephemeral_storage_bytes"]: 10 * 1024 ** 3,
|
||||
},
|
||||
["planning_workers"]: {
|
||||
resources: {
|
||||
["cpu_millis"]: 1_000,
|
||||
["memory_bytes"]: 2 * 1024 ** 3,
|
||||
["ephemeral_storage_bytes"]: 5 * 1024 ** 3,
|
||||
},
|
||||
["max_workers"]: 4,
|
||||
["rows_per_worker"]: 100_000,
|
||||
["fragments_per_worker"]: 8,
|
||||
},
|
||||
["retry_policy"]: {
|
||||
["max_attempts"]: 3,
|
||||
["initial_backoff_seconds"]: 0.2,
|
||||
["max_backoff_seconds"]: 30,
|
||||
["exception_types"]: [],
|
||||
["message_pattern"]: null,
|
||||
["error_codes"]: [],
|
||||
transient: true,
|
||||
["honor_retry_after"]: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("surfaces JSON server errors from remote table operations", async () => {
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
|
||||
@@ -5716,6 +5716,31 @@ describe("computed columns", () => {
|
||||
resourceBudget: { cpuMillis: 1_000, memoryBytes: 1_024 },
|
||||
}),
|
||||
).rejects.toThrow("resource_budget requires a LanceDB Enterprise refresh");
|
||||
await expect(
|
||||
table.refreshColumn("x", {
|
||||
supervisorResources: {
|
||||
cpuMillis: 1_000,
|
||||
memoryBytes: 1_024,
|
||||
ephemeralStorageBytes: 2_048,
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"supervisor_resources requires a LanceDB Enterprise refresh",
|
||||
);
|
||||
await expect(
|
||||
table.refreshColumnAsync("x", {
|
||||
planningWorkers: {
|
||||
resources: {
|
||||
cpuMillis: 1_000,
|
||||
memoryBytes: 1_024,
|
||||
ephemeralStorageBytes: 2_048,
|
||||
},
|
||||
maxWorkers: 2,
|
||||
rowsPerWorker: 100,
|
||||
fragmentsPerWorker: 1,
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("planning_workers requires a LanceDB Enterprise refresh");
|
||||
});
|
||||
|
||||
it("records Function errors only on remote tables", async () => {
|
||||
|
||||
@@ -57,6 +57,9 @@ export {
|
||||
AddResult,
|
||||
AddColumnsResult,
|
||||
RefreshColumnOptions,
|
||||
FunctionRefreshPlanningConfig,
|
||||
FunctionRefreshPodResources,
|
||||
FunctionRetryPolicy,
|
||||
FunctionWorkerResourceBudget,
|
||||
RefreshColumnResult,
|
||||
FunctionErrors,
|
||||
|
||||
@@ -831,12 +831,18 @@ export abstract class Table {
|
||||
* `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
|
||||
* `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
|
||||
* and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
|
||||
* `maxConcurrency`, `resourceBudget` and
|
||||
* `maxExecutorProcessesPerWorker` size registered Function refreshes.
|
||||
* `maxConcurrency`, `resourceBudget`, `supervisorResources`,
|
||||
* `planningWorkers` and `maxExecutorProcessesPerWorker` size registered
|
||||
* Function refreshes. `retryPolicy` overrides user-code retries only when
|
||||
* repeating partial work is safe; set `maxAttempts: 1` to disable retries.
|
||||
* `maxConcurrency` overrides the registered default for this logical
|
||||
* execution only and is a positive integer; it is not an account-wide
|
||||
* quota. A `resourceBudget` requires `cpuMillis` and
|
||||
* `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
|
||||
* `supervisorResources` and each `planningWorkers.resources` entry require
|
||||
* positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`.
|
||||
* `planningWorkers` also requires positive `maxWorkers`,
|
||||
* `rowsPerWorker` and `fragmentsPerWorker`.
|
||||
* The budget applies only to this explicit refresh and does not configure
|
||||
* a later automatic refresh.
|
||||
* `gpuCount` at the top level is the shorthand that preserves the
|
||||
|
||||
@@ -9,6 +9,8 @@ use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
use lancedb::table::{
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration,
|
||||
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
|
||||
FunctionRefreshPlanningConfig as LanceFunctionRefreshPlanningConfig,
|
||||
FunctionRefreshPodResources as LanceFunctionRefreshPodResources,
|
||||
FunctionWorkerResourceBudget as LanceFunctionWorkerResourceBudget, NewColumnTransform, Ref,
|
||||
Table as LanceDbTable,
|
||||
};
|
||||
@@ -1705,6 +1707,58 @@ pub struct FunctionWorkerResourceBudget {
|
||||
pub gpu_count: Option<f64>,
|
||||
}
|
||||
|
||||
/// CPU, memory, and ephemeral-storage limits for one Function refresh
|
||||
/// control-plane pod.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FunctionRefreshPodResources {
|
||||
/// CPU limit in millicores; 1000 is one CPU.
|
||||
pub cpu_millis: f64,
|
||||
/// Memory limit in bytes.
|
||||
pub memory_bytes: f64,
|
||||
/// Ephemeral storage limit in bytes.
|
||||
pub ephemeral_storage_bytes: f64,
|
||||
}
|
||||
|
||||
/// Resources and sizing policy for the planning fleet of one registered
|
||||
/// Function refresh.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FunctionRefreshPlanningConfig {
|
||||
/// CPU, memory, and ephemeral-storage limits for one planning worker.
|
||||
pub resources: FunctionRefreshPodResources,
|
||||
/// Maximum number of planning workers.
|
||||
pub max_workers: f64,
|
||||
/// Maximum rows assigned to one planning worker.
|
||||
pub rows_per_worker: f64,
|
||||
/// Maximum fragments assigned to one planning worker.
|
||||
pub fragments_per_worker: f64,
|
||||
}
|
||||
|
||||
/// Overrides retries of failed user-code calls for one refresh. Omitted
|
||||
/// fields use the Function refresh defaults.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct FunctionRetryPolicy {
|
||||
/// Total calls, including the first; 1 disables retries, maximum 100.
|
||||
pub max_attempts: Option<u32>,
|
||||
/// Initial exponential-backoff ceiling, in seconds.
|
||||
pub initial_backoff_seconds: Option<f64>,
|
||||
/// Maximum backoff, in seconds.
|
||||
pub max_backoff_seconds: Option<f64>,
|
||||
/// Exception class names, preferably qualified.
|
||||
pub exception_types: Option<Vec<String>>,
|
||||
/// Regular expression matched against the exception class and message.
|
||||
pub message_pattern: Option<String>,
|
||||
/// Structured provider error codes or HTTP status codes.
|
||||
pub error_codes: Option<Vec<String>>,
|
||||
/// Match connection/timeouts, HTTP 429/503, SlowDown and
|
||||
/// ServiceUnavailable.
|
||||
pub transient: Option<bool>,
|
||||
/// Honor Retry-After within the configured maximum.
|
||||
pub honor_retry_after: Option<bool>,
|
||||
}
|
||||
|
||||
fn positive_safe_integer(value: f64, name: &str) -> napi::Result<u64> {
|
||||
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
|
||||
if !value.is_finite() || value.fract() != 0.0 || !(1.0..=MAX_SAFE_INTEGER).contains(&value) {
|
||||
@@ -1768,8 +1822,14 @@ pub struct RefreshColumnOptions {
|
||||
/// Complete aggregate CPU and memory budget for a registered Function
|
||||
/// refresh. `gpuCount` inside the budget defaults to zero.
|
||||
pub resource_budget: Option<FunctionWorkerResourceBudget>,
|
||||
/// CPU, memory, and ephemeral-storage limits for the refresh supervisor.
|
||||
pub supervisor_resources: Option<FunctionRefreshPodResources>,
|
||||
/// Resources and sizing for the refresh planning fleet.
|
||||
pub planning_workers: Option<FunctionRefreshPlanningConfig>,
|
||||
/// Positive aggregate GPU budget for Function workers.
|
||||
pub gpu_count: Option<u32>,
|
||||
/// Override retries of failed user-code calls for this refresh.
|
||||
pub retry_policy: Option<FunctionRetryPolicy>,
|
||||
/// `"fail"` (default) stops on a row error; `"skip"` writes null and
|
||||
/// records the error, subject to the failure limits.
|
||||
pub error_policy: Option<String>,
|
||||
@@ -1814,7 +1874,87 @@ impl TryFrom<RefreshColumnOptions> for lancedb::table::RefreshColumnOptions {
|
||||
)
|
||||
})
|
||||
.transpose()?;
|
||||
options.supervisor_resources = value
|
||||
.supervisor_resources
|
||||
.map(|resources| {
|
||||
Ok::<_, napi::Error>(LanceFunctionRefreshPodResources::new(
|
||||
positive_safe_integer(resources.cpu_millis, "supervisorResources.cpuMillis")?,
|
||||
positive_safe_integer(
|
||||
resources.memory_bytes,
|
||||
"supervisorResources.memoryBytes",
|
||||
)?,
|
||||
positive_safe_integer(
|
||||
resources.ephemeral_storage_bytes,
|
||||
"supervisorResources.ephemeralStorageBytes",
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.transpose()?;
|
||||
options.planning_workers = value
|
||||
.planning_workers
|
||||
.map(|planning| {
|
||||
let resources = LanceFunctionRefreshPodResources::new(
|
||||
positive_safe_integer(
|
||||
planning.resources.cpu_millis,
|
||||
"planningWorkers.resources.cpuMillis",
|
||||
)?,
|
||||
positive_safe_integer(
|
||||
planning.resources.memory_bytes,
|
||||
"planningWorkers.resources.memoryBytes",
|
||||
)?,
|
||||
positive_safe_integer(
|
||||
planning.resources.ephemeral_storage_bytes,
|
||||
"planningWorkers.resources.ephemeralStorageBytes",
|
||||
)?,
|
||||
);
|
||||
let max_workers = usize::try_from(positive_safe_integer(
|
||||
planning.max_workers,
|
||||
"planningWorkers.maxWorkers",
|
||||
)?)
|
||||
.map_err(|_| napi::Error::from_reason("planningWorkers.maxWorkers is too large"))?;
|
||||
Ok::<_, napi::Error>(LanceFunctionRefreshPlanningConfig::new(
|
||||
resources,
|
||||
max_workers,
|
||||
positive_safe_integer(
|
||||
planning.rows_per_worker,
|
||||
"planningWorkers.rowsPerWorker",
|
||||
)?,
|
||||
positive_safe_integer(
|
||||
planning.fragments_per_worker,
|
||||
"planningWorkers.fragmentsPerWorker",
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.transpose()?;
|
||||
options.gpu_count = value.gpu_count;
|
||||
options.retry_policy = value.retry_policy.map(|policy| {
|
||||
let mut converted = lancedb::function::FunctionRetryPolicy::default();
|
||||
if let Some(max_attempts) = policy.max_attempts {
|
||||
converted.max_attempts = max_attempts;
|
||||
}
|
||||
if let Some(seconds) = policy.initial_backoff_seconds {
|
||||
converted.initial_backoff_seconds = seconds;
|
||||
}
|
||||
if let Some(seconds) = policy.max_backoff_seconds {
|
||||
converted.max_backoff_seconds = seconds;
|
||||
}
|
||||
if let Some(exception_types) = policy.exception_types {
|
||||
converted.exception_types = exception_types;
|
||||
}
|
||||
if let Some(pattern) = policy.message_pattern {
|
||||
converted.message_pattern = Some(pattern);
|
||||
}
|
||||
if let Some(error_codes) = policy.error_codes {
|
||||
converted.error_codes = error_codes;
|
||||
}
|
||||
if let Some(transient) = policy.transient {
|
||||
converted.transient = transient;
|
||||
}
|
||||
if let Some(honor_retry_after) = policy.honor_retry_after {
|
||||
converted.honor_retry_after = honor_retry_after;
|
||||
}
|
||||
converted
|
||||
});
|
||||
options.error_policy = match value.error_policy.as_deref() {
|
||||
None => None,
|
||||
Some("fail") => Some(lancedb::function::FunctionErrorPolicy::Fail),
|
||||
|
||||
Reference in New Issue
Block a user