From f6f8119ba4fc285914f084fde560ea4e7affd9af Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 20:59:44 +0800 Subject: [PATCH] feat: expose structured Function refresh options in SDKs (#5047) ## Problem ENT-3267 still lacks typed refresh controls for `supervisor_resources` and `planning_workers` in Python, Rust, and Node, and Node has no `retry_policy` refresh option. Callers must drop to SQL for those controls. ## Change - Add canonical Rust `FunctionRefreshPodResources` and `FunctionRefreshPlanningConfig`, and serialize them through `RefreshColumnOptions`. - Expose both options in the Python sync/async wrappers and PyO3 binding. - Expose both options in the Node NAPI object and add a Node `retryPolicy` object backed by the existing Rust `FunctionRetryPolicy`. - Omitted options stay omitted; branch targeting continues through the table handle. No server, dependency, or pin changes are included. Client-side checks only reject non-positive structural values. Deployment resource limits and column-kind validation remain server-side. ## Validation `cargo fmt --all -- --check`, `python3 -m py_compile`, and `git diff --check` pass. No native build was run locally; CI compiles and tests the change. Generated JS API pages are unchanged. --- docs/src/js/classes/Table.md | 10 +- docs/src/js/globals.md | 3 + .../FunctionRefreshPlanningConfig.md | 50 +++++++ .../interfaces/FunctionRefreshPodResources.md | 40 +++++ docs/src/js/interfaces/FunctionRetryPolicy.md | 91 ++++++++++++ .../src/js/interfaces/RefreshColumnOptions.md | 30 ++++ .../feature-engineering/refresh-column.mdx | 128 ++++++++++++++++ nodejs/__test__/remote.test.ts | 121 +++++++++++++++ nodejs/__test__/table.test.ts | 25 ++++ nodejs/lancedb/index.ts | 3 + nodejs/lancedb/table.ts | 10 +- nodejs/src/table.rs | 140 ++++++++++++++++++ python/python/lancedb/_lancedb.pyi | 4 + python/python/lancedb/remote/table.py | 8 + python/python/lancedb/table.py | 64 ++++++++ python/python/tests/test_remote_db.py | 65 ++++++++ python/python/tests/test_table.py | 25 ++++ python/src/table.rs | 118 ++++++++++++++- rust/lancedb/src/remote/table.rs | 106 ++++++++++++- rust/lancedb/src/table.rs | 5 +- rust/lancedb/src/table/refresh.rs | 128 ++++++++++++++++ 21 files changed, 1164 insertions(+), 10 deletions(-) create mode 100644 docs/src/js/interfaces/FunctionRefreshPlanningConfig.md create mode 100644 docs/src/js/interfaces/FunctionRefreshPodResources.md create mode 100644 docs/src/js/interfaces/FunctionRetryPolicy.md diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3150d94ec..f93c1b7cd 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -1225,12 +1225,18 @@ LanceDB Enterprise it is the server's backfill job. `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows, `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds` and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`, - `maxConcurrency`, `resourceBudget` and - `maxExecutorProcessesPerWorker` size registered Function refreshes. + `maxConcurrency`, `resourceBudget`, `supervisorResources`, + `planningWorkers` and `maxExecutorProcessesPerWorker` size registered + Function refreshes. `retryPolicy` overrides user-code retries only when + repeating partial work is safe; set `maxAttempts: 1` to disable retries. `maxConcurrency` overrides the registered default for this logical execution only and is a positive integer; it is not an account-wide quota. A `resourceBudget` requires `cpuMillis` and `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function. + `supervisorResources` and each `planningWorkers.resources` entry require + positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`. + `planningWorkers` also requires positive `maxWorkers`, + `rowsPerWorker` and `fragmentsPerWorker`. The budget applies only to this explicit refresh and does not configure a later automatic refresh. `gpuCount` at the top level is the shorthand that preserves the diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index beed7c8af..23e576877 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -103,6 +103,9 @@ - [FunctionErrorRecord](interfaces/FunctionErrorRecord.md) - [FunctionErrors](interfaces/FunctionErrors.md) - [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md) +- [FunctionRefreshPlanningConfig](interfaces/FunctionRefreshPlanningConfig.md) +- [FunctionRefreshPodResources](interfaces/FunctionRefreshPodResources.md) +- [FunctionRetryPolicy](interfaces/FunctionRetryPolicy.md) - [FunctionWorkerResourceBudget](interfaces/FunctionWorkerResourceBudget.md) - [HnswPqOptions](interfaces/HnswPqOptions.md) - [HnswSqOptions](interfaces/HnswSqOptions.md) diff --git a/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md b/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md new file mode 100644 index 000000000..48a8a209d --- /dev/null +++ b/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md @@ -0,0 +1,50 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionRefreshPlanningConfig + +# Interface: FunctionRefreshPlanningConfig + +Resources and sizing policy for the planning fleet of one registered +Function refresh. + +## Properties + +### fragmentsPerWorker + +```ts +fragmentsPerWorker: number; +``` + +Maximum fragments assigned to one planning worker. + +*** + +### maxWorkers + +```ts +maxWorkers: number; +``` + +Maximum number of planning workers. + +*** + +### resources + +```ts +resources: FunctionRefreshPodResources; +``` + +CPU, memory, and ephemeral-storage limits for one planning worker. + +*** + +### rowsPerWorker + +```ts +rowsPerWorker: number; +``` + +Maximum rows assigned to one planning worker. diff --git a/docs/src/js/interfaces/FunctionRefreshPodResources.md b/docs/src/js/interfaces/FunctionRefreshPodResources.md new file mode 100644 index 000000000..99d55512e --- /dev/null +++ b/docs/src/js/interfaces/FunctionRefreshPodResources.md @@ -0,0 +1,40 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionRefreshPodResources + +# Interface: FunctionRefreshPodResources + +CPU, memory, and ephemeral-storage limits for one Function refresh +control-plane pod. + +## Properties + +### cpuMillis + +```ts +cpuMillis: number; +``` + +CPU limit in millicores; 1000 is one CPU. + +*** + +### ephemeralStorageBytes + +```ts +ephemeralStorageBytes: number; +``` + +Ephemeral storage limit in bytes. + +*** + +### memoryBytes + +```ts +memoryBytes: number; +``` + +Memory limit in bytes. diff --git a/docs/src/js/interfaces/FunctionRetryPolicy.md b/docs/src/js/interfaces/FunctionRetryPolicy.md new file mode 100644 index 000000000..43d4ed8b2 --- /dev/null +++ b/docs/src/js/interfaces/FunctionRetryPolicy.md @@ -0,0 +1,91 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionRetryPolicy + +# Interface: FunctionRetryPolicy + +Overrides retries of failed user-code calls for one refresh. Omitted +fields use the Function refresh defaults. + +## Properties + +### errorCodes? + +```ts +optional errorCodes: string[]; +``` + +Structured provider error codes or HTTP status codes. + +*** + +### exceptionTypes? + +```ts +optional exceptionTypes: string[]; +``` + +Exception class names, preferably qualified. + +*** + +### honorRetryAfter? + +```ts +optional honorRetryAfter: boolean; +``` + +Honor Retry-After within the configured maximum. + +*** + +### initialBackoffSeconds? + +```ts +optional initialBackoffSeconds: number; +``` + +Initial exponential-backoff ceiling, in seconds. + +*** + +### maxAttempts? + +```ts +optional maxAttempts: number; +``` + +Total calls, including the first; 1 disables retries, maximum 100. + +*** + +### maxBackoffSeconds? + +```ts +optional maxBackoffSeconds: number; +``` + +Maximum backoff, in seconds. + +*** + +### messagePattern? + +```ts +optional messagePattern: string; +``` + +Regular expression matched against the exception class and message. + +*** + +### transient? + +```ts +optional transient: boolean; +``` + +Match connection/timeouts, HTTP 429/503, SlowDown and +ServiceUnavailable. diff --git a/docs/src/js/interfaces/RefreshColumnOptions.md b/docs/src/js/interfaces/RefreshColumnOptions.md index c534e0fac..07157e4ab 100644 --- a/docs/src/js/interfaces/RefreshColumnOptions.md +++ b/docs/src/js/interfaces/RefreshColumnOptions.md @@ -117,6 +117,16 @@ Worker count for an expression column; Function columns reject it. *** +### planningWorkers? + +```ts +optional planningWorkers: FunctionRefreshPlanningConfig; +``` + +Resources and sizing for the refresh planning fleet. + +*** + ### publicationIntervalRows? ```ts @@ -141,6 +151,16 @@ refresh. `gpuCount` inside the budget defaults to zero. *** +### retryPolicy? + +```ts +optional retryPolicy: FunctionRetryPolicy; +``` + +Override retries of failed user-code calls for this refresh. + +*** + ### rowTimeoutSeconds? ```ts @@ -151,6 +171,16 @@ Deadline for one Function call, in seconds. *** +### supervisorResources? + +```ts +optional supervisorResources: FunctionRefreshPodResources; +``` + +CPU, memory, and ephemeral-storage limits for the refresh supervisor. + +*** + ### where? ```ts diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index d82fb1eef..f893118fc 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -400,6 +400,101 @@ for changing only the aggregate GPU budget. It cannot be combined with the complete resource budget. Omitting both preserves the server's existing defaults. +{/* +Draft for the Function refresh control-plane resource release. + +`resource_budget` above sizes the Function workers. Two independent options +size the refresh control plane for a registered Function. +`supervisor_resources` sets the CPU, memory and ephemeral-storage limits of +the supervisor pod. `planning_workers` sets the resources of one planning +worker and the sizing of the CPU-only planning fleet: `max_workers`, +`rows_per_worker` and `fragments_per_worker`. Every field is required and +must be positive. Omitting either option leaves the server's configured +defaults unchanged, and either can be combined with `resource_budget`. + + +```python Python icon="python" +job = table.refresh_column_async( + "embedding", + supervisor_resources={ + "cpu_millis": 2_000, + "memory_bytes": 4 * 1024**3, + "ephemeral_storage_bytes": 10 * 1024**3, + }, + planning_workers={ + "resources": { + "cpu_millis": 1_000, + "memory_bytes": 2 * 1024**3, + "ephemeral_storage_bytes": 5 * 1024**3, + }, + "max_workers": 4, + "rows_per_worker": 100_000, + "fragments_per_worker": 8, + }, +) +job.wait() +``` + +```rust Rust icon="rust" +use lancedb::table::{ + FunctionRefreshPlanningConfig, + FunctionRefreshPodResources, + RefreshColumnOptions, +}; + +let _job = table + .refresh_column_async_with_options( + "embedding", + RefreshColumnOptions::default() + .supervisor_resources( + FunctionRefreshPodResources::new( + 2_000, + 4 * 1024 * 1024 * 1024, + 10 * 1024 * 1024 * 1024, + ), + ) + .planning_workers( + FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new( + 1_000, + 2 * 1024 * 1024 * 1024, + 5 * 1024 * 1024 * 1024, + ), + 4, + 100_000, + 8, + ), + ), + ) + .await?; +``` + +```typescript TypeScript icon="typescript" +const job = await table.refreshColumnAsync("embedding", { + supervisorResources: { + cpuMillis: 2_000, + memoryBytes: 4 * 1024 ** 3, + ephemeralStorageBytes: 10 * 1024 ** 3, + }, + planningWorkers: { + resources: { + cpuMillis: 1_000, + memoryBytes: 2 * 1024 ** 3, + ephemeralStorageBytes: 5 * 1024 ** 3, + }, + maxWorkers: 4, + rowsPerWorker: 100_000, + fragmentsPerWorker: 8, + }, +}); +await job.wait(); +``` + + +The SDK validates the shape and sends these values only with this explicit +refresh request. A supporting LanceDB Enterprise server is required. +*/} + These examples configure only the explicit refresh shown. A registered Function declaration can separately set `auto_refresh_resource_budget` on `add_columns`; that declaration controls the budget for its automatic @@ -457,6 +552,39 @@ from `lancedb.functions`. Use `FunctionRetryPolicy(max_attempts=1)` to disable user-error retries. Omitting the override preserves the registered policy. + +```python Python icon="python" +from lancedb.functions import FunctionRetryPolicy + +job = table.refresh_column_async( + "embedding", + retry_policy=FunctionRetryPolicy(transient=True, max_attempts=3), +) +job.wait() +``` + +```rust Rust icon="rust" +use lancedb::{function::FunctionRetryPolicy, table::RefreshColumnOptions}; + +let mut retry_policy = FunctionRetryPolicy::transient(); +retry_policy.max_attempts = 3; + +let _job = table + .refresh_column_async_with_options( + "embedding", + RefreshColumnOptions::default().retry_policy(retry_policy), + ) + .await?; +``` + +```typescript TypeScript icon="typescript" +const job = await table.refreshColumnAsync("embedding", { + retryPolicy: { transient: true, maxAttempts: 3 }, +}); +await job.wait(); +``` + + This requires a supporting LanceDB Enterprise server as well as an SDK that exposes these options. An older Function image may need re-registration before it accepts a refresh override. Only enable retries diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index a9989b84e..3e8d5a79b 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -979,6 +979,42 @@ describe("remote connection", () => { }, "resourceBudget.gpuCount", ], + [ + { + supervisorResources: { + cpuMillis: 0, + memoryBytes: 1_024, + ephemeralStorageBytes: 2_048, + }, + }, + "supervisorResources.cpuMillis", + ], + [ + { + supervisorResources: { + cpuMillis: 1_000, + memoryBytes: 1_024, + ephemeralStorageBytes: 0, + }, + }, + "supervisorResources.ephemeralStorageBytes", + ], + [ + { + planningWorkers: { + resources: { + cpuMillis: 1_000, + memoryBytes: 1_024, + ephemeralStorageBytes: 2_048, + }, + maxWorkers: 0, + rowsPerWorker: 100, + fragmentsPerWorker: 1, + }, + }, + "planningWorkers.maxWorkers", + ], + [{ retryPolicy: { maxAttempts: 0 } }, "retry_policy.max_attempts"], ] as const) { await expect( table.refreshColumnAsync("embedding", options), @@ -1016,6 +1052,91 @@ describe("remote connection", () => { ]); }); + it("forwards structured Function refresh options and Node retry policy", async () => { + const bodies: unknown[] = []; + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "docs", + version: 1, + schema: { fields: [] }, + }), + ); + return; + } + if (path === "/v1/table/docs/backfill_column") { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + bodies.push(JSON.parse(body)); + res + .writeHead(202, { "Content-Type": "application/json" }) + .end('{"job_id": "j-structured"}'); + }); + return; + } + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("docs"); + const job = await table.refreshColumnAsync("embedding", { + supervisorResources: { + cpuMillis: 2_000, + memoryBytes: 4 * 1024 ** 3, + ephemeralStorageBytes: 10 * 1024 ** 3, + }, + planningWorkers: { + resources: { + cpuMillis: 1_000, + memoryBytes: 2 * 1024 ** 3, + ephemeralStorageBytes: 5 * 1024 ** 3, + }, + maxWorkers: 4, + rowsPerWorker: 100_000, + fragmentsPerWorker: 8, + }, + retryPolicy: { transient: true }, + }); + expect(job.id).toBe("j-structured"); + }, + ); + expect(bodies).toEqual([ + { + column: "embedding", + ["supervisor_resources"]: { + ["cpu_millis"]: 2_000, + ["memory_bytes"]: 4 * 1024 ** 3, + ["ephemeral_storage_bytes"]: 10 * 1024 ** 3, + }, + ["planning_workers"]: { + resources: { + ["cpu_millis"]: 1_000, + ["memory_bytes"]: 2 * 1024 ** 3, + ["ephemeral_storage_bytes"]: 5 * 1024 ** 3, + }, + ["max_workers"]: 4, + ["rows_per_worker"]: 100_000, + ["fragments_per_worker"]: 8, + }, + ["retry_policy"]: { + ["max_attempts"]: 3, + ["initial_backoff_seconds"]: 0.2, + ["max_backoff_seconds"]: 30, + ["exception_types"]: [], + ["message_pattern"]: null, + ["error_codes"]: [], + transient: true, + ["honor_retry_after"]: true, + }, + }, + ]); + }); + it("surfaces JSON server errors from remote table operations", async () => { await withMockDatabase( (req, res) => { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 2c45dbccd..e97b32e78 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -5716,6 +5716,31 @@ describe("computed columns", () => { resourceBudget: { cpuMillis: 1_000, memoryBytes: 1_024 }, }), ).rejects.toThrow("resource_budget requires a LanceDB Enterprise refresh"); + await expect( + table.refreshColumn("x", { + supervisorResources: { + cpuMillis: 1_000, + memoryBytes: 1_024, + ephemeralStorageBytes: 2_048, + }, + }), + ).rejects.toThrow( + "supervisor_resources requires a LanceDB Enterprise refresh", + ); + await expect( + table.refreshColumnAsync("x", { + planningWorkers: { + resources: { + cpuMillis: 1_000, + memoryBytes: 1_024, + ephemeralStorageBytes: 2_048, + }, + maxWorkers: 2, + rowsPerWorker: 100, + fragmentsPerWorker: 1, + }, + }), + ).rejects.toThrow("planning_workers requires a LanceDB Enterprise refresh"); }); it("records Function errors only on remote tables", async () => { diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 08a254bdb..a6bcd68f5 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -57,6 +57,9 @@ export { AddResult, AddColumnsResult, RefreshColumnOptions, + FunctionRefreshPlanningConfig, + FunctionRefreshPodResources, + FunctionRetryPolicy, FunctionWorkerResourceBudget, RefreshColumnResult, FunctionErrors, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 136eb1d6c..f949bd00a 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -831,12 +831,18 @@ export abstract class Table { * `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows, * `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds` * and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`, - * `maxConcurrency`, `resourceBudget` and - * `maxExecutorProcessesPerWorker` size registered Function refreshes. + * `maxConcurrency`, `resourceBudget`, `supervisorResources`, + * `planningWorkers` and `maxExecutorProcessesPerWorker` size registered + * Function refreshes. `retryPolicy` overrides user-code retries only when + * repeating partial work is safe; set `maxAttempts: 1` to disable retries. * `maxConcurrency` overrides the registered default for this logical * execution only and is a positive integer; it is not an account-wide * quota. A `resourceBudget` requires `cpuMillis` and * `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function. + * `supervisorResources` and each `planningWorkers.resources` entry require + * positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`. + * `planningWorkers` also requires positive `maxWorkers`, + * `rowsPerWorker` and `fragmentsPerWorker`. * The budget applies only to this explicit refresh and does not configure * a later automatic refresh. * `gpuCount` at the top level is the shorthand that preserves the diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index c82c2cd93..a0a220e24 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -9,6 +9,8 @@ use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; use lancedb::table::{ AddDataMode, ColumnAlteration as LanceColumnAlteration, FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken, + FunctionRefreshPlanningConfig as LanceFunctionRefreshPlanningConfig, + FunctionRefreshPodResources as LanceFunctionRefreshPodResources, FunctionWorkerResourceBudget as LanceFunctionWorkerResourceBudget, NewColumnTransform, Ref, Table as LanceDbTable, }; @@ -1705,6 +1707,58 @@ pub struct FunctionWorkerResourceBudget { pub gpu_count: Option, } +/// CPU, memory, and ephemeral-storage limits for one Function refresh +/// control-plane pod. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct FunctionRefreshPodResources { + /// CPU limit in millicores; 1000 is one CPU. + pub cpu_millis: f64, + /// Memory limit in bytes. + pub memory_bytes: f64, + /// Ephemeral storage limit in bytes. + pub ephemeral_storage_bytes: f64, +} + +/// Resources and sizing policy for the planning fleet of one registered +/// Function refresh. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct FunctionRefreshPlanningConfig { + /// CPU, memory, and ephemeral-storage limits for one planning worker. + pub resources: FunctionRefreshPodResources, + /// Maximum number of planning workers. + pub max_workers: f64, + /// Maximum rows assigned to one planning worker. + pub rows_per_worker: f64, + /// Maximum fragments assigned to one planning worker. + pub fragments_per_worker: f64, +} + +/// Overrides retries of failed user-code calls for one refresh. Omitted +/// fields use the Function refresh defaults. +#[napi(object)] +#[derive(Clone, Debug, Default)] +pub struct FunctionRetryPolicy { + /// Total calls, including the first; 1 disables retries, maximum 100. + pub max_attempts: Option, + /// Initial exponential-backoff ceiling, in seconds. + pub initial_backoff_seconds: Option, + /// Maximum backoff, in seconds. + pub max_backoff_seconds: Option, + /// Exception class names, preferably qualified. + pub exception_types: Option>, + /// Regular expression matched against the exception class and message. + pub message_pattern: Option, + /// Structured provider error codes or HTTP status codes. + pub error_codes: Option>, + /// Match connection/timeouts, HTTP 429/503, SlowDown and + /// ServiceUnavailable. + pub transient: Option, + /// Honor Retry-After within the configured maximum. + pub honor_retry_after: Option, +} + fn positive_safe_integer(value: f64, name: &str) -> napi::Result { const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0; if !value.is_finite() || value.fract() != 0.0 || !(1.0..=MAX_SAFE_INTEGER).contains(&value) { @@ -1768,8 +1822,14 @@ pub struct RefreshColumnOptions { /// Complete aggregate CPU and memory budget for a registered Function /// refresh. `gpuCount` inside the budget defaults to zero. pub resource_budget: Option, + /// CPU, memory, and ephemeral-storage limits for the refresh supervisor. + pub supervisor_resources: Option, + /// Resources and sizing for the refresh planning fleet. + pub planning_workers: Option, /// Positive aggregate GPU budget for Function workers. pub gpu_count: Option, + /// Override retries of failed user-code calls for this refresh. + pub retry_policy: Option, /// `"fail"` (default) stops on a row error; `"skip"` writes null and /// records the error, subject to the failure limits. pub error_policy: Option, @@ -1814,7 +1874,87 @@ impl TryFrom for lancedb::table::RefreshColumnOptions { ) }) .transpose()?; + options.supervisor_resources = value + .supervisor_resources + .map(|resources| { + Ok::<_, napi::Error>(LanceFunctionRefreshPodResources::new( + positive_safe_integer(resources.cpu_millis, "supervisorResources.cpuMillis")?, + positive_safe_integer( + resources.memory_bytes, + "supervisorResources.memoryBytes", + )?, + positive_safe_integer( + resources.ephemeral_storage_bytes, + "supervisorResources.ephemeralStorageBytes", + )?, + )) + }) + .transpose()?; + options.planning_workers = value + .planning_workers + .map(|planning| { + let resources = LanceFunctionRefreshPodResources::new( + positive_safe_integer( + planning.resources.cpu_millis, + "planningWorkers.resources.cpuMillis", + )?, + positive_safe_integer( + planning.resources.memory_bytes, + "planningWorkers.resources.memoryBytes", + )?, + positive_safe_integer( + planning.resources.ephemeral_storage_bytes, + "planningWorkers.resources.ephemeralStorageBytes", + )?, + ); + let max_workers = usize::try_from(positive_safe_integer( + planning.max_workers, + "planningWorkers.maxWorkers", + )?) + .map_err(|_| napi::Error::from_reason("planningWorkers.maxWorkers is too large"))?; + Ok::<_, napi::Error>(LanceFunctionRefreshPlanningConfig::new( + resources, + max_workers, + positive_safe_integer( + planning.rows_per_worker, + "planningWorkers.rowsPerWorker", + )?, + positive_safe_integer( + planning.fragments_per_worker, + "planningWorkers.fragmentsPerWorker", + )?, + )) + }) + .transpose()?; options.gpu_count = value.gpu_count; + options.retry_policy = value.retry_policy.map(|policy| { + let mut converted = lancedb::function::FunctionRetryPolicy::default(); + if let Some(max_attempts) = policy.max_attempts { + converted.max_attempts = max_attempts; + } + if let Some(seconds) = policy.initial_backoff_seconds { + converted.initial_backoff_seconds = seconds; + } + if let Some(seconds) = policy.max_backoff_seconds { + converted.max_backoff_seconds = seconds; + } + if let Some(exception_types) = policy.exception_types { + converted.exception_types = exception_types; + } + if let Some(pattern) = policy.message_pattern { + converted.message_pattern = Some(pattern); + } + if let Some(error_codes) = policy.error_codes { + converted.error_codes = error_codes; + } + if let Some(transient) = policy.transient { + converted.transient = transient; + } + if let Some(honor_retry_after) = policy.honor_retry_after { + converted.honor_retry_after = honor_retry_after; + } + converted + }); options.error_policy = match value.error_policy.as_deref() { None => None, Some("fail") => Some(lancedb::function::FunctionErrorPolicy::Fail), diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 1e811d865..cce3efeef 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -663,6 +663,8 @@ class Table: max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[dict[str, int]] = None, + supervisor_resources: Optional[dict[str, int]] = None, + planning_workers: Optional[dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[str] = None, @@ -683,6 +685,8 @@ class Table: max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[dict[str, int]] = None, + supervisor_resources: Optional[dict[str, int]] = None, + planning_workers: Optional[dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[str] = None, diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index dbfd127ee..b5d97eb62 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -1164,6 +1164,8 @@ class RemoteTable(Table): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -1183,6 +1185,8 @@ class RemoteTable(Table): max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=retry_policy, @@ -1205,6 +1209,8 @@ class RemoteTable(Table): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -1225,6 +1231,8 @@ class RemoteTable(Table): max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=retry_policy, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index e0b96e476..45b2eb4f6 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -3258,6 +3258,8 @@ class Table(ABC): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -3315,6 +3317,17 @@ class Table(ABC): CPU-only budget. This budget cannot be combined with the top-level ``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes reject it. + supervisor_resources: dict[str, int], optional + CPU, memory, and ephemeral-storage limits for the Function refresh + supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and + ``ephemeral_storage_bytes``; all are required and positive. + LanceDB Enterprise only; local refreshes reject it. + planning_workers: dict[str, object], optional + Resources and sizing for the Function refresh planning fleet. + ``resources`` has the same keys as ``supervisor_resources``; the + other keys are ``max_workers``, ``rows_per_worker`` and + ``fragments_per_worker``. LanceDB Enterprise only; local refreshes + reject it. gpu_class: str, optional GPU class for Function workers. Must be a lowercase name of 1-63 letters, digits, or hyphens, without a leading or trailing hyphen. @@ -3367,6 +3380,8 @@ class Table(ABC): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -3423,6 +3438,17 @@ class Table(ABC): CPU-only budget. This budget cannot be combined with the top-level ``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes reject it. + supervisor_resources: dict[str, int], optional + CPU, memory, and ephemeral-storage limits for the Function refresh + supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and + ``ephemeral_storage_bytes``; all are required and positive. + LanceDB Enterprise only; local refreshes reject it. + planning_workers: dict[str, object], optional + Resources and sizing for the Function refresh planning fleet. + ``resources`` has the same keys as ``supervisor_resources``; the + other keys are ``max_workers``, ``rows_per_worker`` and + ``fragments_per_worker``. LanceDB Enterprise only; local refreshes + reject it. gpu_class: str, optional GPU class for Function workers. Must be a lowercase name of 1-63 letters, digits, or hyphens, without a leading or trailing hyphen. @@ -5632,6 +5658,8 @@ class LanceTable(Table): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -5654,6 +5682,8 @@ class LanceTable(Table): max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=retry_policy, @@ -5676,6 +5706,8 @@ class LanceTable(Table): max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -5700,6 +5732,8 @@ class LanceTable(Table): max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=retry_policy, @@ -8109,6 +8143,8 @@ class AsyncTable: max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -8166,6 +8202,17 @@ class AsyncTable: CPU-only budget. This budget cannot be combined with the top-level ``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes reject it. + supervisor_resources: dict[str, int], optional + CPU, memory, and ephemeral-storage limits for the Function refresh + supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and + ``ephemeral_storage_bytes``; all are required and positive. + LanceDB Enterprise only; local refreshes reject it. + planning_workers: dict[str, object], optional + Resources and sizing for the Function refresh planning fleet. + ``resources`` has the same keys as ``supervisor_resources``; the + other keys are ``max_workers``, ``rows_per_worker`` and + ``fragments_per_worker``. LanceDB Enterprise only; local refreshes + reject it. gpu_class: str, optional GPU class for Function workers. Must be a lowercase name of 1-63 letters, digits, or hyphens, without a leading or trailing hyphen. @@ -8213,6 +8260,8 @@ class AsyncTable: max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=None @@ -8351,6 +8400,8 @@ class AsyncTable: max_concurrency: Optional[int] = None, publication_interval_rows: Optional[int] = None, resource_budget: Optional[Dict[str, int]] = None, + supervisor_resources: Optional[Dict[str, int]] = None, + planning_workers: Optional[Dict[str, Any]] = None, gpu_class: Optional[str] = None, gpu_count: Optional[int] = None, retry_policy: Optional[FunctionRetryPolicy] = None, @@ -8407,6 +8458,17 @@ class AsyncTable: CPU-only budget. This budget cannot be combined with the top-level ``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes reject it. + supervisor_resources: dict[str, int], optional + CPU, memory, and ephemeral-storage limits for the Function refresh + supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and + ``ephemeral_storage_bytes``; all are required and positive. + LanceDB Enterprise only; local refreshes reject it. + planning_workers: dict[str, object], optional + Resources and sizing for the Function refresh planning fleet. + ``resources`` has the same keys as ``supervisor_resources``; the + other keys are ``max_workers``, ``rows_per_worker`` and + ``fragments_per_worker``. LanceDB Enterprise only; local refreshes + reject it. gpu_class: str, optional GPU class for Function workers. Must be a lowercase name of 1-63 letters, digits, or hyphens, without a leading or trailing hyphen. @@ -8471,6 +8533,8 @@ class AsyncTable: max_concurrency=max_concurrency, publication_interval_rows=publication_interval_rows, resource_budget=resource_budget, + supervisor_resources=supervisor_resources, + planning_workers=planning_workers, gpu_class=gpu_class, gpu_count=gpu_count, retry_policy=None diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index c5afac308..fc5b5df73 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2076,6 +2076,23 @@ def test_remote_create_index_async_returns_job(): "gpu_count": 2, } }, + { + "supervisor_resources": { + "cpu_millis": 2000, + "memory_bytes": 4 * 1024**3, + "ephemeral_storage_bytes": 10 * 1024**3, + }, + "planning_workers": { + "resources": { + "cpu_millis": 1000, + "memory_bytes": 2 * 1024**3, + "ephemeral_storage_bytes": 5 * 1024**3, + }, + "max_workers": 4, + "rows_per_worker": 100_000, + "fragments_per_worker": 8, + }, + }, {"gpu_class": "a10g", "gpu_count": 2}, { "retry_policy": {"transient": True, "max_attempts": 3}, @@ -2122,6 +2139,9 @@ async def test_remote_refresh_async_returns_typed_terminal_result( **budget, "gpu_count": budget.get("gpu_count", 0), } + for key in ("supervisor_resources", "planning_workers"): + if refresh_kwargs.get(key) is not None: + expected[key] = refresh_kwargs[key] if refresh_kwargs.get("retry_policy") is not None: expected["retry_policy"] = lancedb.FunctionRetryPolicy.model_validate( refresh_kwargs["retry_policy"] @@ -2350,6 +2370,51 @@ def test_remote_refresh_async_forwards_progress_from_submitted_handle(): }, "unsupported key 'unknown'", ), + ( + { + "supervisor_resources": { + "cpu_millis": 0, + "memory_bytes": 1, + "ephemeral_storage_bytes": 1, + } + }, + "supervisor_resources.cpu_millis", + ), + ( + { + "supervisor_resources": { + "cpu_millis": 1, + "memory_bytes": 1, + } + }, + "supervisor_resources.ephemeral_storage_bytes", + ), + ( + { + "supervisor_resources": { + "cpu_millis": 1, + "memory_bytes": 1, + "ephemeral_storage_bytes": 1, + "unknown": 1, + } + }, + "unsupported key 'unknown'", + ), + ( + { + "planning_workers": { + "resources": { + "cpu_millis": 1, + "memory_bytes": 1, + "ephemeral_storage_bytes": 1, + }, + "max_workers": 0, + "rows_per_worker": 1, + "fragments_per_worker": 1, + } + }, + "planning_workers.max_workers", + ), ( {"max_executor_processes_per_worker": -2}, "max_executor_processes_per_worker", diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 604d6afb3..ce4f57c48 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -6013,6 +6013,31 @@ def test_refresh_column_rejects_invalid_batch_size(tmp_path, method, batch_size) {"resource_budget": {"cpu_millis": 1000, "memory_bytes": 1024}}, "resource_budget", ), + ( + { + "supervisor_resources": { + "cpu_millis": 1000, + "memory_bytes": 1024, + "ephemeral_storage_bytes": 2048, + } + }, + "supervisor_resources", + ), + ( + { + "planning_workers": { + "resources": { + "cpu_millis": 1000, + "memory_bytes": 1024, + "ephemeral_storage_bytes": 2048, + }, + "max_workers": 2, + "rows_per_worker": 100, + "fragments_per_worker": 1, + } + }, + "planning_workers", + ), ({"num_workers": 2}, "num_workers"), ({"max_executor_processes_per_worker": 2}, "max_executor_processes_per_worker"), ], diff --git a/python/src/table.rs b/python/src/table.rs index f77ff5a7f..f5270a594 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -30,8 +30,9 @@ use lancedb::connection::LanceFileVersion; use lancedb::index::scalar::FtsIndexBuilder; use lancedb::table::{ AddDataMode, ColumnAlteration, CompactionMode, CompactionOptions, Duration, - FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionWorkerResourceBudget, IndexRemapMode, - NewColumnTransform, Ref, RefreshColumnOptions, Table as LanceDbTable, + FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionRefreshPlanningConfig, + FunctionRefreshPodResources, FunctionWorkerResourceBudget, IndexRemapMode, NewColumnTransform, + Ref, RefreshColumnOptions, Table as LanceDbTable, }; use lancedb::tokenize as lancedb_tokenize; use pyo3::{ @@ -52,6 +53,8 @@ struct RefreshColumnArgs { max_concurrency: Option, publication_interval_rows: Option, resource_budget: Option, + supervisor_resources: Option, + planning_workers: Option, gpu_class: Option, gpu_count: Option, retry_policy: Option, @@ -159,6 +162,101 @@ fn parse_resource_budget( )) } +fn structured_integer(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + if value.is_none() || value.is_instance_of::() { + return Err(PyValueError::new_err(format!("{name} must be an integer"))); + } + let parsed = value + .extract::() + .map_err(|_| PyValueError::new_err(format!("{name} must be an integer")))?; + if parsed == 0 { + return Err(PyValueError::new_err(format!( + "{name} must be greater than zero" + ))); + } + Ok(parsed) +} + +fn parse_pod_resources( + resources: &Bound<'_, PyDict>, + prefix: &str, +) -> PyResult { + for key in resources.keys() { + let key = key + .extract::() + .map_err(|_| PyValueError::new_err(format!("{prefix} keys must be strings")))?; + if !matches!( + key.as_str(), + "cpu_millis" | "memory_bytes" | "ephemeral_storage_bytes" + ) { + return Err(PyValueError::new_err(format!( + "{prefix} contains unsupported key '{key}'" + ))); + } + } + let get = |name: &str| -> PyResult { + let value = resources + .get_item(name)? + .ok_or_else(|| PyValueError::new_err(format!("{prefix}.{name} is required")))?; + structured_integer(&value, &format!("{prefix}.{name}")) + }; + Ok(FunctionRefreshPodResources::new( + get("cpu_millis")?, + get("memory_bytes")?, + get("ephemeral_storage_bytes")?, + )) +} + +fn parse_supervisor_resources( + resources: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + resources + .map(|resources| parse_pod_resources(resources, "supervisor_resources")) + .transpose() +} + +fn parse_planning_workers( + planning: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let Some(planning) = planning else { + return Ok(None); + }; + for key in planning.keys() { + let key = key + .extract::() + .map_err(|_| PyValueError::new_err("planning_workers keys must be strings"))?; + if !matches!( + key.as_str(), + "resources" | "max_workers" | "rows_per_worker" | "fragments_per_worker" + ) { + return Err(PyValueError::new_err(format!( + "planning_workers contains unsupported key '{key}'" + ))); + } + } + let resources = planning + .get_item("resources")? + .ok_or_else(|| PyValueError::new_err("planning_workers.resources is required"))?; + let resources = resources + .cast::() + .map_err(|_| PyValueError::new_err("planning_workers.resources must be a dict"))?; + let resources = parse_pod_resources(resources, "planning_workers.resources")?; + let get = |name: &str| -> PyResult { + let value = planning + .get_item(name)? + .ok_or_else(|| PyValueError::new_err(format!("planning_workers.{name} is required")))?; + structured_integer(&value, &format!("planning_workers.{name}")) + }; + let max_workers = usize::try_from(get("max_workers")?) + .map_err(|_| PyValueError::new_err("planning_workers.max_workers is too large"))?; + Ok(Some(FunctionRefreshPlanningConfig::new( + resources, + max_workers, + get("rows_per_worker")?, + get("fragments_per_worker")?, + ))) +} + fn refresh_column_options(args: RefreshColumnArgs) -> PyResult { let RefreshColumnArgs { where_clause, @@ -166,6 +264,8 @@ fn refresh_column_options(args: RefreshColumnArgs) -> PyResult PyResult( self_: PyRef<'py, Self>, @@ -2502,6 +2604,8 @@ impl Table { max_concurrency: Option>, publication_interval_rows: Option, resource_budget: Option>, + supervisor_resources: Option>, + planning_workers: Option>, gpu_class: Option, gpu_count: Option, retry_policy: Option, @@ -2519,6 +2623,8 @@ impl Table { max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?, publication_interval_rows, resource_budget: parse_resource_budget(resource_budget.as_ref())?, + supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?, + planning_workers: parse_planning_workers(planning_workers.as_ref())?, gpu_class, gpu_count, retry_policy, @@ -2540,7 +2646,7 @@ impl Table { }) } - #[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))] + #[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, supervisor_resources=None, planning_workers=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))] #[allow(clippy::too_many_arguments)] pub fn refresh_column_async<'py>( self_: PyRef<'py, Self>, @@ -2550,6 +2656,8 @@ impl Table { max_concurrency: Option>, publication_interval_rows: Option, resource_budget: Option>, + supervisor_resources: Option>, + planning_workers: Option>, gpu_class: Option, gpu_count: Option, retry_policy: Option, @@ -2567,6 +2675,8 @@ impl Table { max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?, publication_interval_rows, resource_budget: parse_resource_budget(resource_budget.as_ref())?, + supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?, + planning_workers: parse_planning_workers(planning_workers.as_ref())?, gpu_class, gpu_count, retry_policy, diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 4e9458870..e556d131c 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -11560,6 +11560,59 @@ mod tests { assert_eq!(job.id(), Some("j-budget")); } + #[tokio::test] + async fn test_refresh_column_async_structured_resource_options() { + use crate::table::{ + FunctionRefreshPlanningConfig, FunctionRefreshPodResources, RefreshColumnOptions, + }; + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/backfill_column"); + assert_eq!( + request_body_json(&request), + serde_json::json!({ + "column": "quality", + "supervisor_resources": { + "cpu_millis": 2_000, + "memory_bytes": 4_294_967_296_u64, + "ephemeral_storage_bytes": 10_737_418_240_u64, + }, + "planning_workers": { + "resources": { + "cpu_millis": 1_000, + "memory_bytes": 2_147_483_648_u64, + "ephemeral_storage_bytes": 5_368_709_120_u64, + }, + "max_workers": 4, + "rows_per_worker": 100_000, + "fragments_per_worker": 8, + }, + }) + ); + http::Response::builder() + .status(202) + .body(r#"{"job_id":"j-structured"}"#) + .unwrap() + }); + let supervisor = FunctionRefreshPodResources::new(2_000, 4_294_967_296, 10_737_418_240); + let planning = FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new(1_000, 2_147_483_648, 5_368_709_120), + 4, + 100_000, + 8, + ); + let job = table + .refresh_column_async_with_options( + "quality", + RefreshColumnOptions::default() + .supervisor_resources(supervisor) + .planning_workers(planning), + ) + .await + .unwrap(); + assert_eq!(job.id(), Some("j-structured")); + } + #[tokio::test] async fn test_refresh_column_async_retry_policy() { use crate::function::{FunctionErrorPolicy, FunctionRetryPolicy}; @@ -11640,7 +11693,9 @@ mod tests { #[tokio::test] async fn test_refresh_column_rejects_invalid_filter_and_limit_options_before_request() { - use crate::table::RefreshColumnOptions; + use crate::table::{ + FunctionRefreshPlanningConfig, FunctionRefreshPodResources, RefreshColumnOptions, + }; let table = Table::new_with_handler("my_table", |_| -> http::Response { panic!("invalid refresh options must not send a request") @@ -11679,6 +11734,55 @@ mod tests { RefreshColumnOptions::default().publication_interval_rows(0), "publication_interval_rows", ), + ( + RefreshColumnOptions::default() + .supervisor_resources(FunctionRefreshPodResources::new(0, 1, 1)), + "supervisor_resources.cpu_millis", + ), + ( + RefreshColumnOptions::default().planning_workers( + FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new(1, 1, 0), + 1, + 1, + 1, + ), + ), + "planning_workers.resources.ephemeral_storage_bytes", + ), + ( + RefreshColumnOptions::default().planning_workers( + FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new(1, 1, 1), + 0, + 1, + 1, + ), + ), + "planning_workers.max_workers", + ), + ( + RefreshColumnOptions::default().planning_workers( + FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new(1, 1, 1), + 1, + 0, + 1, + ), + ), + "planning_workers.rows_per_worker", + ), + ( + RefreshColumnOptions::default().planning_workers( + FunctionRefreshPlanningConfig::new( + FunctionRefreshPodResources::new(1, 1, 1), + 1, + 1, + 0, + ), + ), + "planning_workers.fragments_per_worker", + ), ] { let error = table .refresh_column_async_with_options("quality", options) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index b9708f16b..00505220b 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -120,7 +120,10 @@ pub use optimize::{ CompactionMode, CompactionOptions, CompactionOverrides, IndexRemapMode, OptimizeAction, OptimizeStats, }; -pub use refresh::{FunctionWorkerResourceBudget, RefreshColumnOptions, RefreshColumnResult}; +pub use refresh::{ + FunctionRefreshPlanningConfig, FunctionRefreshPodResources, FunctionWorkerResourceBudget, + RefreshColumnOptions, RefreshColumnResult, +}; pub use rte_stats::{MemtableStats, RTEStats, SsTableStats, TableShardStats}; pub use schema_evolution::{ AddColumnsResult, AlterColumnsResult, DropColumnsResult, FieldMetadataUpdate, diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index ff73e6366..f38509002 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -104,6 +104,106 @@ impl FunctionWorkerResourceBudget { } } +/// CPU, memory, and ephemeral-storage limits for one Function refresh +/// control-plane pod. +/// +/// This sizes the supervisor or planning worker pod for a registered +/// Function refresh. LanceDB Embedded rejects it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +pub struct FunctionRefreshPodResources { + /// CPU limit in millicores; 1000 is one CPU. + pub cpu_millis: u64, + /// Memory limit in bytes. + pub memory_bytes: u64, + /// Ephemeral storage limit in bytes. + pub ephemeral_storage_bytes: u64, +} + +impl FunctionRefreshPodResources { + /// Create positive CPU, memory, and ephemeral-storage limits. + pub fn new(cpu_millis: u64, memory_bytes: u64, ephemeral_storage_bytes: u64) -> Self { + Self { + cpu_millis, + memory_bytes, + ephemeral_storage_bytes, + } + } + + fn validate(&self, prefix: &str) -> Result<()> { + for (name, value) in [ + ("cpu_millis", self.cpu_millis), + ("memory_bytes", self.memory_bytes), + ("ephemeral_storage_bytes", self.ephemeral_storage_bytes), + ] { + if value == 0 { + return Err(Error::InvalidInput { + message: format!("{prefix}.{name} must be greater than zero"), + }); + } + } + Ok(()) + } +} + +/// Resources and sizing policy for the CPU-only planning fleet of one +/// registered Function refresh. +/// +/// `resources` describes one planning worker pod. The remaining fields bound +/// how many workers may run and how much work each worker receives. LanceDB +/// Embedded rejects this option. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +pub struct FunctionRefreshPlanningConfig { + /// CPU, memory, and ephemeral-storage limits for one planning worker. + pub resources: FunctionRefreshPodResources, + /// Maximum number of planning workers. + pub max_workers: usize, + /// Maximum rows assigned to one planning worker. + pub rows_per_worker: u64, + /// Maximum fragments assigned to one planning worker. + pub fragments_per_worker: u64, +} + +impl FunctionRefreshPlanningConfig { + /// Create a planning configuration; every value must be positive. + pub fn new( + resources: FunctionRefreshPodResources, + max_workers: usize, + rows_per_worker: u64, + fragments_per_worker: u64, + ) -> Self { + Self { + resources, + max_workers, + rows_per_worker, + fragments_per_worker, + } + } + + fn validate(&self) -> Result<()> { + self.resources.validate("planning_workers.resources")?; + if self.max_workers == 0 { + return Err(Error::InvalidInput { + message: "planning_workers.max_workers must be greater than zero".into(), + }); + } + if self.rows_per_worker == 0 { + return Err(Error::InvalidInput { + message: "planning_workers.rows_per_worker must be greater than zero".into(), + }); + } + if self.fragments_per_worker == 0 { + return Err(Error::InvalidInput { + message: "planning_workers.fragments_per_worker must be greater than zero".into(), + }); + } + Ok(()) + } +} + /// Options for refreshing a computed column. /// /// Serializes to the Enterprise backfill request body, one key per set @@ -167,6 +267,14 @@ pub struct RefreshColumnOptions { /// Function refresh. Cannot be combined with `gpu_count`. #[serde(skip_serializing_if = "Option::is_none")] pub resource_budget: Option, + /// CPU, memory, and ephemeral-storage limits for the Function refresh + /// supervisor pod. Enterprise only. + #[serde(skip_serializing_if = "Option::is_none")] + pub supervisor_resources: Option, + /// Resources and sizing limits for the Function refresh planning fleet. + /// Enterprise only. + #[serde(skip_serializing_if = "Option::is_none")] + pub planning_workers: Option, /// GPU class for Function workers in LanceDB Enterprise. /// /// `None` keeps the Function's registered class, or the deployment default @@ -264,6 +372,18 @@ impl RefreshColumnOptions { self } + /// Set the Function refresh supervisor pod resources. + pub fn supervisor_resources(mut self, resources: FunctionRefreshPodResources) -> Self { + self.supervisor_resources = Some(resources); + self + } + + /// Set the Function refresh planning fleet resources and sizing. + pub fn planning_workers(mut self, planning: FunctionRefreshPlanningConfig) -> Self { + self.planning_workers = Some(planning); + self + } + /// Request Function workers from `gpu_class`. pub fn gpu_class(mut self, gpu_class: impl Into) -> Self { self.gpu_class = Some(gpu_class.into()); @@ -297,6 +417,8 @@ impl RefreshColumnOptions { self.publication_interval_rows.is_some(), ), ("resource_budget", self.resource_budget.is_some()), + ("supervisor_resources", self.supervisor_resources.is_some()), + ("planning_workers", self.planning_workers.is_some()), ] .into_iter() .find_map(|(name, set)| set.then_some(name)) @@ -367,6 +489,12 @@ impl RefreshColumnOptions { if let Some(budget) = &self.resource_budget { budget.validate()?; } + if let Some(resources) = &self.supervisor_resources { + resources.validate("supervisor_resources")?; + } + if let Some(planning) = &self.planning_workers { + planning.validate()?; + } if let Some(gpu_class) = &self.gpu_class { validate_gpu_class_name(gpu_class)?; }