diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md
index 3150d94ec..f93c1b7cd 100644
--- a/docs/src/js/classes/Table.md
+++ b/docs/src/js/classes/Table.md
@@ -1225,12 +1225,18 @@ LanceDB Enterprise it is the server's backfill job.
`REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
`errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
- `maxConcurrency`, `resourceBudget` and
- `maxExecutorProcessesPerWorker` size registered Function refreshes.
+ `maxConcurrency`, `resourceBudget`, `supervisorResources`,
+ `planningWorkers` and `maxExecutorProcessesPerWorker` size registered
+ Function refreshes. `retryPolicy` overrides user-code retries only when
+ repeating partial work is safe; set `maxAttempts: 1` to disable retries.
`maxConcurrency` overrides the registered default for this logical
execution only and is a positive integer; it is not an account-wide
quota. A `resourceBudget` requires `cpuMillis` and
`memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
+ `supervisorResources` and each `planningWorkers.resources` entry require
+ positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`.
+ `planningWorkers` also requires positive `maxWorkers`,
+ `rowsPerWorker` and `fragmentsPerWorker`.
The budget applies only to this explicit refresh and does not configure
a later automatic refresh.
`gpuCount` at the top level is the shorthand that preserves the
diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md
index beed7c8af..23e576877 100644
--- a/docs/src/js/globals.md
+++ b/docs/src/js/globals.md
@@ -103,6 +103,9 @@
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
+- [FunctionRefreshPlanningConfig](interfaces/FunctionRefreshPlanningConfig.md)
+- [FunctionRefreshPodResources](interfaces/FunctionRefreshPodResources.md)
+- [FunctionRetryPolicy](interfaces/FunctionRetryPolicy.md)
- [FunctionWorkerResourceBudget](interfaces/FunctionWorkerResourceBudget.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
diff --git a/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md b/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md
new file mode 100644
index 000000000..48a8a209d
--- /dev/null
+++ b/docs/src/js/interfaces/FunctionRefreshPlanningConfig.md
@@ -0,0 +1,50 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FunctionRefreshPlanningConfig
+
+# Interface: FunctionRefreshPlanningConfig
+
+Resources and sizing policy for the planning fleet of one registered
+Function refresh.
+
+## Properties
+
+### fragmentsPerWorker
+
+```ts
+fragmentsPerWorker: number;
+```
+
+Maximum fragments assigned to one planning worker.
+
+***
+
+### maxWorkers
+
+```ts
+maxWorkers: number;
+```
+
+Maximum number of planning workers.
+
+***
+
+### resources
+
+```ts
+resources: FunctionRefreshPodResources;
+```
+
+CPU, memory, and ephemeral-storage limits for one planning worker.
+
+***
+
+### rowsPerWorker
+
+```ts
+rowsPerWorker: number;
+```
+
+Maximum rows assigned to one planning worker.
diff --git a/docs/src/js/interfaces/FunctionRefreshPodResources.md b/docs/src/js/interfaces/FunctionRefreshPodResources.md
new file mode 100644
index 000000000..99d55512e
--- /dev/null
+++ b/docs/src/js/interfaces/FunctionRefreshPodResources.md
@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FunctionRefreshPodResources
+
+# Interface: FunctionRefreshPodResources
+
+CPU, memory, and ephemeral-storage limits for one Function refresh
+control-plane pod.
+
+## Properties
+
+### cpuMillis
+
+```ts
+cpuMillis: number;
+```
+
+CPU limit in millicores; 1000 is one CPU.
+
+***
+
+### ephemeralStorageBytes
+
+```ts
+ephemeralStorageBytes: number;
+```
+
+Ephemeral storage limit in bytes.
+
+***
+
+### memoryBytes
+
+```ts
+memoryBytes: number;
+```
+
+Memory limit in bytes.
diff --git a/docs/src/js/interfaces/FunctionRetryPolicy.md b/docs/src/js/interfaces/FunctionRetryPolicy.md
new file mode 100644
index 000000000..43d4ed8b2
--- /dev/null
+++ b/docs/src/js/interfaces/FunctionRetryPolicy.md
@@ -0,0 +1,91 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FunctionRetryPolicy
+
+# Interface: FunctionRetryPolicy
+
+Overrides retries of failed user-code calls for one refresh. Omitted
+fields use the Function refresh defaults.
+
+## Properties
+
+### errorCodes?
+
+```ts
+optional errorCodes: string[];
+```
+
+Structured provider error codes or HTTP status codes.
+
+***
+
+### exceptionTypes?
+
+```ts
+optional exceptionTypes: string[];
+```
+
+Exception class names, preferably qualified.
+
+***
+
+### honorRetryAfter?
+
+```ts
+optional honorRetryAfter: boolean;
+```
+
+Honor Retry-After within the configured maximum.
+
+***
+
+### initialBackoffSeconds?
+
+```ts
+optional initialBackoffSeconds: number;
+```
+
+Initial exponential-backoff ceiling, in seconds.
+
+***
+
+### maxAttempts?
+
+```ts
+optional maxAttempts: number;
+```
+
+Total calls, including the first; 1 disables retries, maximum 100.
+
+***
+
+### maxBackoffSeconds?
+
+```ts
+optional maxBackoffSeconds: number;
+```
+
+Maximum backoff, in seconds.
+
+***
+
+### messagePattern?
+
+```ts
+optional messagePattern: string;
+```
+
+Regular expression matched against the exception class and message.
+
+***
+
+### transient?
+
+```ts
+optional transient: boolean;
+```
+
+Match connection/timeouts, HTTP 429/503, SlowDown and
+ServiceUnavailable.
diff --git a/docs/src/js/interfaces/RefreshColumnOptions.md b/docs/src/js/interfaces/RefreshColumnOptions.md
index c534e0fac..07157e4ab 100644
--- a/docs/src/js/interfaces/RefreshColumnOptions.md
+++ b/docs/src/js/interfaces/RefreshColumnOptions.md
@@ -117,6 +117,16 @@ Worker count for an expression column; Function columns reject it.
***
+### planningWorkers?
+
+```ts
+optional planningWorkers: FunctionRefreshPlanningConfig;
+```
+
+Resources and sizing for the refresh planning fleet.
+
+***
+
### publicationIntervalRows?
```ts
@@ -141,6 +151,16 @@ refresh. `gpuCount` inside the budget defaults to zero.
***
+### retryPolicy?
+
+```ts
+optional retryPolicy: FunctionRetryPolicy;
+```
+
+Override retries of failed user-code calls for this refresh.
+
+***
+
### rowTimeoutSeconds?
```ts
@@ -151,6 +171,16 @@ Deadline for one Function call, in seconds.
***
+### supervisorResources?
+
+```ts
+optional supervisorResources: FunctionRefreshPodResources;
+```
+
+CPU, memory, and ephemeral-storage limits for the refresh supervisor.
+
+***
+
### where?
```ts
diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx
index d82fb1eef..f893118fc 100644
--- a/docs/web/feature-engineering/refresh-column.mdx
+++ b/docs/web/feature-engineering/refresh-column.mdx
@@ -400,6 +400,101 @@ for changing only the aggregate GPU budget. It cannot be combined with the
complete resource budget. Omitting both preserves the server's existing
defaults.
+{/*
+Draft for the Function refresh control-plane resource release.
+
+`resource_budget` above sizes the Function workers. Two independent options
+size the refresh control plane for a registered Function.
+`supervisor_resources` sets the CPU, memory and ephemeral-storage limits of
+the supervisor pod. `planning_workers` sets the resources of one planning
+worker and the sizing of the CPU-only planning fleet: `max_workers`,
+`rows_per_worker` and `fragments_per_worker`. Every field is required and
+must be positive. Omitting either option leaves the server's configured
+defaults unchanged, and either can be combined with `resource_budget`.
+
+
+```python Python icon="python"
+job = table.refresh_column_async(
+ "embedding",
+ supervisor_resources={
+ "cpu_millis": 2_000,
+ "memory_bytes": 4 * 1024**3,
+ "ephemeral_storage_bytes": 10 * 1024**3,
+ },
+ planning_workers={
+ "resources": {
+ "cpu_millis": 1_000,
+ "memory_bytes": 2 * 1024**3,
+ "ephemeral_storage_bytes": 5 * 1024**3,
+ },
+ "max_workers": 4,
+ "rows_per_worker": 100_000,
+ "fragments_per_worker": 8,
+ },
+)
+job.wait()
+```
+
+```rust Rust icon="rust"
+use lancedb::table::{
+ FunctionRefreshPlanningConfig,
+ FunctionRefreshPodResources,
+ RefreshColumnOptions,
+};
+
+let _job = table
+ .refresh_column_async_with_options(
+ "embedding",
+ RefreshColumnOptions::default()
+ .supervisor_resources(
+ FunctionRefreshPodResources::new(
+ 2_000,
+ 4 * 1024 * 1024 * 1024,
+ 10 * 1024 * 1024 * 1024,
+ ),
+ )
+ .planning_workers(
+ FunctionRefreshPlanningConfig::new(
+ FunctionRefreshPodResources::new(
+ 1_000,
+ 2 * 1024 * 1024 * 1024,
+ 5 * 1024 * 1024 * 1024,
+ ),
+ 4,
+ 100_000,
+ 8,
+ ),
+ ),
+ )
+ .await?;
+```
+
+```typescript TypeScript icon="typescript"
+const job = await table.refreshColumnAsync("embedding", {
+ supervisorResources: {
+ cpuMillis: 2_000,
+ memoryBytes: 4 * 1024 ** 3,
+ ephemeralStorageBytes: 10 * 1024 ** 3,
+ },
+ planningWorkers: {
+ resources: {
+ cpuMillis: 1_000,
+ memoryBytes: 2 * 1024 ** 3,
+ ephemeralStorageBytes: 5 * 1024 ** 3,
+ },
+ maxWorkers: 4,
+ rowsPerWorker: 100_000,
+ fragmentsPerWorker: 8,
+ },
+});
+await job.wait();
+```
+
+
+The SDK validates the shape and sends these values only with this explicit
+refresh request. A supporting LanceDB Enterprise server is required.
+*/}
+
These examples configure only the explicit refresh shown. A registered
Function declaration can separately set `auto_refresh_resource_budget` on
`add_columns`; that declaration controls the budget for its automatic
@@ -457,6 +552,39 @@ from `lancedb.functions`. Use `FunctionRetryPolicy(max_attempts=1)` to
disable user-error retries. Omitting the override preserves the registered
policy.
+
+```python Python icon="python"
+from lancedb.functions import FunctionRetryPolicy
+
+job = table.refresh_column_async(
+ "embedding",
+ retry_policy=FunctionRetryPolicy(transient=True, max_attempts=3),
+)
+job.wait()
+```
+
+```rust Rust icon="rust"
+use lancedb::{function::FunctionRetryPolicy, table::RefreshColumnOptions};
+
+let mut retry_policy = FunctionRetryPolicy::transient();
+retry_policy.max_attempts = 3;
+
+let _job = table
+ .refresh_column_async_with_options(
+ "embedding",
+ RefreshColumnOptions::default().retry_policy(retry_policy),
+ )
+ .await?;
+```
+
+```typescript TypeScript icon="typescript"
+const job = await table.refreshColumnAsync("embedding", {
+ retryPolicy: { transient: true, maxAttempts: 3 },
+});
+await job.wait();
+```
+
+
This requires a supporting LanceDB Enterprise server as well as an SDK
that exposes these options. An older Function image may need
re-registration before it accepts a refresh override. Only enable retries
diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts
index a9989b84e..3e8d5a79b 100644
--- a/nodejs/__test__/remote.test.ts
+++ b/nodejs/__test__/remote.test.ts
@@ -979,6 +979,42 @@ describe("remote connection", () => {
},
"resourceBudget.gpuCount",
],
+ [
+ {
+ supervisorResources: {
+ cpuMillis: 0,
+ memoryBytes: 1_024,
+ ephemeralStorageBytes: 2_048,
+ },
+ },
+ "supervisorResources.cpuMillis",
+ ],
+ [
+ {
+ supervisorResources: {
+ cpuMillis: 1_000,
+ memoryBytes: 1_024,
+ ephemeralStorageBytes: 0,
+ },
+ },
+ "supervisorResources.ephemeralStorageBytes",
+ ],
+ [
+ {
+ planningWorkers: {
+ resources: {
+ cpuMillis: 1_000,
+ memoryBytes: 1_024,
+ ephemeralStorageBytes: 2_048,
+ },
+ maxWorkers: 0,
+ rowsPerWorker: 100,
+ fragmentsPerWorker: 1,
+ },
+ },
+ "planningWorkers.maxWorkers",
+ ],
+ [{ retryPolicy: { maxAttempts: 0 } }, "retry_policy.max_attempts"],
] as const) {
await expect(
table.refreshColumnAsync("embedding", options),
@@ -1016,6 +1052,91 @@ describe("remote connection", () => {
]);
});
+ it("forwards structured Function refresh options and Node retry policy", async () => {
+ const bodies: unknown[] = [];
+ await withMockDatabase(
+ (req, res) => {
+ const path = req.url ?? "";
+ if (path.endsWith("/describe/")) {
+ res.writeHead(200, { "Content-Type": "application/json" }).end(
+ JSON.stringify({
+ name: "docs",
+ version: 1,
+ schema: { fields: [] },
+ }),
+ );
+ return;
+ }
+ if (path === "/v1/table/docs/backfill_column") {
+ let body = "";
+ req.on("data", (chunk) => {
+ body += chunk;
+ });
+ req.on("end", () => {
+ bodies.push(JSON.parse(body));
+ res
+ .writeHead(202, { "Content-Type": "application/json" })
+ .end('{"job_id": "j-structured"}');
+ });
+ return;
+ }
+ res.writeHead(404).end();
+ },
+ async (db) => {
+ const table = await db.openTable("docs");
+ const job = await table.refreshColumnAsync("embedding", {
+ supervisorResources: {
+ cpuMillis: 2_000,
+ memoryBytes: 4 * 1024 ** 3,
+ ephemeralStorageBytes: 10 * 1024 ** 3,
+ },
+ planningWorkers: {
+ resources: {
+ cpuMillis: 1_000,
+ memoryBytes: 2 * 1024 ** 3,
+ ephemeralStorageBytes: 5 * 1024 ** 3,
+ },
+ maxWorkers: 4,
+ rowsPerWorker: 100_000,
+ fragmentsPerWorker: 8,
+ },
+ retryPolicy: { transient: true },
+ });
+ expect(job.id).toBe("j-structured");
+ },
+ );
+ expect(bodies).toEqual([
+ {
+ column: "embedding",
+ ["supervisor_resources"]: {
+ ["cpu_millis"]: 2_000,
+ ["memory_bytes"]: 4 * 1024 ** 3,
+ ["ephemeral_storage_bytes"]: 10 * 1024 ** 3,
+ },
+ ["planning_workers"]: {
+ resources: {
+ ["cpu_millis"]: 1_000,
+ ["memory_bytes"]: 2 * 1024 ** 3,
+ ["ephemeral_storage_bytes"]: 5 * 1024 ** 3,
+ },
+ ["max_workers"]: 4,
+ ["rows_per_worker"]: 100_000,
+ ["fragments_per_worker"]: 8,
+ },
+ ["retry_policy"]: {
+ ["max_attempts"]: 3,
+ ["initial_backoff_seconds"]: 0.2,
+ ["max_backoff_seconds"]: 30,
+ ["exception_types"]: [],
+ ["message_pattern"]: null,
+ ["error_codes"]: [],
+ transient: true,
+ ["honor_retry_after"]: true,
+ },
+ },
+ ]);
+ });
+
it("surfaces JSON server errors from remote table operations", async () => {
await withMockDatabase(
(req, res) => {
diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts
index 2c45dbccd..e97b32e78 100644
--- a/nodejs/__test__/table.test.ts
+++ b/nodejs/__test__/table.test.ts
@@ -5716,6 +5716,31 @@ describe("computed columns", () => {
resourceBudget: { cpuMillis: 1_000, memoryBytes: 1_024 },
}),
).rejects.toThrow("resource_budget requires a LanceDB Enterprise refresh");
+ await expect(
+ table.refreshColumn("x", {
+ supervisorResources: {
+ cpuMillis: 1_000,
+ memoryBytes: 1_024,
+ ephemeralStorageBytes: 2_048,
+ },
+ }),
+ ).rejects.toThrow(
+ "supervisor_resources requires a LanceDB Enterprise refresh",
+ );
+ await expect(
+ table.refreshColumnAsync("x", {
+ planningWorkers: {
+ resources: {
+ cpuMillis: 1_000,
+ memoryBytes: 1_024,
+ ephemeralStorageBytes: 2_048,
+ },
+ maxWorkers: 2,
+ rowsPerWorker: 100,
+ fragmentsPerWorker: 1,
+ },
+ }),
+ ).rejects.toThrow("planning_workers requires a LanceDB Enterprise refresh");
});
it("records Function errors only on remote tables", async () => {
diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts
index 08a254bdb..a6bcd68f5 100644
--- a/nodejs/lancedb/index.ts
+++ b/nodejs/lancedb/index.ts
@@ -57,6 +57,9 @@ export {
AddResult,
AddColumnsResult,
RefreshColumnOptions,
+ FunctionRefreshPlanningConfig,
+ FunctionRefreshPodResources,
+ FunctionRetryPolicy,
FunctionWorkerResourceBudget,
RefreshColumnResult,
FunctionErrors,
diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts
index 136eb1d6c..f949bd00a 100644
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -831,12 +831,18 @@ export abstract class Table {
* `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
* `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
* and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
- * `maxConcurrency`, `resourceBudget` and
- * `maxExecutorProcessesPerWorker` size registered Function refreshes.
+ * `maxConcurrency`, `resourceBudget`, `supervisorResources`,
+ * `planningWorkers` and `maxExecutorProcessesPerWorker` size registered
+ * Function refreshes. `retryPolicy` overrides user-code retries only when
+ * repeating partial work is safe; set `maxAttempts: 1` to disable retries.
* `maxConcurrency` overrides the registered default for this logical
* execution only and is a positive integer; it is not an account-wide
* quota. A `resourceBudget` requires `cpuMillis` and
* `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
+ * `supervisorResources` and each `planningWorkers.resources` entry require
+ * positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`.
+ * `planningWorkers` also requires positive `maxWorkers`,
+ * `rowsPerWorker` and `fragmentsPerWorker`.
* The budget applies only to this explicit refresh and does not configure
* a later automatic refresh.
* `gpuCount` at the top level is the shorthand that preserves the
diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs
index c82c2cd93..a0a220e24 100644
--- a/nodejs/src/table.rs
+++ b/nodejs/src/table.rs
@@ -9,6 +9,8 @@ use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
use lancedb::table::{
AddDataMode, ColumnAlteration as LanceColumnAlteration,
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
+ FunctionRefreshPlanningConfig as LanceFunctionRefreshPlanningConfig,
+ FunctionRefreshPodResources as LanceFunctionRefreshPodResources,
FunctionWorkerResourceBudget as LanceFunctionWorkerResourceBudget, NewColumnTransform, Ref,
Table as LanceDbTable,
};
@@ -1705,6 +1707,58 @@ pub struct FunctionWorkerResourceBudget {
pub gpu_count: Option,
}
+/// CPU, memory, and ephemeral-storage limits for one Function refresh
+/// control-plane pod.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct FunctionRefreshPodResources {
+ /// CPU limit in millicores; 1000 is one CPU.
+ pub cpu_millis: f64,
+ /// Memory limit in bytes.
+ pub memory_bytes: f64,
+ /// Ephemeral storage limit in bytes.
+ pub ephemeral_storage_bytes: f64,
+}
+
+/// Resources and sizing policy for the planning fleet of one registered
+/// Function refresh.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct FunctionRefreshPlanningConfig {
+ /// CPU, memory, and ephemeral-storage limits for one planning worker.
+ pub resources: FunctionRefreshPodResources,
+ /// Maximum number of planning workers.
+ pub max_workers: f64,
+ /// Maximum rows assigned to one planning worker.
+ pub rows_per_worker: f64,
+ /// Maximum fragments assigned to one planning worker.
+ pub fragments_per_worker: f64,
+}
+
+/// Overrides retries of failed user-code calls for one refresh. Omitted
+/// fields use the Function refresh defaults.
+#[napi(object)]
+#[derive(Clone, Debug, Default)]
+pub struct FunctionRetryPolicy {
+ /// Total calls, including the first; 1 disables retries, maximum 100.
+ pub max_attempts: Option,
+ /// Initial exponential-backoff ceiling, in seconds.
+ pub initial_backoff_seconds: Option,
+ /// Maximum backoff, in seconds.
+ pub max_backoff_seconds: Option,
+ /// Exception class names, preferably qualified.
+ pub exception_types: Option>,
+ /// Regular expression matched against the exception class and message.
+ pub message_pattern: Option,
+ /// Structured provider error codes or HTTP status codes.
+ pub error_codes: Option>,
+ /// Match connection/timeouts, HTTP 429/503, SlowDown and
+ /// ServiceUnavailable.
+ pub transient: Option,
+ /// Honor Retry-After within the configured maximum.
+ pub honor_retry_after: Option,
+}
+
fn positive_safe_integer(value: f64, name: &str) -> napi::Result {
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
if !value.is_finite() || value.fract() != 0.0 || !(1.0..=MAX_SAFE_INTEGER).contains(&value) {
@@ -1768,8 +1822,14 @@ pub struct RefreshColumnOptions {
/// Complete aggregate CPU and memory budget for a registered Function
/// refresh. `gpuCount` inside the budget defaults to zero.
pub resource_budget: Option,
+ /// CPU, memory, and ephemeral-storage limits for the refresh supervisor.
+ pub supervisor_resources: Option,
+ /// Resources and sizing for the refresh planning fleet.
+ pub planning_workers: Option,
/// Positive aggregate GPU budget for Function workers.
pub gpu_count: Option,
+ /// Override retries of failed user-code calls for this refresh.
+ pub retry_policy: Option,
/// `"fail"` (default) stops on a row error; `"skip"` writes null and
/// records the error, subject to the failure limits.
pub error_policy: Option,
@@ -1814,7 +1874,87 @@ impl TryFrom for lancedb::table::RefreshColumnOptions {
)
})
.transpose()?;
+ options.supervisor_resources = value
+ .supervisor_resources
+ .map(|resources| {
+ Ok::<_, napi::Error>(LanceFunctionRefreshPodResources::new(
+ positive_safe_integer(resources.cpu_millis, "supervisorResources.cpuMillis")?,
+ positive_safe_integer(
+ resources.memory_bytes,
+ "supervisorResources.memoryBytes",
+ )?,
+ positive_safe_integer(
+ resources.ephemeral_storage_bytes,
+ "supervisorResources.ephemeralStorageBytes",
+ )?,
+ ))
+ })
+ .transpose()?;
+ options.planning_workers = value
+ .planning_workers
+ .map(|planning| {
+ let resources = LanceFunctionRefreshPodResources::new(
+ positive_safe_integer(
+ planning.resources.cpu_millis,
+ "planningWorkers.resources.cpuMillis",
+ )?,
+ positive_safe_integer(
+ planning.resources.memory_bytes,
+ "planningWorkers.resources.memoryBytes",
+ )?,
+ positive_safe_integer(
+ planning.resources.ephemeral_storage_bytes,
+ "planningWorkers.resources.ephemeralStorageBytes",
+ )?,
+ );
+ let max_workers = usize::try_from(positive_safe_integer(
+ planning.max_workers,
+ "planningWorkers.maxWorkers",
+ )?)
+ .map_err(|_| napi::Error::from_reason("planningWorkers.maxWorkers is too large"))?;
+ Ok::<_, napi::Error>(LanceFunctionRefreshPlanningConfig::new(
+ resources,
+ max_workers,
+ positive_safe_integer(
+ planning.rows_per_worker,
+ "planningWorkers.rowsPerWorker",
+ )?,
+ positive_safe_integer(
+ planning.fragments_per_worker,
+ "planningWorkers.fragmentsPerWorker",
+ )?,
+ ))
+ })
+ .transpose()?;
options.gpu_count = value.gpu_count;
+ options.retry_policy = value.retry_policy.map(|policy| {
+ let mut converted = lancedb::function::FunctionRetryPolicy::default();
+ if let Some(max_attempts) = policy.max_attempts {
+ converted.max_attempts = max_attempts;
+ }
+ if let Some(seconds) = policy.initial_backoff_seconds {
+ converted.initial_backoff_seconds = seconds;
+ }
+ if let Some(seconds) = policy.max_backoff_seconds {
+ converted.max_backoff_seconds = seconds;
+ }
+ if let Some(exception_types) = policy.exception_types {
+ converted.exception_types = exception_types;
+ }
+ if let Some(pattern) = policy.message_pattern {
+ converted.message_pattern = Some(pattern);
+ }
+ if let Some(error_codes) = policy.error_codes {
+ converted.error_codes = error_codes;
+ }
+ if let Some(transient) = policy.transient {
+ converted.transient = transient;
+ }
+ if let Some(honor_retry_after) = policy.honor_retry_after {
+ converted.honor_retry_after = honor_retry_after;
+ }
+ converted
+ });
options.error_policy = match value.error_policy.as_deref() {
None => None,
Some("fail") => Some(lancedb::function::FunctionErrorPolicy::Fail),
diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index 1e811d865..cce3efeef 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -663,6 +663,8 @@ class Table:
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[dict[str, int]] = None,
+ supervisor_resources: Optional[dict[str, int]] = None,
+ planning_workers: Optional[dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[str] = None,
@@ -683,6 +685,8 @@ class Table:
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[dict[str, int]] = None,
+ supervisor_resources: Optional[dict[str, int]] = None,
+ planning_workers: Optional[dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[str] = None,
diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py
index dbfd127ee..b5d97eb62 100644
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -1164,6 +1164,8 @@ class RemoteTable(Table):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -1183,6 +1185,8 @@ class RemoteTable(Table):
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=retry_policy,
@@ -1205,6 +1209,8 @@ class RemoteTable(Table):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -1225,6 +1231,8 @@ class RemoteTable(Table):
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=retry_policy,
diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index e0b96e476..45b2eb4f6 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -3258,6 +3258,8 @@ class Table(ABC):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -3315,6 +3317,17 @@ class Table(ABC):
CPU-only budget. This budget cannot be combined with the top-level
``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes
reject it.
+ supervisor_resources: dict[str, int], optional
+ CPU, memory, and ephemeral-storage limits for the Function refresh
+ supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and
+ ``ephemeral_storage_bytes``; all are required and positive.
+ LanceDB Enterprise only; local refreshes reject it.
+ planning_workers: dict[str, object], optional
+ Resources and sizing for the Function refresh planning fleet.
+ ``resources`` has the same keys as ``supervisor_resources``; the
+ other keys are ``max_workers``, ``rows_per_worker`` and
+ ``fragments_per_worker``. LanceDB Enterprise only; local refreshes
+ reject it.
gpu_class: str, optional
GPU class for Function workers. Must be a lowercase name of 1-63
letters, digits, or hyphens, without a leading or trailing hyphen.
@@ -3367,6 +3380,8 @@ class Table(ABC):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -3423,6 +3438,17 @@ class Table(ABC):
CPU-only budget. This budget cannot be combined with the top-level
``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes
reject it.
+ supervisor_resources: dict[str, int], optional
+ CPU, memory, and ephemeral-storage limits for the Function refresh
+ supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and
+ ``ephemeral_storage_bytes``; all are required and positive.
+ LanceDB Enterprise only; local refreshes reject it.
+ planning_workers: dict[str, object], optional
+ Resources and sizing for the Function refresh planning fleet.
+ ``resources`` has the same keys as ``supervisor_resources``; the
+ other keys are ``max_workers``, ``rows_per_worker`` and
+ ``fragments_per_worker``. LanceDB Enterprise only; local refreshes
+ reject it.
gpu_class: str, optional
GPU class for Function workers. Must be a lowercase name of 1-63
letters, digits, or hyphens, without a leading or trailing hyphen.
@@ -5632,6 +5658,8 @@ class LanceTable(Table):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -5654,6 +5682,8 @@ class LanceTable(Table):
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=retry_policy,
@@ -5676,6 +5706,8 @@ class LanceTable(Table):
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -5700,6 +5732,8 @@ class LanceTable(Table):
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=retry_policy,
@@ -8109,6 +8143,8 @@ class AsyncTable:
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -8166,6 +8202,17 @@ class AsyncTable:
CPU-only budget. This budget cannot be combined with the top-level
``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes
reject it.
+ supervisor_resources: dict[str, int], optional
+ CPU, memory, and ephemeral-storage limits for the Function refresh
+ supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and
+ ``ephemeral_storage_bytes``; all are required and positive.
+ LanceDB Enterprise only; local refreshes reject it.
+ planning_workers: dict[str, object], optional
+ Resources and sizing for the Function refresh planning fleet.
+ ``resources`` has the same keys as ``supervisor_resources``; the
+ other keys are ``max_workers``, ``rows_per_worker`` and
+ ``fragments_per_worker``. LanceDB Enterprise only; local refreshes
+ reject it.
gpu_class: str, optional
GPU class for Function workers. Must be a lowercase name of 1-63
letters, digits, or hyphens, without a leading or trailing hyphen.
@@ -8213,6 +8260,8 @@ class AsyncTable:
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=None
@@ -8351,6 +8400,8 @@ class AsyncTable:
max_concurrency: Optional[int] = None,
publication_interval_rows: Optional[int] = None,
resource_budget: Optional[Dict[str, int]] = None,
+ supervisor_resources: Optional[Dict[str, int]] = None,
+ planning_workers: Optional[Dict[str, Any]] = None,
gpu_class: Optional[str] = None,
gpu_count: Optional[int] = None,
retry_policy: Optional[FunctionRetryPolicy] = None,
@@ -8407,6 +8458,17 @@ class AsyncTable:
CPU-only budget. This budget cannot be combined with the top-level
``gpu_count`` shorthand. LanceDB Enterprise only; local refreshes
reject it.
+ supervisor_resources: dict[str, int], optional
+ CPU, memory, and ephemeral-storage limits for the Function refresh
+ supervisor pod. Keys are ``cpu_millis``, ``memory_bytes`` and
+ ``ephemeral_storage_bytes``; all are required and positive.
+ LanceDB Enterprise only; local refreshes reject it.
+ planning_workers: dict[str, object], optional
+ Resources and sizing for the Function refresh planning fleet.
+ ``resources`` has the same keys as ``supervisor_resources``; the
+ other keys are ``max_workers``, ``rows_per_worker`` and
+ ``fragments_per_worker``. LanceDB Enterprise only; local refreshes
+ reject it.
gpu_class: str, optional
GPU class for Function workers. Must be a lowercase name of 1-63
letters, digits, or hyphens, without a leading or trailing hyphen.
@@ -8471,6 +8533,8 @@ class AsyncTable:
max_concurrency=max_concurrency,
publication_interval_rows=publication_interval_rows,
resource_budget=resource_budget,
+ supervisor_resources=supervisor_resources,
+ planning_workers=planning_workers,
gpu_class=gpu_class,
gpu_count=gpu_count,
retry_policy=None
diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py
index c5afac308..fc5b5df73 100644
--- a/python/python/tests/test_remote_db.py
+++ b/python/python/tests/test_remote_db.py
@@ -2076,6 +2076,23 @@ def test_remote_create_index_async_returns_job():
"gpu_count": 2,
}
},
+ {
+ "supervisor_resources": {
+ "cpu_millis": 2000,
+ "memory_bytes": 4 * 1024**3,
+ "ephemeral_storage_bytes": 10 * 1024**3,
+ },
+ "planning_workers": {
+ "resources": {
+ "cpu_millis": 1000,
+ "memory_bytes": 2 * 1024**3,
+ "ephemeral_storage_bytes": 5 * 1024**3,
+ },
+ "max_workers": 4,
+ "rows_per_worker": 100_000,
+ "fragments_per_worker": 8,
+ },
+ },
{"gpu_class": "a10g", "gpu_count": 2},
{
"retry_policy": {"transient": True, "max_attempts": 3},
@@ -2122,6 +2139,9 @@ async def test_remote_refresh_async_returns_typed_terminal_result(
**budget,
"gpu_count": budget.get("gpu_count", 0),
}
+ for key in ("supervisor_resources", "planning_workers"):
+ if refresh_kwargs.get(key) is not None:
+ expected[key] = refresh_kwargs[key]
if refresh_kwargs.get("retry_policy") is not None:
expected["retry_policy"] = lancedb.FunctionRetryPolicy.model_validate(
refresh_kwargs["retry_policy"]
@@ -2350,6 +2370,51 @@ def test_remote_refresh_async_forwards_progress_from_submitted_handle():
},
"unsupported key 'unknown'",
),
+ (
+ {
+ "supervisor_resources": {
+ "cpu_millis": 0,
+ "memory_bytes": 1,
+ "ephemeral_storage_bytes": 1,
+ }
+ },
+ "supervisor_resources.cpu_millis",
+ ),
+ (
+ {
+ "supervisor_resources": {
+ "cpu_millis": 1,
+ "memory_bytes": 1,
+ }
+ },
+ "supervisor_resources.ephemeral_storage_bytes",
+ ),
+ (
+ {
+ "supervisor_resources": {
+ "cpu_millis": 1,
+ "memory_bytes": 1,
+ "ephemeral_storage_bytes": 1,
+ "unknown": 1,
+ }
+ },
+ "unsupported key 'unknown'",
+ ),
+ (
+ {
+ "planning_workers": {
+ "resources": {
+ "cpu_millis": 1,
+ "memory_bytes": 1,
+ "ephemeral_storage_bytes": 1,
+ },
+ "max_workers": 0,
+ "rows_per_worker": 1,
+ "fragments_per_worker": 1,
+ }
+ },
+ "planning_workers.max_workers",
+ ),
(
{"max_executor_processes_per_worker": -2},
"max_executor_processes_per_worker",
diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py
index 604d6afb3..ce4f57c48 100644
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -6013,6 +6013,31 @@ def test_refresh_column_rejects_invalid_batch_size(tmp_path, method, batch_size)
{"resource_budget": {"cpu_millis": 1000, "memory_bytes": 1024}},
"resource_budget",
),
+ (
+ {
+ "supervisor_resources": {
+ "cpu_millis": 1000,
+ "memory_bytes": 1024,
+ "ephemeral_storage_bytes": 2048,
+ }
+ },
+ "supervisor_resources",
+ ),
+ (
+ {
+ "planning_workers": {
+ "resources": {
+ "cpu_millis": 1000,
+ "memory_bytes": 1024,
+ "ephemeral_storage_bytes": 2048,
+ },
+ "max_workers": 2,
+ "rows_per_worker": 100,
+ "fragments_per_worker": 1,
+ }
+ },
+ "planning_workers",
+ ),
({"num_workers": 2}, "num_workers"),
({"max_executor_processes_per_worker": 2}, "max_executor_processes_per_worker"),
],
diff --git a/python/src/table.rs b/python/src/table.rs
index f77ff5a7f..f5270a594 100644
--- a/python/src/table.rs
+++ b/python/src/table.rs
@@ -30,8 +30,9 @@ use lancedb::connection::LanceFileVersion;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::table::{
AddDataMode, ColumnAlteration, CompactionMode, CompactionOptions, Duration,
- FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionWorkerResourceBudget, IndexRemapMode,
- NewColumnTransform, Ref, RefreshColumnOptions, Table as LanceDbTable,
+ FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionRefreshPlanningConfig,
+ FunctionRefreshPodResources, FunctionWorkerResourceBudget, IndexRemapMode, NewColumnTransform,
+ Ref, RefreshColumnOptions, Table as LanceDbTable,
};
use lancedb::tokenize as lancedb_tokenize;
use pyo3::{
@@ -52,6 +53,8 @@ struct RefreshColumnArgs {
max_concurrency: Option,
publication_interval_rows: Option,
resource_budget: Option,
+ supervisor_resources: Option,
+ planning_workers: Option,
gpu_class: Option,
gpu_count: Option,
retry_policy: Option,
@@ -159,6 +162,101 @@ fn parse_resource_budget(
))
}
+fn structured_integer(value: &Bound<'_, PyAny>, name: &str) -> PyResult {
+ if value.is_none() || value.is_instance_of::() {
+ return Err(PyValueError::new_err(format!("{name} must be an integer")));
+ }
+ let parsed = value
+ .extract::()
+ .map_err(|_| PyValueError::new_err(format!("{name} must be an integer")))?;
+ if parsed == 0 {
+ return Err(PyValueError::new_err(format!(
+ "{name} must be greater than zero"
+ )));
+ }
+ Ok(parsed)
+}
+
+fn parse_pod_resources(
+ resources: &Bound<'_, PyDict>,
+ prefix: &str,
+) -> PyResult {
+ for key in resources.keys() {
+ let key = key
+ .extract::()
+ .map_err(|_| PyValueError::new_err(format!("{prefix} keys must be strings")))?;
+ if !matches!(
+ key.as_str(),
+ "cpu_millis" | "memory_bytes" | "ephemeral_storage_bytes"
+ ) {
+ return Err(PyValueError::new_err(format!(
+ "{prefix} contains unsupported key '{key}'"
+ )));
+ }
+ }
+ let get = |name: &str| -> PyResult {
+ let value = resources
+ .get_item(name)?
+ .ok_or_else(|| PyValueError::new_err(format!("{prefix}.{name} is required")))?;
+ structured_integer(&value, &format!("{prefix}.{name}"))
+ };
+ Ok(FunctionRefreshPodResources::new(
+ get("cpu_millis")?,
+ get("memory_bytes")?,
+ get("ephemeral_storage_bytes")?,
+ ))
+}
+
+fn parse_supervisor_resources(
+ resources: Option<&Bound<'_, PyDict>>,
+) -> PyResult