feat: expose the remaining refresh options on the SDKs (#4890)

The server's backfill request takes a row filter, failure limits,
per-call timeouts and worker sizing, and SQL exposes all of them through
`REFRESH COLUMN ... WHERE ... WITH (...)`, but the typed SDKs could send
only batch_size, GPU placement and the retry and error policies. Anyone
needing a filtered refresh or a bounded skip policy had to drop to SQL.

This adds `where`, `max_failed_fraction`, `max_failed_rows`,
`row_timeout_seconds`, `init_timeout_seconds`, `num_workers` and
`max_executor_processes_per_worker` to Python, Rust and Node, passed
through unchanged. `RefreshColumnOptions` now serializes straight into
the request body, so the three SDKs share one producer. Node gets a
`RefreshColumnOptions` object, which also carries the options it lacked
before. A LanceDB Embedded refresh rejects every option other than
batch_size and GPU placement, naming the option.
This commit is contained in:
Wyatt Alt
2026-11-06 09:19:27 -08:00
committed by GitHub
parent d379bbb722
commit 2af85bea89
18 changed files with 974 additions and 53 deletions
+1
View File
@@ -56,6 +56,7 @@ export {
MergeResult,
AddResult,
AddColumnsResult,
RefreshColumnOptions,
RefreshColumnResult,
FunctionErrors,
FunctionErrorsOptions,
+35 -7
View File
@@ -38,6 +38,7 @@ import {
Branches as NativeBranches,
OptimizeStats,
RTEStats,
RefreshColumnOptions,
RefreshColumnResult,
RefreshMaterializedViewResult,
TableStatistics,
@@ -671,10 +672,15 @@ export abstract class Table {
* everything else is left as it is. LanceDB Embedded only: in LanceDB
* Enterprise a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
* @param {string} column The name of the computed column to fill.
* @param {RefreshColumnOptions} options Refresh options, all LanceDB
* Enterprise only; see {@link Table#refreshColumnAsync}.
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
* number of rows filled and the new version number of the table.
*/
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
abstract refreshColumn(
column: string,
options?: RefreshColumnOptions,
): Promise<RefreshColumnResult>;
/**
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
@@ -689,14 +695,30 @@ export abstract class Table {
* than failing the job. In LanceDB Embedded the job runs in-process; in
* LanceDB Enterprise it is the server's backfill job.
* @param {string} column The name of the computed column to fill.
* @param {RefreshColumnOptions} options Refresh options, mirroring the SQL
* `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
* `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
* and `initTimeoutSeconds` govern failing rows, and `batchSize`,
* `gpuClass`, `gpuCount`, `numWorkers` and `maxExecutorProcessesPerWorker`
* size the workers. All are LanceDB Enterprise only; a LanceDB Embedded
* refresh rejects every option other than `batchSize`, `gpuClass` and
* `gpuCount`.
* @example
* ```ts
* const job = await table.refreshColumnAsync("doubled");
* const job = await table.refreshColumnAsync("embedding", {
* where: "published_at > '2026-01-01'",
* errorPolicy: "skip",
* maxFailedFraction: 0.01,
* rowTimeoutSeconds: 30,
* });
* await job.wait();
* console.log(await job.status()); // "finished"
* ```
*/
abstract refreshColumnAsync(column: string): Promise<Job>;
abstract refreshColumnAsync(
column: string,
options?: RefreshColumnOptions,
): Promise<Job>;
/**
* The per-row errors Function refreshes recorded on this table.
@@ -1472,12 +1494,18 @@ export class LocalTable extends Table {
throw new Error("Invalid input type for addColumns");
}
async refreshColumn(column: string): Promise<RefreshColumnResult> {
return await this.inner.refreshColumn(column);
async refreshColumn(
column: string,
options?: RefreshColumnOptions,
): Promise<RefreshColumnResult> {
return await this.inner.refreshColumn(column, options);
}
async refreshColumnAsync(column: string): Promise<Job> {
return new Job(await this.inner.refreshColumnAsync(column));
async refreshColumnAsync(
column: string,
options?: RefreshColumnOptions,
): Promise<Job> {
return new Job(await this.inner.refreshColumnAsync(column, options));
}
async functionErrors(