feat: expose Function publication intervals in refresh options (#4928)

Python refresh APIs currently reject `publication_interval_rows`, so
callers cannot request incremental Function output publication through
the SDK. Add the optional typed parameter through the sync/async Python
APIs, PyO3, and Rust `RefreshColumnOptions`, and serialize it as a
top-level Enterprise refresh option.

An omitted value preserves the existing wire request and whole-fragment
default. Positive values set a publication threshold; zero and negative
Python values fail before a request is sent. Embedded refreshes reject
the Enterprise-only option. Documentation describes a threshold rather
than an exact number of rows per publication.

Validation on the designated development host:
- A matching baseline build reproduced the missing public keyword
argument in both sync and async submission tests.
- The updated PyO3 build passed 54 focused Python tests; Rust payload,
invalid-option, and default-wire tests each passed. Rust formatting and
Python Ruff checks passed.
- The single signed delivery commit differs from that tested
implementation only in a Rustdoc correction describing Embedded
rejection; formatting passed again. No dependency pins or lockfiles
changed.

The matching SDK is prepared for the separate server validation of
failure/cancellation retention and missing-only refill. Those end-to-end
results are still pending and are not implied by these SDK tests.

Refs [ENT-3733](https://linear.app/lancedb/issue/ENT-3733).
This commit is contained in:
Xuanwo
2026-11-10 15:22:56 +08:00
committed by GitHub
parent 0a5c36c1e2
commit 13e4cf7e58
10 changed files with 128 additions and 5 deletions
+10 -2
View File
@@ -46,6 +46,7 @@ mod scannable;
struct RefreshColumnArgs {
where_clause: Option<String>,
batch_size: Option<i64>,
publication_interval_rows: Option<i64>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -73,6 +74,7 @@ fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOpti
let RefreshColumnArgs {
where_clause,
batch_size,
publication_interval_rows,
gpu_class,
gpu_count,
retry_policy,
@@ -114,6 +116,8 @@ fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOpti
.map(usize::try_from)
.transpose()
.map_err(|_| PyValueError::new_err("batch_size must be greater than zero"))?;
options.publication_interval_rows =
positive_count(publication_interval_rows, "publication_interval_rows")?;
options.gpu_class = gpu_class;
options.gpu_count = gpu_count
.map(|count| {
@@ -2206,13 +2210,14 @@ impl Table {
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[pyo3(signature = (column, *, r#where=None, batch_size=None, publication_interval_rows=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column(
self_: PyRef<'_, Self>,
column: String,
r#where: Option<String>,
batch_size: Option<i64>,
publication_interval_rows: Option<i64>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -2227,6 +2232,7 @@ impl Table {
let options = refresh_column_options(RefreshColumnArgs {
where_clause: r#where,
batch_size,
publication_interval_rows,
gpu_class,
gpu_count,
retry_policy,
@@ -2248,13 +2254,14 @@ impl Table {
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[pyo3(signature = (column, *, r#where=None, batch_size=None, publication_interval_rows=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column_async(
self_: PyRef<'_, Self>,
column: String,
r#where: Option<String>,
batch_size: Option<i64>,
publication_interval_rows: Option<i64>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
@@ -2269,6 +2276,7 @@ impl Table {
let options = refresh_column_options(RefreshColumnArgs {
where_clause: r#where,
batch_size,
publication_interval_rows,
gpu_class,
gpu_count,
retry_policy,