feat: expose per-call merge retry controls in Python and Node (#5015)

Python and Node callers could not disable precommit retries for one
merge without changing the connection. Add `execute(..., retries=0)` for
Python sync/async and `execute(data, { retries: 0 })` for TypeScript,
forwarding into the existing Rust control without persisting the
override. Ambiguous 5xx outcomes remain non-replayable; permitted merge
retries now log status and attempt at WARN, and the API docs explain
retry budgets and timeout boundaries.

Validation at `6001300` passed 70 Python HTTP cases, 36 Node HTTP cases,
14 local merge regressions per binding, and 34 scoped Rust tests, with
rebuilt native extensions, TypeScript compilation, Clippy, scoped
formatting/lint, and generated API docs. The public API and feature diff
are unchanged after integrating main and the reviewed ingestion guide;
that binding evidence is retained. Focused merged-tree validation
additionally passed 267 Rust merge/retry/provenance cases and the CI
fixtures. Existing root Ruff issues in docs/web-tests remain; local
fixtures do not establish live proxy/server cancellation behavior.

Refs #4256 and OSS-2370.
This commit is contained in:
Xuanwo
2026-11-14 22:43:56 +08:00
committed by GitHub
parent dc9eb269b1
commit e2f836da99
14 changed files with 433 additions and 10 deletions
@@ -8,6 +8,26 @@
## Properties
### retries?
```ts
optional retries: number;
```
Override the remote retry budget for this execution only (an integer from
0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
precommit conflicts (409) or rate limits (429). This does not change
connection-wide/read budgets or local transaction conflict retries.
By default, remote calls use the connection's `clientConfig.retryConfig`
([RetryConfig](RetryConfig.md)). Its `statuses` (also set through
`LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
it never permits replay of ambiguous merge outcomes. In particular, 502,
ordinary 503, and 504 are never replayed, even if configured. Reconcile table
contents before manually repeating a merge whose commit outcome is unknown.
***
### timeoutMs?
```ts
@@ -24,3 +44,6 @@ the second attempt will be cancelled after 10 seconds, hitting the
first attempt will not be cancelled.
When this is set, the timeout is enforced on all attempts, including the first.
For remote tables this is sent as a server execution timeout. It does not
extend HTTP read or proxy idle timeouts, or prove that a failed request did
not commit.
+7
View File
@@ -221,6 +221,13 @@ and check inserted or deleted counts when the SDK returns them. Repeating a
merge is only safe when its keys, conditions and update values produce the
intended result on retry.
To override the transport retry budget for one merge, pass `retries=0` to
Python's synchronous or asynchronous `builder.execute(data, retries=0)`, or
use TypeScript's `builder.execute(data, { retries: 0 })`. The override applies
only to that execution. A larger budget does not make an ambiguous 5xx result
safe to replay. See [write retry boundaries](/ingestion/insert#performance)
for the distinction between retryable failures and an unknown commit outcome.
### Embeddings and expression predicates {#advanced}
When a table uses an [embedding definition](/embedding), behavior depends on
+157
View File
@@ -542,6 +542,163 @@ describe("remote connection", () => {
expect(attempts).toBe(scenario.attempts);
});
it.each(
[409, 429, 502, 503, 504].flatMap((status) =>
[undefined, 0, 2].flatMap((retries) =>
[[], ["analytics", "features"]].map((namespacePath) => ({
status,
retries,
namespacePath,
})),
),
),
)("applies per-call merge retries: %j", async (scenario) => {
let attempts = 0;
const identifier = [...scenario.namespacePath, "items"].join("$");
const succeeds =
[409, 429].includes(scenario.status) && scenario.retries !== 0;
await withMockDatabase(
(req, res) => {
const url = new URL(req.url ?? "", "http://localhost");
if (url.pathname === `/v1/table/${identifier}/describe/`) {
res.writeHead(200).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: true },
],
},
}),
);
return;
}
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
try {
expect(url.pathname).toBe(`/v1/table/${identifier}/merge_insert/`);
expect(url.searchParams.get("use_index")).toBe("false");
expect(req.headers["x-request-timeout-ms"]).toBe("10000");
// Rust compresses the batches; Arrow JS can inspect the schema without a codec.
const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
expect(reader.schema.fields.map((field) => field.name)).toEqual([
"id",
]);
} finally {
attempts++;
res
.writeHead(attempts === 1 ? scenario.status : 200)
.end('{"version":2}');
}
});
},
async (db) => {
const table = await db.openTable("items", scenario.namespacePath);
const merge = table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useIndex(false);
const result = merge.execute([{ id: 1n }], {
timeoutMs: 10_000,
retries: scenario.retries,
});
if (succeeds) {
await expect(result).resolves.toMatchObject({ version: 2 });
} else {
await expect(result).rejects.toThrow();
}
},
{
clientConfig: {
retryConfig: {
retries: 3,
statuses: [409, 429, 502, 503, 504],
backoffFactor: 0,
backoffJitter: 0,
},
},
},
);
expect(attempts).toBe(succeeds ? 2 : 1);
});
it("isolates merge retry overrides across reused builders and reads", async () => {
let attempts = 0;
let readAttempts = 0;
await withMockDatabase(
(req, res) => {
if (req.url?.endsWith("/describe/")) {
res.writeHead(200).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: true },
],
},
}),
);
return;
}
req.resume();
req.on("end", () => {
if (req.url?.endsWith("/count_rows/")) {
readAttempts++;
res.writeHead(readAttempts % 2 ? 429 : 200).end("1");
} else {
attempts++;
res
.writeHead([1, 2, 4, 6].includes(attempts) ? 429 : 200)
.end('{"version":2}');
expect(req.url).toContain("/merge_insert/");
}
});
},
async (db) => {
const table = await db.openTable("items");
const merge = table.mergeInsert("id").whenNotMatchedInsertAll();
for (const retries of [
-1,
256,
1.5,
Number.NaN,
Number.POSITIVE_INFINITY,
]) {
await expect(
merge.execute([{ id: 1n }], { retries }),
).rejects.toThrow("retries must be an integer between 0 and 255");
}
expect(attempts).toBe(0);
await expect(
merge.execute([{ id: 1n }], { retries: 0 }),
).rejects.toThrow();
expect(attempts).toBe(1);
await expect(merge.execute([{ id: 1n }])).resolves.toMatchObject({
version: 2,
});
await expect(
merge.execute([{ id: 1n }], { retries: 255 }),
).resolves.toMatchObject({ version: 2 });
expect(attempts).toBe(5);
await expect(table.countRows()).resolves.toBe(1);
expect(readAttempts).toBe(2);
await expect(
table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.execute([{ id: 1n }]),
).resolves.toMatchObject({ version: 2 });
expect(attempts).toBe(7);
},
{
clientConfig: {
retryConfig: { retries: 3, backoffFactor: 0, backoffJitter: 0 },
},
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
+2 -1
View File
@@ -938,7 +938,8 @@ describe("merge insert", () => {
.mergeInsert("a")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(newData, { timeoutMs: 10_000 });
.useIndex(false)
.execute(newData, { timeoutMs: 10_000, retries: 0 });
expect(mergeInsertRes).toHaveProperty("version");
expect(mergeInsertRes.version).toBe(2);
expect(mergeInsertRes.numInsertedRows).toBe(1);
+27 -1
View File
@@ -127,6 +127,14 @@ export class MergeInsertBuilder {
data: Data,
execOptions?: Partial<WriteExecutionOptions>,
): Promise<MergeResult> {
if (
execOptions?.retries !== undefined &&
(!Number.isInteger(execOptions.retries) ||
execOptions.retries < 0 ||
execOptions.retries > 255)
) {
throw new Error("retries must be an integer between 0 and 255");
}
let schema: Schema;
if (this.#schema instanceof Promise) {
schema = await this.#schema;
@@ -140,7 +148,7 @@ export class MergeInsertBuilder {
}
const buffer = await fromDataToBuffer(data, undefined, schema);
return await this.#native.execute(buffer);
return await this.#native.execute(buffer, execOptions?.retries);
}
}
@@ -156,6 +164,24 @@ export interface WriteExecutionOptions {
* first attempt will not be cancelled.
*
* When this is set, the timeout is enforced on all attempts, including the first.
* For remote tables this is sent as a server execution timeout. It does not
* extend HTTP read or proxy idle timeouts, or prove that a failed request did
* not commit.
*/
timeoutMs?: number;
/**
* Override the remote retry budget for this execution only (an integer from
* 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
* precommit conflicts (409) or rate limits (429). This does not change
* connection-wide/read budgets or local transaction conflict retries.
*
* By default, remote calls use the connection's `clientConfig.retryConfig`
* ({@link RetryConfig}). Its `statuses` (also set through
* `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
* it never permits replay of ambiguous merge outcomes. In particular, 502,
* ordinary 503, and 504 are never replayed, even if configured. Reconcile table
* contents before manually repeating a merge whose commit outcome is unknown.
*/
retries?: number;
}
+7 -2
View File
@@ -65,12 +65,17 @@ impl NativeMergeInsertBuilder {
}
#[napi(catch_unwind)]
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
pub async fn execute(&self, buf: Buffer, retries: Option<u32>) -> napi::Result<MergeResult> {
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
})?;
let this = self.clone();
let mut this = self.clone();
if let Some(retries) = retries {
let retries = u8::try_from(retries)
.map_err(|_| napi::Error::from_reason("retries must be between 0 and 255"))?;
this.inner.retries(retries);
}
let res = this.inner.execute(data).await.map_err(|e| {
napi::Error::from_reason(format!(
+6
View File
@@ -565,6 +565,12 @@ class Table:
allow_external_blob_outside_bases: bool = False,
retries: Optional[int] = None,
) -> AddResult: ...
async def execute_merge_insert(
self,
data: pa.RecordBatchReader,
parameters: Dict[str, Any],
retries: Optional[int] = None,
) -> MergeResult: ...
async def update(
self, updates: Dict[str, str], where: Optional[str]
) -> UpdateResult: ...
+21 -2
View File
@@ -161,6 +161,8 @@ class LanceMergeInsertBuilder(object):
on_bad_vectors: str = "error",
fill_value: float = 0.0,
timeout: Optional[timedelta] = None,
*,
retries: Optional[int] = None,
) -> MergeInsertResult:
"""
Executes the merge insert operation
@@ -189,7 +191,22 @@ class LanceMergeInsertBuilder(object):
first attempt will not be cancelled.
When this is set, the timeout is enforced on all attempts, including
the first.
the first. For remote tables it is sent as a server execution timeout;
it does not extend HTTP read or proxy idle timeouts, or prove that a
failed request did not commit.
retries: int, optional
Override the remote retry budget for this execution only (0 to 255).
Use ``execute(new_data, retries=0)`` to disable even retries after
precommit conflicts (409) or rate limits (429). This does not change
connection-wide/read budgets or local transaction conflict retries.
By default, remote calls use the connection's
[RetryConfig][lancedb.remote.RetryConfig]. Its ``statuses`` (also set
through ``LANCE_CLIENT_RETRY_STATUSES``) selects candidates within
the retry budget; it never permits replay of ambiguous merge outcomes.
In particular, 502, ordinary 503, and 504 are never replayed, even if
configured. Reconcile table contents before manually repeating a merge
whose commit outcome is unknown.
Returns
-------
@@ -198,4 +215,6 @@ class LanceMergeInsertBuilder(object):
"""
if timeout is not None:
self._timeout = timeout
return self._table._do_merge(self, new_data, on_bad_vectors, fill_value)
return self._table._do_merge(
self, new_data, on_bad_vectors, fill_value, retries=retries
)
+2 -1
View File
@@ -945,9 +945,10 @@ class RemoteTable(Table):
new_data: DATA,
on_bad_vectors: str,
fill_value: float,
retries: Optional[int] = None,
) -> MergeResult:
return LOOP.run(
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries)
)
@_deprecated_keyword("predicate", "where")
+5 -1
View File
@@ -2810,6 +2810,7 @@ class Table(ABC):
new_data: DATA,
on_bad_vectors: OnBadVectorsType,
fill_value: float,
retries: Optional[int] = None,
) -> MergeResult: ...
@abstractmethod
@@ -5443,9 +5444,10 @@ class LanceTable(Table):
new_data: DATA,
on_bad_vectors: OnBadVectorsType,
fill_value: float,
retries: Optional[int] = None,
) -> MergeResult:
return LOOP.run(
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries)
)
@property
@@ -7777,6 +7779,7 @@ class AsyncTable:
new_data: DATA,
on_bad_vectors: OnBadVectorsType,
fill_value: float,
retries: Optional[int] = None,
) -> MergeResult:
schema = await self.schema()
if on_bad_vectors is None:
@@ -7808,6 +7811,7 @@ class AsyncTable:
use_rte=merge._use_rte,
validate_single_shard=merge._validate_single_shard,
),
retries=retries,
)
async def delete(self, where: Union[str, Expr]) -> DeleteResult:
+158
View File
@@ -310,6 +310,164 @@ async def test_add_retry_override(sync_api, retries, status):
assert attempts == (2 if expected_success else 1)
@pytest.mark.parametrize("sync_api", [False, True])
@pytest.mark.parametrize("namespace_path", [[], ["analytics", "features"]])
@pytest.mark.parametrize("retries", [None, 0, 2])
@pytest.mark.parametrize("status", [409, 429, 502, 503, 504])
@pytest.mark.asyncio
async def test_merge_retry_override(sync_api, namespace_path, retries, status, capfd):
attempts = 0
identifier = "$".join(namespace_path + ["test"])
def handler(request):
nonlocal attempts
if request.path == f"/v1/table/{identifier}/describe/":
send_json(
request,
{
"version": 1,
"schema": {
"fields": [
{"name": "id", "type": {"type": "int64"}, "nullable": True}
]
},
},
)
return
assert urlparse(request.path).path == f"/v1/table/{identifier}/merge_insert/"
query = parse_qs(urlparse(request.path).query)
assert query["use_index"] == ["false"]
assert request.headers["x-request-timeout-ms"] == "10000"
body = bytearray()
if request.headers.get("Transfer-Encoding") == "chunked":
while True:
size = int(request.rfile.readline().strip(), 16)
if size == 0:
request.rfile.readline()
break
body.extend(request.rfile.read(size))
request.rfile.read(2)
else:
body.extend(
request.rfile.read(int(request.headers.get("Content-Length", 0)))
)
assert pa.ipc.open_stream(body).read_all().to_pylist() == [{"id": 1}]
attempts += 1
send_json(request, {"version": 2}, status if attempts == 1 else 200)
config = {
"retry_config": {
"retries": 3,
"statuses": [409, 429, 502, 503, 504],
"backoff_factor": 0,
"backoff_jitter": 0,
}
}
expected_success = status in (409, 429) and retries != 0
async def check(table):
merge = table.merge_insert("id").when_not_matched_insert_all().use_index(False)
options = {"timeout": timedelta(seconds=10)}
if retries is not None:
options["retries"] = retries
if expected_success:
result = merge.execute([{"id": 1}], **options)
if not sync_api:
result = await result
assert result.version == 2
else:
with pytest.raises((HttpError, RetryError)):
result = merge.execute([{"id": 1}], **options)
if not sync_api:
await result
if sync_api:
with mock_lancedb_connection(handler, **config) as db:
await check(db.open_table("test", namespace_path=namespace_path))
else:
async with mock_lancedb_connection_async(handler, **config) as db:
await check(await db.open_table("test", namespace_path=namespace_path))
assert attempts == (2 if expected_success else 1)
warnings = [
line
for line in capfd.readouterr().err.splitlines()
if "Retrying merge insert" in line
]
assert len(warnings) == (1 if expected_success else 0)
if expected_success:
assert "WARN" in warnings[0]
assert f"status Some({status}); attempt 2" in warnings[0]
@pytest.mark.parametrize("sync_api", [False, True])
@pytest.mark.asyncio
async def test_merge_retry_override_isolation(sync_api):
attempts = 0
read_attempts = 0
def handler(request):
nonlocal attempts, read_attempts
if request.path.endswith("/describe/"):
send_json(
request,
{
"version": 1,
"schema": {
"fields": [
{"name": "id", "type": {"type": "int64"}, "nullable": True}
]
},
},
)
elif request.path.endswith("/count_rows/"):
read_attempts += 1
send_json(request, 1, 429 if read_attempts % 2 else 200)
else:
assert urlparse(request.path).path.endswith("/merge_insert/")
attempts += 1
send_json(request, {"version": 2}, 429 if attempts in (1, 2, 4, 6) else 200)
async def check(table):
merge = table.merge_insert("id").when_not_matched_insert_all()
for retries in [-1, 256, 1.5, "1"]:
with pytest.raises((OverflowError, TypeError, ValueError)):
result = merge.execute([{"id": 1}], retries=retries)
if not sync_api:
await result
assert attempts == 0
with pytest.raises((HttpError, RetryError)):
result = merge.execute([{"id": 1}], retries=0)
if not sync_api:
await result
assert attempts == 1
for options in ({}, {"retries": 255}):
result = merge.execute([{"id": 1}], **options)
if not sync_api:
result = await result
assert result.version == 2
assert attempts == 5
result = table.count_rows()
if not sync_api:
result = await result
assert result == 1
assert read_attempts == 2
result = (
table.merge_insert("id").when_not_matched_insert_all().execute([{"id": 1}])
)
if not sync_api:
result = await result
assert result.version == 2
assert attempts == 7
config = {"retry_config": {"retries": 3, "backoff_factor": 0, "backoff_jitter": 0}}
if sync_api:
with mock_lancedb_connection(handler, **config) as db:
await check(db.open_table("test"))
else:
async with mock_lancedb_connection_async(handler, **config) as db:
await check(await db.open_table("test"))
@pytest.fixture
def missing_table_handler(cached, namespace_path):
request_ids = []
+3 -2
View File
@@ -3448,7 +3448,8 @@ def test_merge_insert(mem_db: DBConnection):
table.merge_insert("a")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(new_data, timeout=timedelta(seconds=10))
.use_index(False)
.execute(new_data, timeout=timedelta(seconds=10), retries=0)
)
assert merge_insert_res.version == 2
assert merge_insert_res.num_inserted_rows == 1
@@ -3527,7 +3528,7 @@ def test_merge_insert(mem_db: DBConnection):
# timeout
with pytest.raises(Exception, match="merge insert timed out"):
table.merge_insert("a").when_matched_update_all().execute(
new_data, timeout=timedelta(0)
new_data, timeout=timedelta(0), retries=255
)
+5
View File
@@ -2269,10 +2269,12 @@ impl Table {
})
}
#[pyo3(signature = (data, parameters, retries=None))]
pub fn execute_merge_insert<'a>(
self_: PyRef<'a, Self>,
data: Bound<'a, PyAny>,
parameters: MergeInsertParams,
retries: Option<u8>,
) -> PyResult<Bound<'a, PyAny>> {
let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
let on = parameters.on.iter().map(|s| s.as_str()).collect::<Vec<_>>();
@@ -2295,6 +2297,9 @@ impl Table {
if let Some(timeout) = parameters.timeout {
builder.timeout(timeout);
}
if let Some(retries) = retries {
builder.retries(retries);
}
if let Some(use_index) = parameters.use_index {
builder.use_index(use_index);
}
+10
View File
@@ -3707,7 +3707,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
return Ok(merge_result);
}
Err(err) if rescannable && self.is_retryable_write_error(&err) => {
let status = err.status_code();
retry_counter.increment_from_error(err)?;
log::warn!(
"Retrying merge insert {} after status {:?}; attempt {}",
retry_counter.request_id,
status.map(|s| s.as_u16()),
u16::from(retry_counter.request_failures)
+ u16::from(retry_counter.connect_failures)
+ u16::from(retry_counter.read_failures)
+ 1,
);
tokio::time::sleep(retry_counter.next_sleep_time()).await;
merge = merge.reset_state()?;
continue;