feat: expose per-call merge retry controls in Python and Node (#5015)

Python and Node callers could not disable precommit retries for one
merge without changing the connection. Add `execute(..., retries=0)` for
Python sync/async and `execute(data, { retries: 0 })` for TypeScript,
forwarding into the existing Rust control without persisting the
override. Ambiguous 5xx outcomes remain non-replayable; permitted merge
retries now log status and attempt at WARN, and the API docs explain
retry budgets and timeout boundaries.

Validation at `6001300` passed 70 Python HTTP cases, 36 Node HTTP cases,
14 local merge regressions per binding, and 34 scoped Rust tests, with
rebuilt native extensions, TypeScript compilation, Clippy, scoped
formatting/lint, and generated API docs. The public API and feature diff
are unchanged after integrating main and the reviewed ingestion guide;
that binding evidence is retained. Focused merged-tree validation
additionally passed 267 Rust merge/retry/provenance cases and the CI
fixtures. Existing root Ruff issues in docs/web-tests remain; local
fixtures do not establish live proxy/server cancellation behavior.

Refs #4256 and OSS-2370.
This commit is contained in:
Xuanwo
2026-11-14 22:43:56 +08:00
committed by GitHub
parent dc9eb269b1
commit e2f836da99
14 changed files with 433 additions and 10 deletions
+157
View File
@@ -542,6 +542,163 @@ describe("remote connection", () => {
expect(attempts).toBe(scenario.attempts);
});
it.each(
[409, 429, 502, 503, 504].flatMap((status) =>
[undefined, 0, 2].flatMap((retries) =>
[[], ["analytics", "features"]].map((namespacePath) => ({
status,
retries,
namespacePath,
})),
),
),
)("applies per-call merge retries: %j", async (scenario) => {
let attempts = 0;
const identifier = [...scenario.namespacePath, "items"].join("$");
const succeeds =
[409, 429].includes(scenario.status) && scenario.retries !== 0;
await withMockDatabase(
(req, res) => {
const url = new URL(req.url ?? "", "http://localhost");
if (url.pathname === `/v1/table/${identifier}/describe/`) {
res.writeHead(200).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: true },
],
},
}),
);
return;
}
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
try {
expect(url.pathname).toBe(`/v1/table/${identifier}/merge_insert/`);
expect(url.searchParams.get("use_index")).toBe("false");
expect(req.headers["x-request-timeout-ms"]).toBe("10000");
// Rust compresses the batches; Arrow JS can inspect the schema without a codec.
const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
expect(reader.schema.fields.map((field) => field.name)).toEqual([
"id",
]);
} finally {
attempts++;
res
.writeHead(attempts === 1 ? scenario.status : 200)
.end('{"version":2}');
}
});
},
async (db) => {
const table = await db.openTable("items", scenario.namespacePath);
const merge = table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useIndex(false);
const result = merge.execute([{ id: 1n }], {
timeoutMs: 10_000,
retries: scenario.retries,
});
if (succeeds) {
await expect(result).resolves.toMatchObject({ version: 2 });
} else {
await expect(result).rejects.toThrow();
}
},
{
clientConfig: {
retryConfig: {
retries: 3,
statuses: [409, 429, 502, 503, 504],
backoffFactor: 0,
backoffJitter: 0,
},
},
},
);
expect(attempts).toBe(succeeds ? 2 : 1);
});
it("isolates merge retry overrides across reused builders and reads", async () => {
let attempts = 0;
let readAttempts = 0;
await withMockDatabase(
(req, res) => {
if (req.url?.endsWith("/describe/")) {
res.writeHead(200).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: true },
],
},
}),
);
return;
}
req.resume();
req.on("end", () => {
if (req.url?.endsWith("/count_rows/")) {
readAttempts++;
res.writeHead(readAttempts % 2 ? 429 : 200).end("1");
} else {
attempts++;
res
.writeHead([1, 2, 4, 6].includes(attempts) ? 429 : 200)
.end('{"version":2}');
expect(req.url).toContain("/merge_insert/");
}
});
},
async (db) => {
const table = await db.openTable("items");
const merge = table.mergeInsert("id").whenNotMatchedInsertAll();
for (const retries of [
-1,
256,
1.5,
Number.NaN,
Number.POSITIVE_INFINITY,
]) {
await expect(
merge.execute([{ id: 1n }], { retries }),
).rejects.toThrow("retries must be an integer between 0 and 255");
}
expect(attempts).toBe(0);
await expect(
merge.execute([{ id: 1n }], { retries: 0 }),
).rejects.toThrow();
expect(attempts).toBe(1);
await expect(merge.execute([{ id: 1n }])).resolves.toMatchObject({
version: 2,
});
await expect(
merge.execute([{ id: 1n }], { retries: 255 }),
).resolves.toMatchObject({ version: 2 });
expect(attempts).toBe(5);
await expect(table.countRows()).resolves.toBe(1);
expect(readAttempts).toBe(2);
await expect(
table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.execute([{ id: 1n }]),
).resolves.toMatchObject({ version: 2 });
expect(attempts).toBe(7);
},
{
clientConfig: {
retryConfig: { retries: 3, backoffFactor: 0, backoffJitter: 0 },
},
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
+2 -1
View File
@@ -938,7 +938,8 @@ describe("merge insert", () => {
.mergeInsert("a")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(newData, { timeoutMs: 10_000 });
.useIndex(false)
.execute(newData, { timeoutMs: 10_000, retries: 0 });
expect(mergeInsertRes).toHaveProperty("version");
expect(mergeInsertRes.version).toBe(2);
expect(mergeInsertRes.numInsertedRows).toBe(1);
+27 -1
View File
@@ -127,6 +127,14 @@ export class MergeInsertBuilder {
data: Data,
execOptions?: Partial<WriteExecutionOptions>,
): Promise<MergeResult> {
if (
execOptions?.retries !== undefined &&
(!Number.isInteger(execOptions.retries) ||
execOptions.retries < 0 ||
execOptions.retries > 255)
) {
throw new Error("retries must be an integer between 0 and 255");
}
let schema: Schema;
if (this.#schema instanceof Promise) {
schema = await this.#schema;
@@ -140,7 +148,7 @@ export class MergeInsertBuilder {
}
const buffer = await fromDataToBuffer(data, undefined, schema);
return await this.#native.execute(buffer);
return await this.#native.execute(buffer, execOptions?.retries);
}
}
@@ -156,6 +164,24 @@ export interface WriteExecutionOptions {
* first attempt will not be cancelled.
*
* When this is set, the timeout is enforced on all attempts, including the first.
* For remote tables this is sent as a server execution timeout. It does not
* extend HTTP read or proxy idle timeouts, or prove that a failed request did
* not commit.
*/
timeoutMs?: number;
/**
* Override the remote retry budget for this execution only (an integer from
* 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
* precommit conflicts (409) or rate limits (429). This does not change
* connection-wide/read budgets or local transaction conflict retries.
*
* By default, remote calls use the connection's `clientConfig.retryConfig`
* ({@link RetryConfig}). Its `statuses` (also set through
* `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
* it never permits replay of ambiguous merge outcomes. In particular, 502,
* ordinary 503, and 504 are never replayed, even if configured. Reconcile table
* contents before manually repeating a merge whose commit outcome is unknown.
*/
retries?: number;
}
+7 -2
View File
@@ -65,12 +65,17 @@ impl NativeMergeInsertBuilder {
}
#[napi(catch_unwind)]
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
pub async fn execute(&self, buf: Buffer, retries: Option<u32>) -> napi::Result<MergeResult> {
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
})?;
let this = self.clone();
let mut this = self.clone();
if let Some(retries) = retries {
let retries = u8::try_from(retries)
.map_err(|_| napi::Error::from_reason("retries must be between 0 and 255"))?;
this.inner.retries(retries);
}
let res = this.inner.execute(data).await.map_err(|e| {
napi::Error::from_reason(format!(