mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: expose per-call merge retry controls in Python and Node (#5015)
Python and Node callers could not disable precommit retries for one
merge without changing the connection. Add `execute(..., retries=0)` for
Python sync/async and `execute(data, { retries: 0 })` for TypeScript,
forwarding into the existing Rust control without persisting the
override. Ambiguous 5xx outcomes remain non-replayable; permitted merge
retries now log status and attempt at WARN, and the API docs explain
retry budgets and timeout boundaries.
Validation at `6001300` passed 70 Python HTTP cases, 36 Node HTTP cases,
14 local merge regressions per binding, and 34 scoped Rust tests, with
rebuilt native extensions, TypeScript compilation, Clippy, scoped
formatting/lint, and generated API docs. The public API and feature diff
are unchanged after integrating main and the reviewed ingestion guide;
that binding evidence is retained. Focused merged-tree validation
additionally passed 267 Rust merge/retry/provenance cases and the CI
fixtures. Existing root Ruff issues in docs/web-tests remain; local
fixtures do not establish live proxy/server cancellation behavior.
Refs #4256 and OSS-2370.
This commit is contained in:
@@ -542,6 +542,163 @@ describe("remote connection", () => {
|
||||
expect(attempts).toBe(scenario.attempts);
|
||||
});
|
||||
|
||||
it.each(
|
||||
[409, 429, 502, 503, 504].flatMap((status) =>
|
||||
[undefined, 0, 2].flatMap((retries) =>
|
||||
[[], ["analytics", "features"]].map((namespacePath) => ({
|
||||
status,
|
||||
retries,
|
||||
namespacePath,
|
||||
})),
|
||||
),
|
||||
),
|
||||
)("applies per-call merge retries: %j", async (scenario) => {
|
||||
let attempts = 0;
|
||||
const identifier = [...scenario.namespacePath, "items"].join("$");
|
||||
const succeeds =
|
||||
[409, 429].includes(scenario.status) && scenario.retries !== 0;
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const url = new URL(req.url ?? "", "http://localhost");
|
||||
if (url.pathname === `/v1/table/${identifier}/describe/`) {
|
||||
res.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "id", type: { type: "int64" }, nullable: true },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
expect(url.pathname).toBe(`/v1/table/${identifier}/merge_insert/`);
|
||||
expect(url.searchParams.get("use_index")).toBe("false");
|
||||
expect(req.headers["x-request-timeout-ms"]).toBe("10000");
|
||||
// Rust compresses the batches; Arrow JS can inspect the schema without a codec.
|
||||
const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
|
||||
expect(reader.schema.fields.map((field) => field.name)).toEqual([
|
||||
"id",
|
||||
]);
|
||||
} finally {
|
||||
attempts++;
|
||||
res
|
||||
.writeHead(attempts === 1 ? scenario.status : 200)
|
||||
.end('{"version":2}');
|
||||
}
|
||||
});
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("items", scenario.namespacePath);
|
||||
const merge = table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.useIndex(false);
|
||||
const result = merge.execute([{ id: 1n }], {
|
||||
timeoutMs: 10_000,
|
||||
retries: scenario.retries,
|
||||
});
|
||||
if (succeeds) {
|
||||
await expect(result).resolves.toMatchObject({ version: 2 });
|
||||
} else {
|
||||
await expect(result).rejects.toThrow();
|
||||
}
|
||||
},
|
||||
{
|
||||
clientConfig: {
|
||||
retryConfig: {
|
||||
retries: 3,
|
||||
statuses: [409, 429, 502, 503, 504],
|
||||
backoffFactor: 0,
|
||||
backoffJitter: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(attempts).toBe(succeeds ? 2 : 1);
|
||||
});
|
||||
|
||||
it("isolates merge retry overrides across reused builders and reads", async () => {
|
||||
let attempts = 0;
|
||||
let readAttempts = 0;
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
if (req.url?.endsWith("/describe/")) {
|
||||
res.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "id", type: { type: "int64" }, nullable: true },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
req.resume();
|
||||
req.on("end", () => {
|
||||
if (req.url?.endsWith("/count_rows/")) {
|
||||
readAttempts++;
|
||||
res.writeHead(readAttempts % 2 ? 429 : 200).end("1");
|
||||
} else {
|
||||
attempts++;
|
||||
res
|
||||
.writeHead([1, 2, 4, 6].includes(attempts) ? 429 : 200)
|
||||
.end('{"version":2}');
|
||||
expect(req.url).toContain("/merge_insert/");
|
||||
}
|
||||
});
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("items");
|
||||
const merge = table.mergeInsert("id").whenNotMatchedInsertAll();
|
||||
for (const retries of [
|
||||
-1,
|
||||
256,
|
||||
1.5,
|
||||
Number.NaN,
|
||||
Number.POSITIVE_INFINITY,
|
||||
]) {
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries }),
|
||||
).rejects.toThrow("retries must be an integer between 0 and 255");
|
||||
}
|
||||
expect(attempts).toBe(0);
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries: 0 }),
|
||||
).rejects.toThrow();
|
||||
expect(attempts).toBe(1);
|
||||
await expect(merge.execute([{ id: 1n }])).resolves.toMatchObject({
|
||||
version: 2,
|
||||
});
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries: 255 }),
|
||||
).resolves.toMatchObject({ version: 2 });
|
||||
expect(attempts).toBe(5);
|
||||
await expect(table.countRows()).resolves.toBe(1);
|
||||
expect(readAttempts).toBe(2);
|
||||
await expect(
|
||||
table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.execute([{ id: 1n }]),
|
||||
).resolves.toMatchObject({ version: 2 });
|
||||
expect(attempts).toBe(7);
|
||||
},
|
||||
{
|
||||
clientConfig: {
|
||||
retryConfig: { retries: 3, backoffFactor: 0, backoffJitter: 0 },
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept partial connection options", async () => {
|
||||
await connect("db://test", {
|
||||
apiKey: "fake",
|
||||
|
||||
@@ -938,7 +938,8 @@ describe("merge insert", () => {
|
||||
.mergeInsert("a")
|
||||
.whenMatchedUpdateAll()
|
||||
.whenNotMatchedInsertAll()
|
||||
.execute(newData, { timeoutMs: 10_000 });
|
||||
.useIndex(false)
|
||||
.execute(newData, { timeoutMs: 10_000, retries: 0 });
|
||||
expect(mergeInsertRes).toHaveProperty("version");
|
||||
expect(mergeInsertRes.version).toBe(2);
|
||||
expect(mergeInsertRes.numInsertedRows).toBe(1);
|
||||
|
||||
+27
-1
@@ -127,6 +127,14 @@ export class MergeInsertBuilder {
|
||||
data: Data,
|
||||
execOptions?: Partial<WriteExecutionOptions>,
|
||||
): Promise<MergeResult> {
|
||||
if (
|
||||
execOptions?.retries !== undefined &&
|
||||
(!Number.isInteger(execOptions.retries) ||
|
||||
execOptions.retries < 0 ||
|
||||
execOptions.retries > 255)
|
||||
) {
|
||||
throw new Error("retries must be an integer between 0 and 255");
|
||||
}
|
||||
let schema: Schema;
|
||||
if (this.#schema instanceof Promise) {
|
||||
schema = await this.#schema;
|
||||
@@ -140,7 +148,7 @@ export class MergeInsertBuilder {
|
||||
}
|
||||
|
||||
const buffer = await fromDataToBuffer(data, undefined, schema);
|
||||
return await this.#native.execute(buffer);
|
||||
return await this.#native.execute(buffer, execOptions?.retries);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,6 +164,24 @@ export interface WriteExecutionOptions {
|
||||
* first attempt will not be cancelled.
|
||||
*
|
||||
* When this is set, the timeout is enforced on all attempts, including the first.
|
||||
* For remote tables this is sent as a server execution timeout. It does not
|
||||
* extend HTTP read or proxy idle timeouts, or prove that a failed request did
|
||||
* not commit.
|
||||
*/
|
||||
timeoutMs?: number;
|
||||
|
||||
/**
|
||||
* Override the remote retry budget for this execution only (an integer from
|
||||
* 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
|
||||
* precommit conflicts (409) or rate limits (429). This does not change
|
||||
* connection-wide/read budgets or local transaction conflict retries.
|
||||
*
|
||||
* By default, remote calls use the connection's `clientConfig.retryConfig`
|
||||
* ({@link RetryConfig}). Its `statuses` (also set through
|
||||
* `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
|
||||
* it never permits replay of ambiguous merge outcomes. In particular, 502,
|
||||
* ordinary 503, and 504 are never replayed, even if configured. Reconcile table
|
||||
* contents before manually repeating a merge whose commit outcome is unknown.
|
||||
*/
|
||||
retries?: number;
|
||||
}
|
||||
|
||||
+7
-2
@@ -65,12 +65,17 @@ impl NativeMergeInsertBuilder {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||
pub async fn execute(&self, buf: Buffer, retries: Option<u32>) -> napi::Result<MergeResult> {
|
||||
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
|
||||
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
|
||||
})?;
|
||||
|
||||
let this = self.clone();
|
||||
let mut this = self.clone();
|
||||
if let Some(retries) = retries {
|
||||
let retries = u8::try_from(retries)
|
||||
.map_err(|_| napi::Error::from_reason("retries must be between 0 and 255"))?;
|
||||
this.inner.retries(retries);
|
||||
}
|
||||
|
||||
let res = this.inner.execute(data).await.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
|
||||
Reference in New Issue
Block a user