mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: expose per-call merge retry controls in Python and Node (#5015)
Python and Node callers could not disable precommit retries for one
merge without changing the connection. Add `execute(..., retries=0)` for
Python sync/async and `execute(data, { retries: 0 })` for TypeScript,
forwarding into the existing Rust control without persisting the
override. Ambiguous 5xx outcomes remain non-replayable; permitted merge
retries now log status and attempt at WARN, and the API docs explain
retry budgets and timeout boundaries.
Validation at `6001300` passed 70 Python HTTP cases, 36 Node HTTP cases,
14 local merge regressions per binding, and 34 scoped Rust tests, with
rebuilt native extensions, TypeScript compilation, Clippy, scoped
formatting/lint, and generated API docs. The public API and feature diff
are unchanged after integrating main and the reviewed ingestion guide;
that binding evidence is retained. Focused merged-tree validation
additionally passed 267 Rust merge/retry/provenance cases and the CI
fixtures. Existing root Ruff issues in docs/web-tests remain; local
fixtures do not establish live proxy/server cancellation behavior.
Refs #4256 and OSS-2370.
This commit is contained in:
@@ -8,6 +8,26 @@
|
||||
|
||||
## Properties
|
||||
|
||||
### retries?
|
||||
|
||||
```ts
|
||||
optional retries: number;
|
||||
```
|
||||
|
||||
Override the remote retry budget for this execution only (an integer from
|
||||
0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
|
||||
precommit conflicts (409) or rate limits (429). This does not change
|
||||
connection-wide/read budgets or local transaction conflict retries.
|
||||
|
||||
By default, remote calls use the connection's `clientConfig.retryConfig`
|
||||
([RetryConfig](RetryConfig.md)). Its `statuses` (also set through
|
||||
`LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
|
||||
it never permits replay of ambiguous merge outcomes. In particular, 502,
|
||||
ordinary 503, and 504 are never replayed, even if configured. Reconcile table
|
||||
contents before manually repeating a merge whose commit outcome is unknown.
|
||||
|
||||
***
|
||||
|
||||
### timeoutMs?
|
||||
|
||||
```ts
|
||||
@@ -24,3 +44,6 @@ the second attempt will be cancelled after 10 seconds, hitting the
|
||||
first attempt will not be cancelled.
|
||||
|
||||
When this is set, the timeout is enforced on all attempts, including the first.
|
||||
For remote tables this is sent as a server execution timeout. It does not
|
||||
extend HTTP read or proxy idle timeouts, or prove that a failed request did
|
||||
not commit.
|
||||
|
||||
@@ -221,6 +221,13 @@ and check inserted or deleted counts when the SDK returns them. Repeating a
|
||||
merge is only safe when its keys, conditions and update values produce the
|
||||
intended result on retry.
|
||||
|
||||
To override the transport retry budget for one merge, pass `retries=0` to
|
||||
Python's synchronous or asynchronous `builder.execute(data, retries=0)`, or
|
||||
use TypeScript's `builder.execute(data, { retries: 0 })`. The override applies
|
||||
only to that execution. A larger budget does not make an ambiguous 5xx result
|
||||
safe to replay. See [write retry boundaries](/ingestion/insert#performance)
|
||||
for the distinction between retryable failures and an unknown commit outcome.
|
||||
|
||||
### Embeddings and expression predicates {#advanced}
|
||||
|
||||
When a table uses an [embedding definition](/embedding), behavior depends on
|
||||
|
||||
@@ -542,6 +542,163 @@ describe("remote connection", () => {
|
||||
expect(attempts).toBe(scenario.attempts);
|
||||
});
|
||||
|
||||
it.each(
|
||||
[409, 429, 502, 503, 504].flatMap((status) =>
|
||||
[undefined, 0, 2].flatMap((retries) =>
|
||||
[[], ["analytics", "features"]].map((namespacePath) => ({
|
||||
status,
|
||||
retries,
|
||||
namespacePath,
|
||||
})),
|
||||
),
|
||||
),
|
||||
)("applies per-call merge retries: %j", async (scenario) => {
|
||||
let attempts = 0;
|
||||
const identifier = [...scenario.namespacePath, "items"].join("$");
|
||||
const succeeds =
|
||||
[409, 429].includes(scenario.status) && scenario.retries !== 0;
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const url = new URL(req.url ?? "", "http://localhost");
|
||||
if (url.pathname === `/v1/table/${identifier}/describe/`) {
|
||||
res.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "id", type: { type: "int64" }, nullable: true },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
expect(url.pathname).toBe(`/v1/table/${identifier}/merge_insert/`);
|
||||
expect(url.searchParams.get("use_index")).toBe("false");
|
||||
expect(req.headers["x-request-timeout-ms"]).toBe("10000");
|
||||
// Rust compresses the batches; Arrow JS can inspect the schema without a codec.
|
||||
const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
|
||||
expect(reader.schema.fields.map((field) => field.name)).toEqual([
|
||||
"id",
|
||||
]);
|
||||
} finally {
|
||||
attempts++;
|
||||
res
|
||||
.writeHead(attempts === 1 ? scenario.status : 200)
|
||||
.end('{"version":2}');
|
||||
}
|
||||
});
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("items", scenario.namespacePath);
|
||||
const merge = table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.useIndex(false);
|
||||
const result = merge.execute([{ id: 1n }], {
|
||||
timeoutMs: 10_000,
|
||||
retries: scenario.retries,
|
||||
});
|
||||
if (succeeds) {
|
||||
await expect(result).resolves.toMatchObject({ version: 2 });
|
||||
} else {
|
||||
await expect(result).rejects.toThrow();
|
||||
}
|
||||
},
|
||||
{
|
||||
clientConfig: {
|
||||
retryConfig: {
|
||||
retries: 3,
|
||||
statuses: [409, 429, 502, 503, 504],
|
||||
backoffFactor: 0,
|
||||
backoffJitter: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(attempts).toBe(succeeds ? 2 : 1);
|
||||
});
|
||||
|
||||
it("isolates merge retry overrides across reused builders and reads", async () => {
|
||||
let attempts = 0;
|
||||
let readAttempts = 0;
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
if (req.url?.endsWith("/describe/")) {
|
||||
res.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "id", type: { type: "int64" }, nullable: true },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
req.resume();
|
||||
req.on("end", () => {
|
||||
if (req.url?.endsWith("/count_rows/")) {
|
||||
readAttempts++;
|
||||
res.writeHead(readAttempts % 2 ? 429 : 200).end("1");
|
||||
} else {
|
||||
attempts++;
|
||||
res
|
||||
.writeHead([1, 2, 4, 6].includes(attempts) ? 429 : 200)
|
||||
.end('{"version":2}');
|
||||
expect(req.url).toContain("/merge_insert/");
|
||||
}
|
||||
});
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("items");
|
||||
const merge = table.mergeInsert("id").whenNotMatchedInsertAll();
|
||||
for (const retries of [
|
||||
-1,
|
||||
256,
|
||||
1.5,
|
||||
Number.NaN,
|
||||
Number.POSITIVE_INFINITY,
|
||||
]) {
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries }),
|
||||
).rejects.toThrow("retries must be an integer between 0 and 255");
|
||||
}
|
||||
expect(attempts).toBe(0);
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries: 0 }),
|
||||
).rejects.toThrow();
|
||||
expect(attempts).toBe(1);
|
||||
await expect(merge.execute([{ id: 1n }])).resolves.toMatchObject({
|
||||
version: 2,
|
||||
});
|
||||
await expect(
|
||||
merge.execute([{ id: 1n }], { retries: 255 }),
|
||||
).resolves.toMatchObject({ version: 2 });
|
||||
expect(attempts).toBe(5);
|
||||
await expect(table.countRows()).resolves.toBe(1);
|
||||
expect(readAttempts).toBe(2);
|
||||
await expect(
|
||||
table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.execute([{ id: 1n }]),
|
||||
).resolves.toMatchObject({ version: 2 });
|
||||
expect(attempts).toBe(7);
|
||||
},
|
||||
{
|
||||
clientConfig: {
|
||||
retryConfig: { retries: 3, backoffFactor: 0, backoffJitter: 0 },
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept partial connection options", async () => {
|
||||
await connect("db://test", {
|
||||
apiKey: "fake",
|
||||
|
||||
@@ -938,7 +938,8 @@ describe("merge insert", () => {
|
||||
.mergeInsert("a")
|
||||
.whenMatchedUpdateAll()
|
||||
.whenNotMatchedInsertAll()
|
||||
.execute(newData, { timeoutMs: 10_000 });
|
||||
.useIndex(false)
|
||||
.execute(newData, { timeoutMs: 10_000, retries: 0 });
|
||||
expect(mergeInsertRes).toHaveProperty("version");
|
||||
expect(mergeInsertRes.version).toBe(2);
|
||||
expect(mergeInsertRes.numInsertedRows).toBe(1);
|
||||
|
||||
+27
-1
@@ -127,6 +127,14 @@ export class MergeInsertBuilder {
|
||||
data: Data,
|
||||
execOptions?: Partial<WriteExecutionOptions>,
|
||||
): Promise<MergeResult> {
|
||||
if (
|
||||
execOptions?.retries !== undefined &&
|
||||
(!Number.isInteger(execOptions.retries) ||
|
||||
execOptions.retries < 0 ||
|
||||
execOptions.retries > 255)
|
||||
) {
|
||||
throw new Error("retries must be an integer between 0 and 255");
|
||||
}
|
||||
let schema: Schema;
|
||||
if (this.#schema instanceof Promise) {
|
||||
schema = await this.#schema;
|
||||
@@ -140,7 +148,7 @@ export class MergeInsertBuilder {
|
||||
}
|
||||
|
||||
const buffer = await fromDataToBuffer(data, undefined, schema);
|
||||
return await this.#native.execute(buffer);
|
||||
return await this.#native.execute(buffer, execOptions?.retries);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,6 +164,24 @@ export interface WriteExecutionOptions {
|
||||
* first attempt will not be cancelled.
|
||||
*
|
||||
* When this is set, the timeout is enforced on all attempts, including the first.
|
||||
* For remote tables this is sent as a server execution timeout. It does not
|
||||
* extend HTTP read or proxy idle timeouts, or prove that a failed request did
|
||||
* not commit.
|
||||
*/
|
||||
timeoutMs?: number;
|
||||
|
||||
/**
|
||||
* Override the remote retry budget for this execution only (an integer from
|
||||
* 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
|
||||
* precommit conflicts (409) or rate limits (429). This does not change
|
||||
* connection-wide/read budgets or local transaction conflict retries.
|
||||
*
|
||||
* By default, remote calls use the connection's `clientConfig.retryConfig`
|
||||
* ({@link RetryConfig}). Its `statuses` (also set through
|
||||
* `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
|
||||
* it never permits replay of ambiguous merge outcomes. In particular, 502,
|
||||
* ordinary 503, and 504 are never replayed, even if configured. Reconcile table
|
||||
* contents before manually repeating a merge whose commit outcome is unknown.
|
||||
*/
|
||||
retries?: number;
|
||||
}
|
||||
|
||||
+7
-2
@@ -65,12 +65,17 @@ impl NativeMergeInsertBuilder {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||
pub async fn execute(&self, buf: Buffer, retries: Option<u32>) -> napi::Result<MergeResult> {
|
||||
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
|
||||
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
|
||||
})?;
|
||||
|
||||
let this = self.clone();
|
||||
let mut this = self.clone();
|
||||
if let Some(retries) = retries {
|
||||
let retries = u8::try_from(retries)
|
||||
.map_err(|_| napi::Error::from_reason("retries must be between 0 and 255"))?;
|
||||
this.inner.retries(retries);
|
||||
}
|
||||
|
||||
let res = this.inner.execute(data).await.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
|
||||
@@ -565,6 +565,12 @@ class Table:
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
retries: Optional[int] = None,
|
||||
) -> AddResult: ...
|
||||
async def execute_merge_insert(
|
||||
self,
|
||||
data: pa.RecordBatchReader,
|
||||
parameters: Dict[str, Any],
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeResult: ...
|
||||
async def update(
|
||||
self, updates: Dict[str, str], where: Optional[str]
|
||||
) -> UpdateResult: ...
|
||||
|
||||
@@ -161,6 +161,8 @@ class LanceMergeInsertBuilder(object):
|
||||
on_bad_vectors: str = "error",
|
||||
fill_value: float = 0.0,
|
||||
timeout: Optional[timedelta] = None,
|
||||
*,
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeInsertResult:
|
||||
"""
|
||||
Executes the merge insert operation
|
||||
@@ -189,7 +191,22 @@ class LanceMergeInsertBuilder(object):
|
||||
first attempt will not be cancelled.
|
||||
|
||||
When this is set, the timeout is enforced on all attempts, including
|
||||
the first.
|
||||
the first. For remote tables it is sent as a server execution timeout;
|
||||
it does not extend HTTP read or proxy idle timeouts, or prove that a
|
||||
failed request did not commit.
|
||||
retries: int, optional
|
||||
Override the remote retry budget for this execution only (0 to 255).
|
||||
Use ``execute(new_data, retries=0)`` to disable even retries after
|
||||
precommit conflicts (409) or rate limits (429). This does not change
|
||||
connection-wide/read budgets or local transaction conflict retries.
|
||||
|
||||
By default, remote calls use the connection's
|
||||
[RetryConfig][lancedb.remote.RetryConfig]. Its ``statuses`` (also set
|
||||
through ``LANCE_CLIENT_RETRY_STATUSES``) selects candidates within
|
||||
the retry budget; it never permits replay of ambiguous merge outcomes.
|
||||
In particular, 502, ordinary 503, and 504 are never replayed, even if
|
||||
configured. Reconcile table contents before manually repeating a merge
|
||||
whose commit outcome is unknown.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -198,4 +215,6 @@ class LanceMergeInsertBuilder(object):
|
||||
"""
|
||||
if timeout is not None:
|
||||
self._timeout = timeout
|
||||
return self._table._do_merge(self, new_data, on_bad_vectors, fill_value)
|
||||
return self._table._do_merge(
|
||||
self, new_data, on_bad_vectors, fill_value, retries=retries
|
||||
)
|
||||
|
||||
@@ -945,9 +945,10 @@ class RemoteTable(Table):
|
||||
new_data: DATA,
|
||||
on_bad_vectors: str,
|
||||
fill_value: float,
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeResult:
|
||||
return LOOP.run(
|
||||
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
|
||||
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries)
|
||||
)
|
||||
|
||||
@_deprecated_keyword("predicate", "where")
|
||||
|
||||
@@ -2810,6 +2810,7 @@ class Table(ABC):
|
||||
new_data: DATA,
|
||||
on_bad_vectors: OnBadVectorsType,
|
||||
fill_value: float,
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeResult: ...
|
||||
|
||||
@abstractmethod
|
||||
@@ -5443,9 +5444,10 @@ class LanceTable(Table):
|
||||
new_data: DATA,
|
||||
on_bad_vectors: OnBadVectorsType,
|
||||
fill_value: float,
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeResult:
|
||||
return LOOP.run(
|
||||
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
|
||||
self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries)
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -7777,6 +7779,7 @@ class AsyncTable:
|
||||
new_data: DATA,
|
||||
on_bad_vectors: OnBadVectorsType,
|
||||
fill_value: float,
|
||||
retries: Optional[int] = None,
|
||||
) -> MergeResult:
|
||||
schema = await self.schema()
|
||||
if on_bad_vectors is None:
|
||||
@@ -7808,6 +7811,7 @@ class AsyncTable:
|
||||
use_rte=merge._use_rte,
|
||||
validate_single_shard=merge._validate_single_shard,
|
||||
),
|
||||
retries=retries,
|
||||
)
|
||||
|
||||
async def delete(self, where: Union[str, Expr]) -> DeleteResult:
|
||||
|
||||
@@ -310,6 +310,164 @@ async def test_add_retry_override(sync_api, retries, status):
|
||||
assert attempts == (2 if expected_success else 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_api", [False, True])
|
||||
@pytest.mark.parametrize("namespace_path", [[], ["analytics", "features"]])
|
||||
@pytest.mark.parametrize("retries", [None, 0, 2])
|
||||
@pytest.mark.parametrize("status", [409, 429, 502, 503, 504])
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_retry_override(sync_api, namespace_path, retries, status, capfd):
|
||||
attempts = 0
|
||||
identifier = "$".join(namespace_path + ["test"])
|
||||
|
||||
def handler(request):
|
||||
nonlocal attempts
|
||||
if request.path == f"/v1/table/{identifier}/describe/":
|
||||
send_json(
|
||||
request,
|
||||
{
|
||||
"version": 1,
|
||||
"schema": {
|
||||
"fields": [
|
||||
{"name": "id", "type": {"type": "int64"}, "nullable": True}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
return
|
||||
assert urlparse(request.path).path == f"/v1/table/{identifier}/merge_insert/"
|
||||
query = parse_qs(urlparse(request.path).query)
|
||||
assert query["use_index"] == ["false"]
|
||||
assert request.headers["x-request-timeout-ms"] == "10000"
|
||||
body = bytearray()
|
||||
if request.headers.get("Transfer-Encoding") == "chunked":
|
||||
while True:
|
||||
size = int(request.rfile.readline().strip(), 16)
|
||||
if size == 0:
|
||||
request.rfile.readline()
|
||||
break
|
||||
body.extend(request.rfile.read(size))
|
||||
request.rfile.read(2)
|
||||
else:
|
||||
body.extend(
|
||||
request.rfile.read(int(request.headers.get("Content-Length", 0)))
|
||||
)
|
||||
assert pa.ipc.open_stream(body).read_all().to_pylist() == [{"id": 1}]
|
||||
attempts += 1
|
||||
send_json(request, {"version": 2}, status if attempts == 1 else 200)
|
||||
|
||||
config = {
|
||||
"retry_config": {
|
||||
"retries": 3,
|
||||
"statuses": [409, 429, 502, 503, 504],
|
||||
"backoff_factor": 0,
|
||||
"backoff_jitter": 0,
|
||||
}
|
||||
}
|
||||
expected_success = status in (409, 429) and retries != 0
|
||||
|
||||
async def check(table):
|
||||
merge = table.merge_insert("id").when_not_matched_insert_all().use_index(False)
|
||||
options = {"timeout": timedelta(seconds=10)}
|
||||
if retries is not None:
|
||||
options["retries"] = retries
|
||||
if expected_success:
|
||||
result = merge.execute([{"id": 1}], **options)
|
||||
if not sync_api:
|
||||
result = await result
|
||||
assert result.version == 2
|
||||
else:
|
||||
with pytest.raises((HttpError, RetryError)):
|
||||
result = merge.execute([{"id": 1}], **options)
|
||||
if not sync_api:
|
||||
await result
|
||||
|
||||
if sync_api:
|
||||
with mock_lancedb_connection(handler, **config) as db:
|
||||
await check(db.open_table("test", namespace_path=namespace_path))
|
||||
else:
|
||||
async with mock_lancedb_connection_async(handler, **config) as db:
|
||||
await check(await db.open_table("test", namespace_path=namespace_path))
|
||||
assert attempts == (2 if expected_success else 1)
|
||||
warnings = [
|
||||
line
|
||||
for line in capfd.readouterr().err.splitlines()
|
||||
if "Retrying merge insert" in line
|
||||
]
|
||||
assert len(warnings) == (1 if expected_success else 0)
|
||||
if expected_success:
|
||||
assert "WARN" in warnings[0]
|
||||
assert f"status Some({status}); attempt 2" in warnings[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_api", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_retry_override_isolation(sync_api):
|
||||
attempts = 0
|
||||
read_attempts = 0
|
||||
|
||||
def handler(request):
|
||||
nonlocal attempts, read_attempts
|
||||
if request.path.endswith("/describe/"):
|
||||
send_json(
|
||||
request,
|
||||
{
|
||||
"version": 1,
|
||||
"schema": {
|
||||
"fields": [
|
||||
{"name": "id", "type": {"type": "int64"}, "nullable": True}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
elif request.path.endswith("/count_rows/"):
|
||||
read_attempts += 1
|
||||
send_json(request, 1, 429 if read_attempts % 2 else 200)
|
||||
else:
|
||||
assert urlparse(request.path).path.endswith("/merge_insert/")
|
||||
attempts += 1
|
||||
send_json(request, {"version": 2}, 429 if attempts in (1, 2, 4, 6) else 200)
|
||||
|
||||
async def check(table):
|
||||
merge = table.merge_insert("id").when_not_matched_insert_all()
|
||||
for retries in [-1, 256, 1.5, "1"]:
|
||||
with pytest.raises((OverflowError, TypeError, ValueError)):
|
||||
result = merge.execute([{"id": 1}], retries=retries)
|
||||
if not sync_api:
|
||||
await result
|
||||
assert attempts == 0
|
||||
with pytest.raises((HttpError, RetryError)):
|
||||
result = merge.execute([{"id": 1}], retries=0)
|
||||
if not sync_api:
|
||||
await result
|
||||
assert attempts == 1
|
||||
for options in ({}, {"retries": 255}):
|
||||
result = merge.execute([{"id": 1}], **options)
|
||||
if not sync_api:
|
||||
result = await result
|
||||
assert result.version == 2
|
||||
assert attempts == 5
|
||||
result = table.count_rows()
|
||||
if not sync_api:
|
||||
result = await result
|
||||
assert result == 1
|
||||
assert read_attempts == 2
|
||||
result = (
|
||||
table.merge_insert("id").when_not_matched_insert_all().execute([{"id": 1}])
|
||||
)
|
||||
if not sync_api:
|
||||
result = await result
|
||||
assert result.version == 2
|
||||
assert attempts == 7
|
||||
|
||||
config = {"retry_config": {"retries": 3, "backoff_factor": 0, "backoff_jitter": 0}}
|
||||
if sync_api:
|
||||
with mock_lancedb_connection(handler, **config) as db:
|
||||
await check(db.open_table("test"))
|
||||
else:
|
||||
async with mock_lancedb_connection_async(handler, **config) as db:
|
||||
await check(await db.open_table("test"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def missing_table_handler(cached, namespace_path):
|
||||
request_ids = []
|
||||
|
||||
@@ -3448,7 +3448,8 @@ def test_merge_insert(mem_db: DBConnection):
|
||||
table.merge_insert("a")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(new_data, timeout=timedelta(seconds=10))
|
||||
.use_index(False)
|
||||
.execute(new_data, timeout=timedelta(seconds=10), retries=0)
|
||||
)
|
||||
assert merge_insert_res.version == 2
|
||||
assert merge_insert_res.num_inserted_rows == 1
|
||||
@@ -3527,7 +3528,7 @@ def test_merge_insert(mem_db: DBConnection):
|
||||
# timeout
|
||||
with pytest.raises(Exception, match="merge insert timed out"):
|
||||
table.merge_insert("a").when_matched_update_all().execute(
|
||||
new_data, timeout=timedelta(0)
|
||||
new_data, timeout=timedelta(0), retries=255
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2269,10 +2269,12 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (data, parameters, retries=None))]
|
||||
pub fn execute_merge_insert<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
data: Bound<'a, PyAny>,
|
||||
parameters: MergeInsertParams,
|
||||
retries: Option<u8>,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
|
||||
let on = parameters.on.iter().map(|s| s.as_str()).collect::<Vec<_>>();
|
||||
@@ -2295,6 +2297,9 @@ impl Table {
|
||||
if let Some(timeout) = parameters.timeout {
|
||||
builder.timeout(timeout);
|
||||
}
|
||||
if let Some(retries) = retries {
|
||||
builder.retries(retries);
|
||||
}
|
||||
if let Some(use_index) = parameters.use_index {
|
||||
builder.use_index(use_index);
|
||||
}
|
||||
|
||||
@@ -3707,7 +3707,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
return Ok(merge_result);
|
||||
}
|
||||
Err(err) if rescannable && self.is_retryable_write_error(&err) => {
|
||||
let status = err.status_code();
|
||||
retry_counter.increment_from_error(err)?;
|
||||
log::warn!(
|
||||
"Retrying merge insert {} after status {:?}; attempt {}",
|
||||
retry_counter.request_id,
|
||||
status.map(|s| s.as_u16()),
|
||||
u16::from(retry_counter.request_failures)
|
||||
+ u16::from(retry_counter.connect_failures)
|
||||
+ u16::from(retry_counter.read_failures)
|
||||
+ 1,
|
||||
);
|
||||
tokio::time::sleep(retry_counter.next_sleep_time()).await;
|
||||
merge = merge.reset_state()?;
|
||||
continue;
|
||||
|
||||
Reference in New Issue
Block a user