From e2f836da99deacb6b1a3a9ca7acc1222e7d39d3f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 22:43:56 +0800 Subject: [PATCH] feat: expose per-call merge retry controls in Python and Node (#5015) Python and Node callers could not disable precommit retries for one merge without changing the connection. Add `execute(..., retries=0)` for Python sync/async and `execute(data, { retries: 0 })` for TypeScript, forwarding into the existing Rust control without persisting the override. Ambiguous 5xx outcomes remain non-replayable; permitted merge retries now log status and attempt at WARN, and the API docs explain retry budgets and timeout boundaries. Validation at `6001300` passed 70 Python HTTP cases, 36 Node HTTP cases, 14 local merge regressions per binding, and 34 scoped Rust tests, with rebuilt native extensions, TypeScript compilation, Clippy, scoped formatting/lint, and generated API docs. The public API and feature diff are unchanged after integrating main and the reviewed ingestion guide; that binding evidence is retained. Focused merged-tree validation additionally passed 267 Rust merge/retry/provenance cases and the CI fixtures. Existing root Ruff issues in docs/web-tests remain; local fixtures do not establish live proxy/server cancellation behavior. Refs #4256 and OSS-2370. --- .../js/interfaces/WriteExecutionOptions.md | 23 +++ docs/web/ingestion/merge-insert.mdx | 7 + nodejs/__test__/remote.test.ts | 157 +++++++++++++++++ nodejs/__test__/table.test.ts | 3 +- nodejs/lancedb/merge.ts | 28 +++- nodejs/src/merge.rs | 9 +- python/python/lancedb/_lancedb.pyi | 6 + python/python/lancedb/merge.py | 23 ++- python/python/lancedb/remote/table.py | 3 +- python/python/lancedb/table.py | 6 +- python/python/tests/test_remote_db.py | 158 ++++++++++++++++++ python/python/tests/test_table.py | 5 +- python/src/table.rs | 5 + rust/lancedb/src/remote/table.rs | 10 ++ 14 files changed, 433 insertions(+), 10 deletions(-) diff --git a/docs/src/js/interfaces/WriteExecutionOptions.md b/docs/src/js/interfaces/WriteExecutionOptions.md index 0ca796b6b..9c9be4489 100644 --- a/docs/src/js/interfaces/WriteExecutionOptions.md +++ b/docs/src/js/interfaces/WriteExecutionOptions.md @@ -8,6 +8,26 @@ ## Properties +### retries? + +```ts +optional retries: number; +``` + +Override the remote retry budget for this execution only (an integer from +0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after +precommit conflicts (409) or rate limits (429). This does not change +connection-wide/read budgets or local transaction conflict retries. + +By default, remote calls use the connection's `clientConfig.retryConfig` +([RetryConfig](RetryConfig.md)). Its `statuses` (also set through +`LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget; +it never permits replay of ambiguous merge outcomes. In particular, 502, +ordinary 503, and 504 are never replayed, even if configured. Reconcile table +contents before manually repeating a merge whose commit outcome is unknown. + +*** + ### timeoutMs? ```ts @@ -24,3 +44,6 @@ the second attempt will be cancelled after 10 seconds, hitting the first attempt will not be cancelled. When this is set, the timeout is enforced on all attempts, including the first. +For remote tables this is sent as a server execution timeout. It does not +extend HTTP read or proxy idle timeouts, or prove that a failed request did +not commit. diff --git a/docs/web/ingestion/merge-insert.mdx b/docs/web/ingestion/merge-insert.mdx index 8106e7706..2d6fd8470 100644 --- a/docs/web/ingestion/merge-insert.mdx +++ b/docs/web/ingestion/merge-insert.mdx @@ -221,6 +221,13 @@ and check inserted or deleted counts when the SDK returns them. Repeating a merge is only safe when its keys, conditions and update values produce the intended result on retry. +To override the transport retry budget for one merge, pass `retries=0` to +Python's synchronous or asynchronous `builder.execute(data, retries=0)`, or +use TypeScript's `builder.execute(data, { retries: 0 })`. The override applies +only to that execution. A larger budget does not make an ambiguous 5xx result +safe to replay. See [write retry boundaries](/ingestion/insert#performance) +for the distinction between retryable failures and an unknown commit outcome. + ### Embeddings and expression predicates {#advanced} When a table uses an [embedding definition](/embedding), behavior depends on diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 3e8d5a79b..215df992d 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -542,6 +542,163 @@ describe("remote connection", () => { expect(attempts).toBe(scenario.attempts); }); + it.each( + [409, 429, 502, 503, 504].flatMap((status) => + [undefined, 0, 2].flatMap((retries) => + [[], ["analytics", "features"]].map((namespacePath) => ({ + status, + retries, + namespacePath, + })), + ), + ), + )("applies per-call merge retries: %j", async (scenario) => { + let attempts = 0; + const identifier = [...scenario.namespacePath, "items"].join("$"); + const succeeds = + [409, 429].includes(scenario.status) && scenario.retries !== 0; + await withMockDatabase( + (req, res) => { + const url = new URL(req.url ?? "", "http://localhost"); + if (url.pathname === `/v1/table/${identifier}/describe/`) { + res.writeHead(200).end( + JSON.stringify({ + version: 1, + schema: { + fields: [ + { name: "id", type: { type: "int64" }, nullable: true }, + ], + }, + }), + ); + return; + } + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + try { + expect(url.pathname).toBe(`/v1/table/${identifier}/merge_insert/`); + expect(url.searchParams.get("use_index")).toBe("false"); + expect(req.headers["x-request-timeout-ms"]).toBe("10000"); + // Rust compresses the batches; Arrow JS can inspect the schema without a codec. + const reader = RecordBatchReader.from(Buffer.concat(chunks)).open(); + expect(reader.schema.fields.map((field) => field.name)).toEqual([ + "id", + ]); + } finally { + attempts++; + res + .writeHead(attempts === 1 ? scenario.status : 200) + .end('{"version":2}'); + } + }); + }, + async (db) => { + const table = await db.openTable("items", scenario.namespacePath); + const merge = table + .mergeInsert("id") + .whenNotMatchedInsertAll() + .useIndex(false); + const result = merge.execute([{ id: 1n }], { + timeoutMs: 10_000, + retries: scenario.retries, + }); + if (succeeds) { + await expect(result).resolves.toMatchObject({ version: 2 }); + } else { + await expect(result).rejects.toThrow(); + } + }, + { + clientConfig: { + retryConfig: { + retries: 3, + statuses: [409, 429, 502, 503, 504], + backoffFactor: 0, + backoffJitter: 0, + }, + }, + }, + ); + expect(attempts).toBe(succeeds ? 2 : 1); + }); + + it("isolates merge retry overrides across reused builders and reads", async () => { + let attempts = 0; + let readAttempts = 0; + await withMockDatabase( + (req, res) => { + if (req.url?.endsWith("/describe/")) { + res.writeHead(200).end( + JSON.stringify({ + version: 1, + schema: { + fields: [ + { name: "id", type: { type: "int64" }, nullable: true }, + ], + }, + }), + ); + return; + } + req.resume(); + req.on("end", () => { + if (req.url?.endsWith("/count_rows/")) { + readAttempts++; + res.writeHead(readAttempts % 2 ? 429 : 200).end("1"); + } else { + attempts++; + res + .writeHead([1, 2, 4, 6].includes(attempts) ? 429 : 200) + .end('{"version":2}'); + expect(req.url).toContain("/merge_insert/"); + } + }); + }, + async (db) => { + const table = await db.openTable("items"); + const merge = table.mergeInsert("id").whenNotMatchedInsertAll(); + for (const retries of [ + -1, + 256, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + ]) { + await expect( + merge.execute([{ id: 1n }], { retries }), + ).rejects.toThrow("retries must be an integer between 0 and 255"); + } + expect(attempts).toBe(0); + await expect( + merge.execute([{ id: 1n }], { retries: 0 }), + ).rejects.toThrow(); + expect(attempts).toBe(1); + await expect(merge.execute([{ id: 1n }])).resolves.toMatchObject({ + version: 2, + }); + await expect( + merge.execute([{ id: 1n }], { retries: 255 }), + ).resolves.toMatchObject({ version: 2 }); + expect(attempts).toBe(5); + await expect(table.countRows()).resolves.toBe(1); + expect(readAttempts).toBe(2); + await expect( + table + .mergeInsert("id") + .whenNotMatchedInsertAll() + .execute([{ id: 1n }]), + ).resolves.toMatchObject({ version: 2 }); + expect(attempts).toBe(7); + }, + { + clientConfig: { + retryConfig: { retries: 3, backoffFactor: 0, backoffJitter: 0 }, + }, + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index e97b32e78..99c58a8c9 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -938,7 +938,8 @@ describe("merge insert", () => { .mergeInsert("a") .whenMatchedUpdateAll() .whenNotMatchedInsertAll() - .execute(newData, { timeoutMs: 10_000 }); + .useIndex(false) + .execute(newData, { timeoutMs: 10_000, retries: 0 }); expect(mergeInsertRes).toHaveProperty("version"); expect(mergeInsertRes.version).toBe(2); expect(mergeInsertRes.numInsertedRows).toBe(1); diff --git a/nodejs/lancedb/merge.ts b/nodejs/lancedb/merge.ts index 7bb36e678..7d95d4a36 100644 --- a/nodejs/lancedb/merge.ts +++ b/nodejs/lancedb/merge.ts @@ -127,6 +127,14 @@ export class MergeInsertBuilder { data: Data, execOptions?: Partial, ): Promise { + if ( + execOptions?.retries !== undefined && + (!Number.isInteger(execOptions.retries) || + execOptions.retries < 0 || + execOptions.retries > 255) + ) { + throw new Error("retries must be an integer between 0 and 255"); + } let schema: Schema; if (this.#schema instanceof Promise) { schema = await this.#schema; @@ -140,7 +148,7 @@ export class MergeInsertBuilder { } const buffer = await fromDataToBuffer(data, undefined, schema); - return await this.#native.execute(buffer); + return await this.#native.execute(buffer, execOptions?.retries); } } @@ -156,6 +164,24 @@ export interface WriteExecutionOptions { * first attempt will not be cancelled. * * When this is set, the timeout is enforced on all attempts, including the first. + * For remote tables this is sent as a server execution timeout. It does not + * extend HTTP read or proxy idle timeouts, or prove that a failed request did + * not commit. */ timeoutMs?: number; + + /** + * Override the remote retry budget for this execution only (an integer from + * 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after + * precommit conflicts (409) or rate limits (429). This does not change + * connection-wide/read budgets or local transaction conflict retries. + * + * By default, remote calls use the connection's `clientConfig.retryConfig` + * ({@link RetryConfig}). Its `statuses` (also set through + * `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget; + * it never permits replay of ambiguous merge outcomes. In particular, 502, + * ordinary 503, and 504 are never replayed, even if configured. Reconcile table + * contents before manually repeating a merge whose commit outcome is unknown. + */ + retries?: number; } diff --git a/nodejs/src/merge.rs b/nodejs/src/merge.rs index ccff2a689..86901c04f 100644 --- a/nodejs/src/merge.rs +++ b/nodejs/src/merge.rs @@ -65,12 +65,17 @@ impl NativeMergeInsertBuilder { } #[napi(catch_unwind)] - pub async fn execute(&self, buf: Buffer) -> napi::Result { + pub async fn execute(&self, buf: Buffer, retries: Option) -> napi::Result { let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| { napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e))) })?; - let this = self.clone(); + let mut this = self.clone(); + if let Some(retries) = retries { + let retries = u8::try_from(retries) + .map_err(|_| napi::Error::from_reason("retries must be between 0 and 255"))?; + this.inner.retries(retries); + } let res = this.inner.execute(data).await.map_err(|e| { napi::Error::from_reason(format!( diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index a87b82a21..a86bae9d5 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -565,6 +565,12 @@ class Table: allow_external_blob_outside_bases: bool = False, retries: Optional[int] = None, ) -> AddResult: ... + async def execute_merge_insert( + self, + data: pa.RecordBatchReader, + parameters: Dict[str, Any], + retries: Optional[int] = None, + ) -> MergeResult: ... async def update( self, updates: Dict[str, str], where: Optional[str] ) -> UpdateResult: ... diff --git a/python/python/lancedb/merge.py b/python/python/lancedb/merge.py index 49f4793ed..c861414aa 100644 --- a/python/python/lancedb/merge.py +++ b/python/python/lancedb/merge.py @@ -161,6 +161,8 @@ class LanceMergeInsertBuilder(object): on_bad_vectors: str = "error", fill_value: float = 0.0, timeout: Optional[timedelta] = None, + *, + retries: Optional[int] = None, ) -> MergeInsertResult: """ Executes the merge insert operation @@ -189,7 +191,22 @@ class LanceMergeInsertBuilder(object): first attempt will not be cancelled. When this is set, the timeout is enforced on all attempts, including - the first. + the first. For remote tables it is sent as a server execution timeout; + it does not extend HTTP read or proxy idle timeouts, or prove that a + failed request did not commit. + retries: int, optional + Override the remote retry budget for this execution only (0 to 255). + Use ``execute(new_data, retries=0)`` to disable even retries after + precommit conflicts (409) or rate limits (429). This does not change + connection-wide/read budgets or local transaction conflict retries. + + By default, remote calls use the connection's + [RetryConfig][lancedb.remote.RetryConfig]. Its ``statuses`` (also set + through ``LANCE_CLIENT_RETRY_STATUSES``) selects candidates within + the retry budget; it never permits replay of ambiguous merge outcomes. + In particular, 502, ordinary 503, and 504 are never replayed, even if + configured. Reconcile table contents before manually repeating a merge + whose commit outcome is unknown. Returns ------- @@ -198,4 +215,6 @@ class LanceMergeInsertBuilder(object): """ if timeout is not None: self._timeout = timeout - return self._table._do_merge(self, new_data, on_bad_vectors, fill_value) + return self._table._do_merge( + self, new_data, on_bad_vectors, fill_value, retries=retries + ) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index b5d97eb62..818d2a257 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -945,9 +945,10 @@ class RemoteTable(Table): new_data: DATA, on_bad_vectors: str, fill_value: float, + retries: Optional[int] = None, ) -> MergeResult: return LOOP.run( - self._table._do_merge(merge, new_data, on_bad_vectors, fill_value) + self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries) ) @_deprecated_keyword("predicate", "where") diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 45b2eb4f6..4a713da06 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2810,6 +2810,7 @@ class Table(ABC): new_data: DATA, on_bad_vectors: OnBadVectorsType, fill_value: float, + retries: Optional[int] = None, ) -> MergeResult: ... @abstractmethod @@ -5443,9 +5444,10 @@ class LanceTable(Table): new_data: DATA, on_bad_vectors: OnBadVectorsType, fill_value: float, + retries: Optional[int] = None, ) -> MergeResult: return LOOP.run( - self._table._do_merge(merge, new_data, on_bad_vectors, fill_value) + self._table._do_merge(merge, new_data, on_bad_vectors, fill_value, retries) ) @property @@ -7777,6 +7779,7 @@ class AsyncTable: new_data: DATA, on_bad_vectors: OnBadVectorsType, fill_value: float, + retries: Optional[int] = None, ) -> MergeResult: schema = await self.schema() if on_bad_vectors is None: @@ -7808,6 +7811,7 @@ class AsyncTable: use_rte=merge._use_rte, validate_single_shard=merge._validate_single_shard, ), + retries=retries, ) async def delete(self, where: Union[str, Expr]) -> DeleteResult: diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index fc5b5df73..f404e36f7 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -310,6 +310,164 @@ async def test_add_retry_override(sync_api, retries, status): assert attempts == (2 if expected_success else 1) +@pytest.mark.parametrize("sync_api", [False, True]) +@pytest.mark.parametrize("namespace_path", [[], ["analytics", "features"]]) +@pytest.mark.parametrize("retries", [None, 0, 2]) +@pytest.mark.parametrize("status", [409, 429, 502, 503, 504]) +@pytest.mark.asyncio +async def test_merge_retry_override(sync_api, namespace_path, retries, status, capfd): + attempts = 0 + identifier = "$".join(namespace_path + ["test"]) + + def handler(request): + nonlocal attempts + if request.path == f"/v1/table/{identifier}/describe/": + send_json( + request, + { + "version": 1, + "schema": { + "fields": [ + {"name": "id", "type": {"type": "int64"}, "nullable": True} + ] + }, + }, + ) + return + assert urlparse(request.path).path == f"/v1/table/{identifier}/merge_insert/" + query = parse_qs(urlparse(request.path).query) + assert query["use_index"] == ["false"] + assert request.headers["x-request-timeout-ms"] == "10000" + body = bytearray() + if request.headers.get("Transfer-Encoding") == "chunked": + while True: + size = int(request.rfile.readline().strip(), 16) + if size == 0: + request.rfile.readline() + break + body.extend(request.rfile.read(size)) + request.rfile.read(2) + else: + body.extend( + request.rfile.read(int(request.headers.get("Content-Length", 0))) + ) + assert pa.ipc.open_stream(body).read_all().to_pylist() == [{"id": 1}] + attempts += 1 + send_json(request, {"version": 2}, status if attempts == 1 else 200) + + config = { + "retry_config": { + "retries": 3, + "statuses": [409, 429, 502, 503, 504], + "backoff_factor": 0, + "backoff_jitter": 0, + } + } + expected_success = status in (409, 429) and retries != 0 + + async def check(table): + merge = table.merge_insert("id").when_not_matched_insert_all().use_index(False) + options = {"timeout": timedelta(seconds=10)} + if retries is not None: + options["retries"] = retries + if expected_success: + result = merge.execute([{"id": 1}], **options) + if not sync_api: + result = await result + assert result.version == 2 + else: + with pytest.raises((HttpError, RetryError)): + result = merge.execute([{"id": 1}], **options) + if not sync_api: + await result + + if sync_api: + with mock_lancedb_connection(handler, **config) as db: + await check(db.open_table("test", namespace_path=namespace_path)) + else: + async with mock_lancedb_connection_async(handler, **config) as db: + await check(await db.open_table("test", namespace_path=namespace_path)) + assert attempts == (2 if expected_success else 1) + warnings = [ + line + for line in capfd.readouterr().err.splitlines() + if "Retrying merge insert" in line + ] + assert len(warnings) == (1 if expected_success else 0) + if expected_success: + assert "WARN" in warnings[0] + assert f"status Some({status}); attempt 2" in warnings[0] + + +@pytest.mark.parametrize("sync_api", [False, True]) +@pytest.mark.asyncio +async def test_merge_retry_override_isolation(sync_api): + attempts = 0 + read_attempts = 0 + + def handler(request): + nonlocal attempts, read_attempts + if request.path.endswith("/describe/"): + send_json( + request, + { + "version": 1, + "schema": { + "fields": [ + {"name": "id", "type": {"type": "int64"}, "nullable": True} + ] + }, + }, + ) + elif request.path.endswith("/count_rows/"): + read_attempts += 1 + send_json(request, 1, 429 if read_attempts % 2 else 200) + else: + assert urlparse(request.path).path.endswith("/merge_insert/") + attempts += 1 + send_json(request, {"version": 2}, 429 if attempts in (1, 2, 4, 6) else 200) + + async def check(table): + merge = table.merge_insert("id").when_not_matched_insert_all() + for retries in [-1, 256, 1.5, "1"]: + with pytest.raises((OverflowError, TypeError, ValueError)): + result = merge.execute([{"id": 1}], retries=retries) + if not sync_api: + await result + assert attempts == 0 + with pytest.raises((HttpError, RetryError)): + result = merge.execute([{"id": 1}], retries=0) + if not sync_api: + await result + assert attempts == 1 + for options in ({}, {"retries": 255}): + result = merge.execute([{"id": 1}], **options) + if not sync_api: + result = await result + assert result.version == 2 + assert attempts == 5 + result = table.count_rows() + if not sync_api: + result = await result + assert result == 1 + assert read_attempts == 2 + result = ( + table.merge_insert("id").when_not_matched_insert_all().execute([{"id": 1}]) + ) + if not sync_api: + result = await result + assert result.version == 2 + assert attempts == 7 + + config = {"retry_config": {"retries": 3, "backoff_factor": 0, "backoff_jitter": 0}} + if sync_api: + with mock_lancedb_connection(handler, **config) as db: + await check(db.open_table("test")) + else: + async with mock_lancedb_connection_async(handler, **config) as db: + await check(await db.open_table("test")) + + @pytest.fixture def missing_table_handler(cached, namespace_path): request_ids = [] diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index ce4f57c48..4b26a9003 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3448,7 +3448,8 @@ def test_merge_insert(mem_db: DBConnection): table.merge_insert("a") .when_matched_update_all() .when_not_matched_insert_all() - .execute(new_data, timeout=timedelta(seconds=10)) + .use_index(False) + .execute(new_data, timeout=timedelta(seconds=10), retries=0) ) assert merge_insert_res.version == 2 assert merge_insert_res.num_inserted_rows == 1 @@ -3527,7 +3528,7 @@ def test_merge_insert(mem_db: DBConnection): # timeout with pytest.raises(Exception, match="merge insert timed out"): table.merge_insert("a").when_matched_update_all().execute( - new_data, timeout=timedelta(0) + new_data, timeout=timedelta(0), retries=255 ) diff --git a/python/src/table.rs b/python/src/table.rs index f5270a594..25a99e5db 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -2269,10 +2269,12 @@ impl Table { }) } + #[pyo3(signature = (data, parameters, retries=None))] pub fn execute_merge_insert<'a>( self_: PyRef<'a, Self>, data: Bound<'a, PyAny>, parameters: MergeInsertParams, + retries: Option, ) -> PyResult> { let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?; let on = parameters.on.iter().map(|s| s.as_str()).collect::>(); @@ -2295,6 +2297,9 @@ impl Table { if let Some(timeout) = parameters.timeout { builder.timeout(timeout); } + if let Some(retries) = retries { + builder.retries(retries); + } if let Some(use_index) = parameters.use_index { builder.use_index(use_index); } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 29f3ab50e..2224203cf 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -3707,7 +3707,17 @@ impl BaseTable for RemoteTable { return Ok(merge_result); } Err(err) if rescannable && self.is_retryable_write_error(&err) => { + let status = err.status_code(); retry_counter.increment_from_error(err)?; + log::warn!( + "Retrying merge insert {} after status {:?}; attempt {}", + retry_counter.request_id, + status.map(|s| s.as_u16()), + u16::from(retry_counter.request_failures) + + u16::from(retry_counter.connect_failures) + + u16::from(retry_counter.read_failures) + + 1, + ); tokio::time::sleep(retry_counter.next_sleep_time()).await; merge = merge.reset_state()?; continue;