From 2f496322ef23bdb27d57e7aea9d151336ab570a4 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 20:51:25 +0800 Subject: [PATCH] fix: report multipart rows only after part acknowledgement (#5020) Remote multipart progress currently counts every scanned row before the corresponding IPC part is acknowledged. A rejected first part can therefore report uploaded rows, even though its acknowledged byte count remains zero. Count both rows and IPC bytes only after each part succeeds, including parts assembled from multiple batches or split from one batch. Retried parts count once. Native and single-request writes retain their existing accounting; acknowledged multipart progress describes staged data, not a successful commit. Validation follows the actual TCP upload path: one scanned batch becomes six decoded IPC parts, the sixth response is lost, and each request/retry observes only previously acknowledged rows and bytes. Rejection and complete-outcome cases preserve the same accounting boundary. This validation covers local HTTP behavior; deployed service throughput and the original disconnect investigation remain outside its scope. Frozen candidate validation: 839 remote tests and 7 progress tests passed. Scoped Clippy (`lancedb`, remote feature, lib/tests/examples, warnings denied) passed with unchanged source. Python/Node bindings were unchanged and were not rebuilt for this patch. --- docs/web/ingestion/insert.mdx | 7 ++- rust/lancedb/src/remote/table.rs | 2 +- rust/lancedb/src/remote/table/insert.rs | 19 ++++-- rust/lancedb/src/remote/table/write_tests.rs | 63 ++++++++++++++++++-- rust/lancedb/src/table/write_progress.rs | 38 ++++++++---- 5 files changed, 103 insertions(+), 26 deletions(-) diff --git a/docs/web/ingestion/insert.mdx b/docs/web/ingestion/insert.mdx index 923a71b0e..7633584c6 100644 --- a/docs/web/ingestion/insert.mdx +++ b/docs/web/ingestion/insert.mdx @@ -150,9 +150,10 @@ Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`. The expected row count may be unknown for streaming input. On the final update, it falls back to the number of rows written. -For remote multipart writes, rows count input scanning, while bytes count -acknowledged IPC parts. Bytes start at zero and a retried part is counted once, -after acknowledgement. Scanned rows and staged bytes do not imply a commit. +For remote multipart writes, both rows and IPC bytes count acknowledged parts. +Both start at zero; a retried part is counted once, after acknowledgement. +Scanning input or sending a rejected part does not advance these counts. +Acknowledged parts are staged and do not imply a successful commit. Keep callbacks short and use the write operation's result to determine success. The final `done` update means progress reporting has ended; it can also occur diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index a7cf3f80d..5c00232f7 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2262,7 +2262,7 @@ impl RemoteTable { num_partitions: usize, ) -> Result { if let Some(tracker) = &output.tracker { - tracker.use_wire_bytes(); + tracker.use_acknowledged_parts(); } let freshness_request = self.snapshot_freshness_headers(); let upload_id = self.create_multipart_write(output.retries).await?; diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index 0de1c21cc..0801880f2 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -480,6 +480,7 @@ impl PartRequestCtx<'_, S> { let started = Instant::now(); let mut pending = Some(first); let mut has_batch = false; + let mut part_rows = 0; let mut input_ended = false; loop { let batch = match pending.take().or_else(|| queued.pop_front()) { @@ -520,6 +521,7 @@ impl PartRequestCtx<'_, S> { // A single row and its schema cannot be split further. } has_batch = true; + part_rows += batch.num_rows(); if writer.get_ref().len() as u64 >= max_bytes || max_duration.is_some_and(|limit| started.elapsed() >= limit) { @@ -536,7 +538,7 @@ impl PartRequestCtx<'_, S> { match self.send_part_request(request, freshness).await { Ok(()) => { if let Some(tracker) = tracker { - tracker.record_bytes(body.len()); + tracker.record_acknowledged_part(part_rows, body.len()); } return Ok(input_ended); } @@ -1648,7 +1650,7 @@ mod tests { } #[tokio::test] - async fn test_multipart_records_acknowledged_bytes() { + async fn test_multipart_records_acknowledged_rows_and_bytes() { use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker}; use futures::StreamExt; @@ -1667,14 +1669,18 @@ mod tests { ]; let input = input_plan_from_batches(schema, batches).await; - let observed = Arc::new(Mutex::new(Vec::::new())); + let observed = Arc::new(Mutex::new(Vec::<(usize, usize)>::new())); let observed_cb = observed.clone(); let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| { - observed_cb.lock().unwrap().push(p.output_bytes()); + observed_cb + .lock() + .unwrap() + .push((p.output_rows(), p.output_bytes())); })); let tracker = Arc::new(WriteProgressTracker::new(callback, None)); - // Count encoded bytes once when the part is acknowledged. + tracker.use_acknowledged_parts(); + // Count rows from all encoded batches once when the part is acknowledged. let exec = RemoteWriteExec::new_multipart( "my_table".to_string(), "my_table".to_string(), @@ -1697,6 +1703,7 @@ mod tests { "batches should all land in a single part" ); let observed = observed.lock().unwrap(); + assert_eq!(observed.last().unwrap().0, 6); assert!( observed.len() == 1, "expected one progress update per acknowledged part: {observed:?}" @@ -1706,7 +1713,7 @@ mod tests { "progress bytes should be monotonic: {observed:?}" ); assert!( - *observed.last().unwrap() > 0, + observed.last().unwrap().1 > 0, "final progress should report bytes: {observed:?}" ); } diff --git a/rust/lancedb/src/remote/table/write_tests.rs b/rust/lancedb/src/remote/table/write_tests.rs index c97619af1..2e0ca1d45 100644 --- a/rust/lancedb/src/remote/table/write_tests.rs +++ b/rust/lancedb/src/remote/table/write_tests.rs @@ -267,17 +267,44 @@ async fn multipart_http_faults_preserve_committed_blob_payloads() { #[tokio::test] async fn multipart_retries_sixth_part_after_reused_connection_closes() { + use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker}; use datafusion_execution::TaskContext; use datafusion_physical_plan::ExecutionPlan; use futures::TryStreamExt; + let progress = Arc::new(Mutex::new(Vec::new())); + let observed = progress.clone(); + let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| { + observed + .lock() + .unwrap() + .push((p.output_rows(), p.output_bytes())); + })); + let tracker = Arc::new(WriteProgressTracker::new(callback, Some(6))); + tracker.use_acknowledged_parts(); + let during_request = progress.clone(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { let mut attempts = Vec::new(); + let mut acknowledged_bytes = 0; let (mut socket, _) = listener.accept().await.unwrap(); for n in 0..6 { let (path, body) = read_request(&mut socket).await; + let rows = StreamReader::try_new(body.as_slice(), None) + .unwrap() + .map(|batch| batch.unwrap().num_rows()) + .sum::(); + assert_eq!(rows, 1, "one scanned batch must split into six parts"); + let reported = *during_request.lock().unwrap().last().unwrap(); + assert_eq!( + reported, + (n, acknowledged_bytes), + "progress must exclude the unacknowledged part" + ); + if n < 5 { + acknowledged_bytes += body.len(); + } attempts.push((path, body)); if n == 5 { break; @@ -291,9 +318,17 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { // The sixth part was staged, but the peer closes before response headers. drop(socket); let (mut socket, _) = listener.accept().await.unwrap(); - attempts.push(read_request(&mut socket).await); + let (path, body) = read_request(&mut socket).await; + let reported = *during_request.lock().unwrap().last().unwrap(); + assert_eq!( + reported, + (5, acknowledged_bytes), + "retry must not report the lost acknowledgement" + ); + acknowledged_bytes += body.len(); + attempts.push((path, body)); respond(&mut socket, 200, "{}").await; - attempts + (attempts, acknowledged_bytes) }); let client = RestfulLanceDbClient::try_new( &crate::remote::client::parse_db_url("db://test").unwrap(), @@ -316,7 +351,10 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { .unwrap(); let input = arrow_array::record_batch!(("id", Int32, [1, 2, 3, 4, 5, 6])).unwrap(); let plan = Arc::new( - crate::table::datafusion::scannable_exec::ScannableExec::new(Box::new(input), None), + crate::table::datafusion::scannable_exec::ScannableExec::new( + Box::new(input), + Some(tracker.clone()), + ), ); let exec = super::insert::RemoteWriteExec::new_multipart( "test".into(), @@ -325,7 +363,7 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { plan, false, "upload-1".into(), - None, + Some(tracker), None, Some(1), None, @@ -335,7 +373,15 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { .try_collect::>() .await .unwrap(); - let attempts = server.await.unwrap(); + let (attempts, acknowledged_bytes) = server.await.unwrap(); + let progress = progress.lock().unwrap(); + assert_eq!(*progress.first().unwrap(), (0, 0)); + assert_eq!(*progress.last().unwrap(), (6, acknowledged_bytes)); + assert!( + progress + .windows(2) + .all(|pair| pair[0].0 <= pair[1].0 && pair[0].1 <= pair[1].1) + ); assert_eq!(attempts.len(), 7); assert_eq!( attempts[5], attempts[6], @@ -424,7 +470,12 @@ async fn multipart_rejections_preserve_progress_and_cleanup_boundaries() { let progress = progress.lock().unwrap(); let last = progress.last().unwrap(); assert!(last.2); - assert_eq!(last.0, 2); + assert_eq!( + progress.first().unwrap().0, + 0, + "scan is not an acknowledgement" + ); + assert_eq!(last.0, if stage_status == 200 { 2 } else { 0 }); if stage_status != 200 { assert!( progress.iter().all(|p| p.1 == 0), diff --git a/rust/lancedb/src/table/write_progress.rs b/rust/lancedb/src/table/write_progress.rs index 12e4dd62c..efea4586c 100644 --- a/rust/lancedb/src/table/write_progress.rs +++ b/rust/lancedb/src/table/write_progress.rs @@ -30,9 +30,11 @@ impl WriteProgress { self.elapsed } - /// Number of rows processed by the input scan. + /// Number of rows processed by the writer. /// - /// This does not indicate that the rows have been committed. + /// Remote multipart writes count rows in acknowledged IPC parts, starting + /// at zero and counting retransmissions once. Other writers report rows + /// processed by the input scan. Neither indicates a successful commit. pub fn output_rows(&self) -> usize { self.output_rows } @@ -101,8 +103,8 @@ pub(crate) struct WriteProgressTracker { /// Wire bytes tracked separately by the insert layer. When set (> 0), /// this takes precedence over the in-memory bytes from `rows_and_bytes`. wire_bytes: AtomicUsize, - /// Multipart writes select this before scanning, including zero acknowledgements. - wire_bytes_enabled: AtomicBool, + /// Multipart progress excludes input scanning and counts only acknowledged parts. + acknowledged_parts: AtomicBool, active_tasks: Arc, total_tasks: AtomicUsize, start: Instant, @@ -116,7 +118,7 @@ impl WriteProgressTracker { Self { rows_and_bytes: std::sync::Mutex::new((0, 0)), wire_bytes: AtomicUsize::new(0), - wire_bytes_enabled: AtomicBool::new(false), + acknowledged_parts: AtomicBool::new(false), active_tasks: Arc::new(AtomicUsize::new(0)), total_tasks: AtomicUsize::new(1), start: Instant::now(), @@ -125,9 +127,9 @@ impl WriteProgressTracker { } } - /// Select wire-byte progress, including zero, before polling the input plan. - pub fn use_wire_bytes(&self) { - self.wire_bytes_enabled.store(true, Ordering::Relaxed); + /// Select acknowledged rows and wire bytes before polling the input plan. + pub fn use_acknowledged_parts(&self) { + self.acknowledged_parts.store(true, Ordering::Relaxed); } /// Set the total number of parallel write tasks (the write parallelism). @@ -145,13 +147,29 @@ impl WriteProgressTracker { pub fn record_batch(&self, rows: usize, bytes: usize) { // Lock order: callback first, then rows_and_bytes. This is the only // order used anywhere, so deadlocks cannot occur. + let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner()); + let mut guard = self + .rows_and_bytes + .lock() + .unwrap_or_else(|e| e.into_inner()); + if !self.acknowledged_parts.load(Ordering::Relaxed) { + guard.0 += rows; + guard.1 += bytes; + } + let progress = self.snapshot(guard.0, guard.1, false); + drop(guard); + cb(&progress); + } + + /// Record one successfully staged part, after any retries have finished. + pub fn record_acknowledged_part(&self, rows: usize, bytes: usize) { let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner()); let mut guard = self .rows_and_bytes .lock() .unwrap_or_else(|e| e.into_inner()); guard.0 += rows; - guard.1 += bytes; + self.wire_bytes.fetch_add(bytes, Ordering::Relaxed); let progress = self.snapshot(guard.0, guard.1, false); drop(guard); cb(&progress); @@ -194,7 +212,7 @@ impl WriteProgressTracker { let wire = self.wire_bytes.load(Ordering::Relaxed); // Prefer wire bytes (actual I/O size) when the insert layer is // tracking them; fall back to in-memory Arrow size otherwise. - let output_bytes = if self.wire_bytes_enabled.load(Ordering::Relaxed) || wire > 0 { + let output_bytes = if self.acknowledged_parts.load(Ordering::Relaxed) || wire > 0 { wire } else { in_memory_bytes