diff --git a/docs/web/ingestion/insert.mdx b/docs/web/ingestion/insert.mdx index 923a71b0e..7633584c6 100644 --- a/docs/web/ingestion/insert.mdx +++ b/docs/web/ingestion/insert.mdx @@ -150,9 +150,10 @@ Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`. The expected row count may be unknown for streaming input. On the final update, it falls back to the number of rows written. -For remote multipart writes, rows count input scanning, while bytes count -acknowledged IPC parts. Bytes start at zero and a retried part is counted once, -after acknowledgement. Scanned rows and staged bytes do not imply a commit. +For remote multipart writes, both rows and IPC bytes count acknowledged parts. +Both start at zero; a retried part is counted once, after acknowledgement. +Scanning input or sending a rejected part does not advance these counts. +Acknowledged parts are staged and do not imply a successful commit. Keep callbacks short and use the write operation's result to determine success. The final `done` update means progress reporting has ended; it can also occur diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index a7cf3f80d..5c00232f7 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2262,7 +2262,7 @@ impl RemoteTable { num_partitions: usize, ) -> Result { if let Some(tracker) = &output.tracker { - tracker.use_wire_bytes(); + tracker.use_acknowledged_parts(); } let freshness_request = self.snapshot_freshness_headers(); let upload_id = self.create_multipart_write(output.retries).await?; diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index 0de1c21cc..0801880f2 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -480,6 +480,7 @@ impl PartRequestCtx<'_, S> { let started = Instant::now(); let mut pending = Some(first); let mut has_batch = false; + let mut part_rows = 0; let mut input_ended = false; loop { let batch = match pending.take().or_else(|| queued.pop_front()) { @@ -520,6 +521,7 @@ impl PartRequestCtx<'_, S> { // A single row and its schema cannot be split further. } has_batch = true; + part_rows += batch.num_rows(); if writer.get_ref().len() as u64 >= max_bytes || max_duration.is_some_and(|limit| started.elapsed() >= limit) { @@ -536,7 +538,7 @@ impl PartRequestCtx<'_, S> { match self.send_part_request(request, freshness).await { Ok(()) => { if let Some(tracker) = tracker { - tracker.record_bytes(body.len()); + tracker.record_acknowledged_part(part_rows, body.len()); } return Ok(input_ended); } @@ -1648,7 +1650,7 @@ mod tests { } #[tokio::test] - async fn test_multipart_records_acknowledged_bytes() { + async fn test_multipart_records_acknowledged_rows_and_bytes() { use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker}; use futures::StreamExt; @@ -1667,14 +1669,18 @@ mod tests { ]; let input = input_plan_from_batches(schema, batches).await; - let observed = Arc::new(Mutex::new(Vec::::new())); + let observed = Arc::new(Mutex::new(Vec::<(usize, usize)>::new())); let observed_cb = observed.clone(); let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| { - observed_cb.lock().unwrap().push(p.output_bytes()); + observed_cb + .lock() + .unwrap() + .push((p.output_rows(), p.output_bytes())); })); let tracker = Arc::new(WriteProgressTracker::new(callback, None)); - // Count encoded bytes once when the part is acknowledged. + tracker.use_acknowledged_parts(); + // Count rows from all encoded batches once when the part is acknowledged. let exec = RemoteWriteExec::new_multipart( "my_table".to_string(), "my_table".to_string(), @@ -1697,6 +1703,7 @@ mod tests { "batches should all land in a single part" ); let observed = observed.lock().unwrap(); + assert_eq!(observed.last().unwrap().0, 6); assert!( observed.len() == 1, "expected one progress update per acknowledged part: {observed:?}" @@ -1706,7 +1713,7 @@ mod tests { "progress bytes should be monotonic: {observed:?}" ); assert!( - *observed.last().unwrap() > 0, + observed.last().unwrap().1 > 0, "final progress should report bytes: {observed:?}" ); } diff --git a/rust/lancedb/src/remote/table/write_tests.rs b/rust/lancedb/src/remote/table/write_tests.rs index c97619af1..2e0ca1d45 100644 --- a/rust/lancedb/src/remote/table/write_tests.rs +++ b/rust/lancedb/src/remote/table/write_tests.rs @@ -267,17 +267,44 @@ async fn multipart_http_faults_preserve_committed_blob_payloads() { #[tokio::test] async fn multipart_retries_sixth_part_after_reused_connection_closes() { + use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker}; use datafusion_execution::TaskContext; use datafusion_physical_plan::ExecutionPlan; use futures::TryStreamExt; + let progress = Arc::new(Mutex::new(Vec::new())); + let observed = progress.clone(); + let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| { + observed + .lock() + .unwrap() + .push((p.output_rows(), p.output_bytes())); + })); + let tracker = Arc::new(WriteProgressTracker::new(callback, Some(6))); + tracker.use_acknowledged_parts(); + let during_request = progress.clone(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { let mut attempts = Vec::new(); + let mut acknowledged_bytes = 0; let (mut socket, _) = listener.accept().await.unwrap(); for n in 0..6 { let (path, body) = read_request(&mut socket).await; + let rows = StreamReader::try_new(body.as_slice(), None) + .unwrap() + .map(|batch| batch.unwrap().num_rows()) + .sum::(); + assert_eq!(rows, 1, "one scanned batch must split into six parts"); + let reported = *during_request.lock().unwrap().last().unwrap(); + assert_eq!( + reported, + (n, acknowledged_bytes), + "progress must exclude the unacknowledged part" + ); + if n < 5 { + acknowledged_bytes += body.len(); + } attempts.push((path, body)); if n == 5 { break; @@ -291,9 +318,17 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { // The sixth part was staged, but the peer closes before response headers. drop(socket); let (mut socket, _) = listener.accept().await.unwrap(); - attempts.push(read_request(&mut socket).await); + let (path, body) = read_request(&mut socket).await; + let reported = *during_request.lock().unwrap().last().unwrap(); + assert_eq!( + reported, + (5, acknowledged_bytes), + "retry must not report the lost acknowledgement" + ); + acknowledged_bytes += body.len(); + attempts.push((path, body)); respond(&mut socket, 200, "{}").await; - attempts + (attempts, acknowledged_bytes) }); let client = RestfulLanceDbClient::try_new( &crate::remote::client::parse_db_url("db://test").unwrap(), @@ -316,7 +351,10 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { .unwrap(); let input = arrow_array::record_batch!(("id", Int32, [1, 2, 3, 4, 5, 6])).unwrap(); let plan = Arc::new( - crate::table::datafusion::scannable_exec::ScannableExec::new(Box::new(input), None), + crate::table::datafusion::scannable_exec::ScannableExec::new( + Box::new(input), + Some(tracker.clone()), + ), ); let exec = super::insert::RemoteWriteExec::new_multipart( "test".into(), @@ -325,7 +363,7 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { plan, false, "upload-1".into(), - None, + Some(tracker), None, Some(1), None, @@ -335,7 +373,15 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() { .try_collect::>() .await .unwrap(); - let attempts = server.await.unwrap(); + let (attempts, acknowledged_bytes) = server.await.unwrap(); + let progress = progress.lock().unwrap(); + assert_eq!(*progress.first().unwrap(), (0, 0)); + assert_eq!(*progress.last().unwrap(), (6, acknowledged_bytes)); + assert!( + progress + .windows(2) + .all(|pair| pair[0].0 <= pair[1].0 && pair[0].1 <= pair[1].1) + ); assert_eq!(attempts.len(), 7); assert_eq!( attempts[5], attempts[6], @@ -424,7 +470,12 @@ async fn multipart_rejections_preserve_progress_and_cleanup_boundaries() { let progress = progress.lock().unwrap(); let last = progress.last().unwrap(); assert!(last.2); - assert_eq!(last.0, 2); + assert_eq!( + progress.first().unwrap().0, + 0, + "scan is not an acknowledgement" + ); + assert_eq!(last.0, if stage_status == 200 { 2 } else { 0 }); if stage_status != 200 { assert!( progress.iter().all(|p| p.1 == 0), diff --git a/rust/lancedb/src/table/write_progress.rs b/rust/lancedb/src/table/write_progress.rs index 12e4dd62c..efea4586c 100644 --- a/rust/lancedb/src/table/write_progress.rs +++ b/rust/lancedb/src/table/write_progress.rs @@ -30,9 +30,11 @@ impl WriteProgress { self.elapsed } - /// Number of rows processed by the input scan. + /// Number of rows processed by the writer. /// - /// This does not indicate that the rows have been committed. + /// Remote multipart writes count rows in acknowledged IPC parts, starting + /// at zero and counting retransmissions once. Other writers report rows + /// processed by the input scan. Neither indicates a successful commit. pub fn output_rows(&self) -> usize { self.output_rows } @@ -101,8 +103,8 @@ pub(crate) struct WriteProgressTracker { /// Wire bytes tracked separately by the insert layer. When set (> 0), /// this takes precedence over the in-memory bytes from `rows_and_bytes`. wire_bytes: AtomicUsize, - /// Multipart writes select this before scanning, including zero acknowledgements. - wire_bytes_enabled: AtomicBool, + /// Multipart progress excludes input scanning and counts only acknowledged parts. + acknowledged_parts: AtomicBool, active_tasks: Arc, total_tasks: AtomicUsize, start: Instant, @@ -116,7 +118,7 @@ impl WriteProgressTracker { Self { rows_and_bytes: std::sync::Mutex::new((0, 0)), wire_bytes: AtomicUsize::new(0), - wire_bytes_enabled: AtomicBool::new(false), + acknowledged_parts: AtomicBool::new(false), active_tasks: Arc::new(AtomicUsize::new(0)), total_tasks: AtomicUsize::new(1), start: Instant::now(), @@ -125,9 +127,9 @@ impl WriteProgressTracker { } } - /// Select wire-byte progress, including zero, before polling the input plan. - pub fn use_wire_bytes(&self) { - self.wire_bytes_enabled.store(true, Ordering::Relaxed); + /// Select acknowledged rows and wire bytes before polling the input plan. + pub fn use_acknowledged_parts(&self) { + self.acknowledged_parts.store(true, Ordering::Relaxed); } /// Set the total number of parallel write tasks (the write parallelism). @@ -145,13 +147,29 @@ impl WriteProgressTracker { pub fn record_batch(&self, rows: usize, bytes: usize) { // Lock order: callback first, then rows_and_bytes. This is the only // order used anywhere, so deadlocks cannot occur. + let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner()); + let mut guard = self + .rows_and_bytes + .lock() + .unwrap_or_else(|e| e.into_inner()); + if !self.acknowledged_parts.load(Ordering::Relaxed) { + guard.0 += rows; + guard.1 += bytes; + } + let progress = self.snapshot(guard.0, guard.1, false); + drop(guard); + cb(&progress); + } + + /// Record one successfully staged part, after any retries have finished. + pub fn record_acknowledged_part(&self, rows: usize, bytes: usize) { let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner()); let mut guard = self .rows_and_bytes .lock() .unwrap_or_else(|e| e.into_inner()); guard.0 += rows; - guard.1 += bytes; + self.wire_bytes.fetch_add(bytes, Ordering::Relaxed); let progress = self.snapshot(guard.0, guard.1, false); drop(guard); cb(&progress); @@ -194,7 +212,7 @@ impl WriteProgressTracker { let wire = self.wire_bytes.load(Ordering::Relaxed); // Prefer wire bytes (actual I/O size) when the insert layer is // tracking them; fall back to in-memory Arrow size otherwise. - let output_bytes = if self.wire_bytes_enabled.load(Ordering::Relaxed) || wire > 0 { + let output_bytes = if self.acknowledged_parts.load(Ordering::Relaxed) || wire > 0 { wire } else { in_memory_bytes