fix: report multipart rows only after part acknowledgement (#5020)

Remote multipart progress currently counts every scanned row before the
corresponding IPC part is acknowledged. A rejected first part can
therefore report uploaded rows, even though its acknowledged byte count
remains zero.

Count both rows and IPC bytes only after each part succeeds, including
parts assembled from multiple batches or split from one batch. Retried
parts count once. Native and single-request writes retain their existing
accounting; acknowledged multipart progress describes staged data, not a
successful commit.

Validation follows the actual TCP upload path: one scanned batch becomes
six decoded IPC parts, the sixth response is lost, and each
request/retry observes only previously acknowledged rows and bytes.
Rejection and complete-outcome cases preserve the same accounting
boundary. This validation covers local HTTP behavior; deployed service
throughput and the original disconnect investigation remain outside its
scope.


Frozen candidate validation: 839 remote tests and 7 progress tests
passed. Scoped Clippy (`lancedb`, remote feature, lib/tests/examples,
warnings denied) passed with unchanged source. Python/Node bindings were
unchanged and were not rebuilt for this patch.
This commit is contained in:
Xuanwo
2026-11-14 20:51:25 +08:00
committed by GitHub
parent 5de76bd8ed
commit 2f496322ef
5 changed files with 103 additions and 26 deletions
+4 -3
View File
@@ -150,9 +150,10 @@ Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`.
The expected row count may be unknown for streaming input. On the final update,
it falls back to the number of rows written.
For remote multipart writes, rows count input scanning, while bytes count
acknowledged IPC parts. Bytes start at zero and a retried part is counted once,
after acknowledgement. Scanned rows and staged bytes do not imply a commit.
For remote multipart writes, both rows and IPC bytes count acknowledged parts.
Both start at zero; a retried part is counted once, after acknowledgement.
Scanning input or sending a rejected part does not advance these counts.
Acknowledged parts are staged and do not imply a successful commit.
Keep callbacks short and use the write operation's result to determine success.
The final `done` update means progress reporting has ended; it can also occur
+1 -1
View File
@@ -2262,7 +2262,7 @@ impl<S: HttpSend + 'static> RemoteTable<S> {
num_partitions: usize,
) -> Result<AddResult> {
if let Some(tracker) = &output.tracker {
tracker.use_wire_bytes();
tracker.use_acknowledged_parts();
}
let freshness_request = self.snapshot_freshness_headers();
let upload_id = self.create_multipart_write(output.retries).await?;
+13 -6
View File
@@ -480,6 +480,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
let started = Instant::now();
let mut pending = Some(first);
let mut has_batch = false;
let mut part_rows = 0;
let mut input_ended = false;
loop {
let batch = match pending.take().or_else(|| queued.pop_front()) {
@@ -520,6 +521,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
// A single row and its schema cannot be split further.
}
has_batch = true;
part_rows += batch.num_rows();
if writer.get_ref().len() as u64 >= max_bytes
|| max_duration.is_some_and(|limit| started.elapsed() >= limit)
{
@@ -536,7 +538,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
match self.send_part_request(request, freshness).await {
Ok(()) => {
if let Some(tracker) = tracker {
tracker.record_bytes(body.len());
tracker.record_acknowledged_part(part_rows, body.len());
}
return Ok(input_ended);
}
@@ -1648,7 +1650,7 @@ mod tests {
}
#[tokio::test]
async fn test_multipart_records_acknowledged_bytes() {
async fn test_multipart_records_acknowledged_rows_and_bytes() {
use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker};
use futures::StreamExt;
@@ -1667,14 +1669,18 @@ mod tests {
];
let input = input_plan_from_batches(schema, batches).await;
let observed = Arc::new(Mutex::new(Vec::<usize>::new()));
let observed = Arc::new(Mutex::new(Vec::<(usize, usize)>::new()));
let observed_cb = observed.clone();
let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| {
observed_cb.lock().unwrap().push(p.output_bytes());
observed_cb
.lock()
.unwrap()
.push((p.output_rows(), p.output_bytes()));
}));
let tracker = Arc::new(WriteProgressTracker::new(callback, None));
// Count encoded bytes once when the part is acknowledged.
tracker.use_acknowledged_parts();
// Count rows from all encoded batches once when the part is acknowledged.
let exec = RemoteWriteExec::new_multipart(
"my_table".to_string(),
"my_table".to_string(),
@@ -1697,6 +1703,7 @@ mod tests {
"batches should all land in a single part"
);
let observed = observed.lock().unwrap();
assert_eq!(observed.last().unwrap().0, 6);
assert!(
observed.len() == 1,
"expected one progress update per acknowledged part: {observed:?}"
@@ -1706,7 +1713,7 @@ mod tests {
"progress bytes should be monotonic: {observed:?}"
);
assert!(
*observed.last().unwrap() > 0,
observed.last().unwrap().1 > 0,
"final progress should report bytes: {observed:?}"
);
}
+57 -6
View File
@@ -267,17 +267,44 @@ async fn multipart_http_faults_preserve_committed_blob_payloads() {
#[tokio::test]
async fn multipart_retries_sixth_part_after_reused_connection_closes() {
use crate::table::write_progress::{ProgressCallback, WriteProgress, WriteProgressTracker};
use datafusion_execution::TaskContext;
use datafusion_physical_plan::ExecutionPlan;
use futures::TryStreamExt;
let progress = Arc::new(Mutex::new(Vec::new()));
let observed = progress.clone();
let callback: ProgressCallback = Arc::new(Mutex::new(move |p: &WriteProgress| {
observed
.lock()
.unwrap()
.push((p.output_rows(), p.output_bytes()));
}));
let tracker = Arc::new(WriteProgressTracker::new(callback, Some(6)));
tracker.use_acknowledged_parts();
let during_request = progress.clone();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let mut attempts = Vec::new();
let mut acknowledged_bytes = 0;
let (mut socket, _) = listener.accept().await.unwrap();
for n in 0..6 {
let (path, body) = read_request(&mut socket).await;
let rows = StreamReader::try_new(body.as_slice(), None)
.unwrap()
.map(|batch| batch.unwrap().num_rows())
.sum::<usize>();
assert_eq!(rows, 1, "one scanned batch must split into six parts");
let reported = *during_request.lock().unwrap().last().unwrap();
assert_eq!(
reported,
(n, acknowledged_bytes),
"progress must exclude the unacknowledged part"
);
if n < 5 {
acknowledged_bytes += body.len();
}
attempts.push((path, body));
if n == 5 {
break;
@@ -291,9 +318,17 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() {
// The sixth part was staged, but the peer closes before response headers.
drop(socket);
let (mut socket, _) = listener.accept().await.unwrap();
attempts.push(read_request(&mut socket).await);
let (path, body) = read_request(&mut socket).await;
let reported = *during_request.lock().unwrap().last().unwrap();
assert_eq!(
reported,
(5, acknowledged_bytes),
"retry must not report the lost acknowledgement"
);
acknowledged_bytes += body.len();
attempts.push((path, body));
respond(&mut socket, 200, "{}").await;
attempts
(attempts, acknowledged_bytes)
});
let client = RestfulLanceDbClient::try_new(
&crate::remote::client::parse_db_url("db://test").unwrap(),
@@ -316,7 +351,10 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() {
.unwrap();
let input = arrow_array::record_batch!(("id", Int32, [1, 2, 3, 4, 5, 6])).unwrap();
let plan = Arc::new(
crate::table::datafusion::scannable_exec::ScannableExec::new(Box::new(input), None),
crate::table::datafusion::scannable_exec::ScannableExec::new(
Box::new(input),
Some(tracker.clone()),
),
);
let exec = super::insert::RemoteWriteExec::new_multipart(
"test".into(),
@@ -325,7 +363,7 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() {
plan,
false,
"upload-1".into(),
None,
Some(tracker),
None,
Some(1),
None,
@@ -335,7 +373,15 @@ async fn multipart_retries_sixth_part_after_reused_connection_closes() {
.try_collect::<Vec<_>>()
.await
.unwrap();
let attempts = server.await.unwrap();
let (attempts, acknowledged_bytes) = server.await.unwrap();
let progress = progress.lock().unwrap();
assert_eq!(*progress.first().unwrap(), (0, 0));
assert_eq!(*progress.last().unwrap(), (6, acknowledged_bytes));
assert!(
progress
.windows(2)
.all(|pair| pair[0].0 <= pair[1].0 && pair[0].1 <= pair[1].1)
);
assert_eq!(attempts.len(), 7);
assert_eq!(
attempts[5], attempts[6],
@@ -424,7 +470,12 @@ async fn multipart_rejections_preserve_progress_and_cleanup_boundaries() {
let progress = progress.lock().unwrap();
let last = progress.last().unwrap();
assert!(last.2);
assert_eq!(last.0, 2);
assert_eq!(
progress.first().unwrap().0,
0,
"scan is not an acknowledgement"
);
assert_eq!(last.0, if stage_status == 200 { 2 } else { 0 });
if stage_status != 200 {
assert!(
progress.iter().all(|p| p.1 == 0),
+28 -10
View File
@@ -30,9 +30,11 @@ impl WriteProgress {
self.elapsed
}
/// Number of rows processed by the input scan.
/// Number of rows processed by the writer.
///
/// This does not indicate that the rows have been committed.
/// Remote multipart writes count rows in acknowledged IPC parts, starting
/// at zero and counting retransmissions once. Other writers report rows
/// processed by the input scan. Neither indicates a successful commit.
pub fn output_rows(&self) -> usize {
self.output_rows
}
@@ -101,8 +103,8 @@ pub(crate) struct WriteProgressTracker {
/// Wire bytes tracked separately by the insert layer. When set (> 0),
/// this takes precedence over the in-memory bytes from `rows_and_bytes`.
wire_bytes: AtomicUsize,
/// Multipart writes select this before scanning, including zero acknowledgements.
wire_bytes_enabled: AtomicBool,
/// Multipart progress excludes input scanning and counts only acknowledged parts.
acknowledged_parts: AtomicBool,
active_tasks: Arc<AtomicUsize>,
total_tasks: AtomicUsize,
start: Instant,
@@ -116,7 +118,7 @@ impl WriteProgressTracker {
Self {
rows_and_bytes: std::sync::Mutex::new((0, 0)),
wire_bytes: AtomicUsize::new(0),
wire_bytes_enabled: AtomicBool::new(false),
acknowledged_parts: AtomicBool::new(false),
active_tasks: Arc::new(AtomicUsize::new(0)),
total_tasks: AtomicUsize::new(1),
start: Instant::now(),
@@ -125,9 +127,9 @@ impl WriteProgressTracker {
}
}
/// Select wire-byte progress, including zero, before polling the input plan.
pub fn use_wire_bytes(&self) {
self.wire_bytes_enabled.store(true, Ordering::Relaxed);
/// Select acknowledged rows and wire bytes before polling the input plan.
pub fn use_acknowledged_parts(&self) {
self.acknowledged_parts.store(true, Ordering::Relaxed);
}
/// Set the total number of parallel write tasks (the write parallelism).
@@ -145,13 +147,29 @@ impl WriteProgressTracker {
pub fn record_batch(&self, rows: usize, bytes: usize) {
// Lock order: callback first, then rows_and_bytes. This is the only
// order used anywhere, so deadlocks cannot occur.
let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner());
let mut guard = self
.rows_and_bytes
.lock()
.unwrap_or_else(|e| e.into_inner());
if !self.acknowledged_parts.load(Ordering::Relaxed) {
guard.0 += rows;
guard.1 += bytes;
}
let progress = self.snapshot(guard.0, guard.1, false);
drop(guard);
cb(&progress);
}
/// Record one successfully staged part, after any retries have finished.
pub fn record_acknowledged_part(&self, rows: usize, bytes: usize) {
let mut cb = self.callback.lock().unwrap_or_else(|e| e.into_inner());
let mut guard = self
.rows_and_bytes
.lock()
.unwrap_or_else(|e| e.into_inner());
guard.0 += rows;
guard.1 += bytes;
self.wire_bytes.fetch_add(bytes, Ordering::Relaxed);
let progress = self.snapshot(guard.0, guard.1, false);
drop(guard);
cb(&progress);
@@ -194,7 +212,7 @@ impl WriteProgressTracker {
let wire = self.wire_bytes.load(Ordering::Relaxed);
// Prefer wire bytes (actual I/O size) when the insert layer is
// tracking them; fall back to in-memory Arrow size otherwise.
let output_bytes = if self.wire_bytes_enabled.load(Ordering::Relaxed) || wire > 0 {
let output_bytes = if self.acknowledged_parts.load(Ordering::Relaxed) || wire > 0 {
wire
} else {
in_memory_bytes