feat(fetch): add acknowledged browser upload reporting

Separate transport byte accounting from browser upload notifications. Poll on the existing curl owner with a 100ms cadence, coalesce reports until owner acknowledgement, and flush at final response headers/completion. Queued reports retain only weak observer identity; native immediate observers keep their existing behavior.

Cover held responses, acknowledgement backpressure, final-header flush, empty/absent bodies, early responses and cancellation with real TCP regressions.

Validation of the combined final tree: cargo fmt --all; cargo clippy --workspace --all-targets --all-features -- -D warnings; cargo nextest run --no-fail-fast (23,723 passed, 18 existing skips, 32 threads, zero retries). Fresh normal CGU=1 release and independent CLI/CDP regressions passed.
This commit is contained in:
ldm0
2026-11-20 22:47:57 +08:00
parent 7e22ddb480
commit 03c2e43c50
8 changed files with 477 additions and 39 deletions
+6
View File
@@ -54,6 +54,12 @@ impl<H: Handler, C> HttpRegistry<H, C> {
!self.active.is_empty()
}
pub(crate) fn poll_handlers(&mut self, poll: fn(&mut H)) {
for active in self.active.values_mut() {
poll(active.handle.get_mut());
}
}
/// DNS-only work can wait on its completion channel without polling curl.
pub(crate) fn has_curl_work(&self) -> bool {
self.has_active() || !self.pending.is_empty()
+17
View File
@@ -47,6 +47,22 @@ pub struct CurlMultiRuntime<H: Handler + Send + 'static, C: Send + 'static> {
impl<H: Handler + Send + 'static, C: Send + 'static> CurlMultiRuntime<H, C> {
pub fn new(
config: CurlMultiRuntimeConfig,
) -> Result<(Self, Receiver<CurlMultiCompletion<H, C>>)> {
Self::construct(config, None)
}
/// Polls active HTTP handlers on their existing owner between curl turns.
/// The callback must be nonblocking; it cannot enter a script engine.
pub fn new_with_http_handler_poll(
config: CurlMultiRuntimeConfig,
poll: fn(&mut H),
) -> Result<(Self, Receiver<CurlMultiCompletion<H, C>>)> {
Self::construct(config, Some(poll))
}
fn construct(
config: CurlMultiRuntimeConfig,
http_handler_poll: Option<fn(&mut H)>,
) -> Result<(Self, Receiver<CurlMultiCompletion<H, C>>)> {
config.validate()?;
let (command_tx, command_rx) = crossbeam_channel::unbounded();
@@ -70,6 +86,7 @@ impl<H: Handler + Send + 'static, C: Send + 'static> CurlMultiRuntime<H, C> {
waker_tx,
owner_shutdown,
websocket_rx,
http_handler_poll,
#[cfg(test)]
started,
)
+6
View File
@@ -45,6 +45,7 @@ pub(super) struct CurlRuntimeOwner<H: Handler, C> {
shutdown_requested: Arc<AtomicBool>,
closed: bool,
poll_interval: Duration,
http_handler_poll: Option<fn(&mut H)>,
diagnostics: Diagnostics,
http: HttpRegistry<H, C>,
websockets: WebSocketRegistry,
@@ -66,6 +67,7 @@ impl<H: Handler, C> CurlRuntimeOwner<H, C> {
waker_tx: Sender<MultiWaker>,
shutdown_requested: Arc<AtomicBool>,
websocket_rx: Receiver<Submission>,
http_handler_poll: Option<fn(&mut H)>,
#[cfg(test)] owner_started: Arc<AtomicBool>,
) {
#[cfg(test)]
@@ -79,6 +81,7 @@ impl<H: Handler, C> CurlRuntimeOwner<H, C> {
shutdown_requested,
closed: false,
poll_interval: config.poll_interval,
http_handler_poll,
diagnostics: Diagnostics::from_env(),
http: HttpRegistry::new(config, completion_tx),
websockets: WebSocketRegistry::new(websocket_rx, owner_waker),
@@ -95,6 +98,9 @@ impl<H: Handler, C> CurlRuntimeOwner<H, C> {
self.drain_commands();
self.http.advance(&mut self.multi);
self.process_completed_transfers();
if let Some(poll) = self.http_handler_poll {
self.http.poll_handlers(poll);
}
if self.websocket_multi.is_none() && !self.websockets.submissions().is_empty() {
self.websocket_multi = Some(make_websocket_multi(self.max_websocket_connections));
}
+1
View File
@@ -109,6 +109,7 @@ fn completion_starts_queued_job(per_origin: bool) {
shutdown_requested,
closed: false,
poll_interval: config.poll_interval,
http_handler_poll: None,
diagnostics: Diagnostics::from_env(),
http: HttpRegistry::new(config, completion_tx),
websockets: WebSocketRegistry::new(websocket_rx, multi.waker()),
+1 -1
View File
@@ -107,5 +107,5 @@ pub use runtime::{
FetchRuntimeIdentity, FetchRuntimeJoinReport, FetchRuntimeJoinStatus, FetchRuntimePanicReport,
};
pub use streaming_response::{StreamingHtmlResponse, StreamingRawResponse};
pub use upload::{UploadEvent, UploadObserver};
pub use upload::{UploadEvent, UploadObserver, UploadReport};
pub use url_pattern::url_pattern_matches;
+40 -2
View File
@@ -400,8 +400,11 @@ impl FetchRuntimeOwner {
let shutdown_requested = Arc::new(AtomicBool::new(false));
#[cfg(test)]
let owner_started = Arc::new(AtomicBool::new(false));
let (curl_runtime, curl_completion_rx) = CurlMultiRuntime::new(curl_runtime_config(config))
.expect("failed to start fetch curl multi runtime");
let (curl_runtime, curl_completion_rx) = CurlMultiRuntime::new_with_http_handler_poll(
curl_runtime_config(config),
FetchTransferHandler::poll_upload_progress,
)
.expect("failed to start fetch curl multi runtime");
let websocket_connector = curl_runtime
.websocket_connector()
.with_network_address_policy(config.network_address_policy());
@@ -1616,6 +1619,13 @@ impl RuntimeOwner {
state: &mut OwnerState,
mut completion: RuntimeCurlCompletion,
) {
if let Some(observer) = completion
.easy
.as_ref()
.and_then(|easy| easy.get_ref().upload_observer.as_ref())
{
observer.finish_reporting();
}
completion
.context
.response_progress()
@@ -3136,6 +3146,8 @@ pub(crate) struct FetchTransferHandler {
response_body_observed: bool,
network_observation_recorder: Option<NetworkObservationRecorder>,
upload_observer: Option<crate::UploadObserver>,
upload_response_status: Option<u16>,
upload_request_is_connect: bool,
proxy_connect_response_recorder: ProxyConnectResponseRecorder,
dictionary_response: Option<crate::compression_dictionary::DictionaryResponse>,
http_authentication: Option<crate::http_authentication::HttpAuthenticationResponse>,
@@ -3191,6 +3203,8 @@ impl FetchTransferHandler {
response_body_observed: false,
network_observation_recorder: None,
upload_observer: None,
upload_response_status: None,
upload_request_is_connect: false,
proxy_connect_response_recorder: ProxyConnectResponseRecorder::default(),
dictionary_response: None,
http_authentication: None,
@@ -3249,11 +3263,22 @@ impl FetchTransferHandler {
) {
self.response_body_observed = false;
self.upload_observer = upload_observer;
self.upload_response_status = None;
self.upload_request_is_connect = false;
if let Some(observer) = &self.upload_observer {
observer.begin_transfer();
}
self.network_observation_recorder = network_observation_recorder;
self.proxy_connect_response_recorder
.begin_transfer(capture_proxy_connect_response);
}
fn poll_upload_progress(&mut self) {
if let Some(observer) = &self.upload_observer {
observer.report_if_due();
}
}
pub(crate) fn begin_dictionary_response(&mut self, request: &Request, url: &Url) {
self.http_authentication =
crate::http_authentication::HttpAuthenticationResponse::new(request);
@@ -3430,6 +3455,18 @@ impl Handler for FetchTransferHandler {
}
fn header(&mut self, data: &[u8]) -> bool {
if let Some(status) = crate::response::parse_http_response_status_line(data) {
self.upload_response_status = Some(status.status);
} else if data.iter().all(|byte| matches!(*byte, b'\r' | b'\n'))
&& self
.upload_response_status
.take()
.is_some_and(|status| status >= 200)
&& !self.upload_request_is_connect
&& let Some(observer) = &self.upload_observer
{
observer.finish_reporting();
}
if let Some(recorder) = self.network_observation_recorder.as_ref() {
recorder.record_response_header_line(data);
}
@@ -3481,6 +3518,7 @@ impl Handler for FetchTransferHandler {
let is_proxy_connect = self
.proxy_connect_response_recorder
.record_outgoing_header_block(data);
self.upload_request_is_connect = is_proxy_connect;
if !is_proxy_connect && let Some(observer) = &self.upload_observer {
observer.request_headers_sent();
}
+232 -32
View File
@@ -1,5 +1,196 @@
use super::*;
use crate::{UploadEvent, UploadObserver};
use std::time::Instant;
#[tokio::test]
async fn browser_upload_reports_progress_while_response_is_held() -> Result<()> {
for transport in ["buffered", "html", "raw"] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/browser-upload", listener.local_addr()?);
let (release_tx, release_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let _ = read_http_request_head(&mut stream).await.unwrap();
let mut body = [0; 12];
stream.read_exact(&mut body).await.unwrap();
assert_eq!(&body, b"Test Message");
release_rx.await.unwrap();
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
.await
.unwrap();
});
let (event_tx, mut events) = mpsc::unbounded_channel();
let started = Instant::now();
let observer = UploadObserver::new_browser(12, move |report| {
let _ = event_tx.send((report.event, Instant::now()));
});
let request = Request::new_bytes(
"POST",
&url,
Some(b"Test Message".to_vec().into()),
Vec::new(),
)?
.with_upload_observer(observer);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let fetch = upload_response(&client, request, transport, FetchCancelHandle::new());
let observe = async {
let (event, at) = tokio::time::timeout(Duration::from_secs(3), events.recv())
.await
.unwrap()
.unwrap();
assert_eq!(
event,
UploadEvent::Complete {
loaded: 12,
total: 12
}
);
assert!(
at.duration_since(started) >= Duration::from_millis(90),
"browser progress must follow the reporting cadence: {transport}"
);
release_tx.send(()).unwrap();
};
let (result, ()) = tokio::join!(fetch, observe);
assert_eq!(result?, 200);
assert!(
events.try_recv().is_err(),
"duplicate final progress: {transport}"
);
server.await?;
}
Ok(())
}
#[tokio::test]
async fn browser_upload_holds_reports_until_owner_acknowledges() -> Result<()> {
for flush_at_response_start in [false, true] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/backpressure", listener.local_addr()?);
let total = 16 * 1024 * 1024;
let (received_tx, received_rx) = oneshot::channel();
let (read_tx, read_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let _ = read_http_request_head(&mut stream).await.unwrap();
// Keep the receive window closed until a partial report is observed.
// This also makes the first report independent of test scheduling.
read_rx.await.unwrap();
let mut body = vec![0; total];
stream.read_exact(&mut body).await.unwrap();
if flush_at_response_start {
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n")
.await
.unwrap();
}
received_tx.send(()).unwrap();
release_rx.await.unwrap();
if !flush_at_response_start {
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n")
.await
.unwrap();
}
stream.write_all(b"ok").await.unwrap();
});
let (event_tx, mut events) = mpsc::unbounded_channel();
let observer = UploadObserver::new_browser(total as u64, move |report| {
let _ = event_tx.send(report);
});
let request = Request::new_bytes(
"POST",
&url,
Some(vec![b'x'; total].into()),
vec![("Expect".into(), String::new())],
)?
.with_upload_observer(observer);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let fetch = upload_response(&client, request, "raw", FetchCancelHandle::new());
let observe = async {
let report = tokio::time::timeout(Duration::from_secs(3), events.recv())
.await
.unwrap()
.unwrap();
assert!(
matches!(report.event, UploadEvent::Progress { loaded, total } if loaded > 0 && loaded < total)
);
read_tx.send(()).unwrap();
received_rx.await.unwrap();
let report = if flush_at_response_start {
// Response headers must flush the final byte counts even while
// the previous report is unacknowledged and the body is held.
Some(report)
} else {
tokio::time::sleep(Duration::from_millis(220)).await;
assert!(
events.try_recv().is_err(),
"pending owner must coalesce further upload reports"
);
drop(report);
None
};
let final_report = tokio::time::timeout(Duration::from_secs(3), events.recv())
.await
.unwrap()
.unwrap();
assert_eq!(
final_report.event,
UploadEvent::Complete {
loaded: total as u64,
total: total as u64
}
);
drop(report);
drop(final_report);
release_tx.send(()).unwrap();
};
let (result, ()) = tokio::join!(fetch, observe);
assert_eq!(result?, 200);
server.await?;
}
Ok(())
}
#[tokio::test]
async fn browser_upload_does_not_report_empty_or_absent_bodies() -> Result<()> {
for transport in ["buffered", "html", "raw"] {
for body in [None, Some(Vec::new())] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/empty-upload", listener.local_addr()?);
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let _ = read_http_request_head(&mut stream).await.unwrap();
stream
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)
.await
.unwrap();
});
let (event_tx, mut events) = mpsc::unbounded_channel();
let observer = UploadObserver::new_browser(0, move |report| {
let _ = event_tx.send(report.event);
});
let request = Request::new_bytes("POST", &url, body.map(Into::into), Vec::new())?
.with_upload_observer(observer);
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
assert_eq!(
upload_response(&client, request, transport, FetchCancelHandle::new()).await?,
200
);
assert!(
events.try_recv().is_err(),
"empty upload reported progress: {transport}"
);
server.await?;
}
}
Ok(())
}
async fn upload_response(
client: &FetchClient,
@@ -102,39 +293,48 @@ async fn upload_observer_finishes_before_response_and_distinguishes_empty_from_a
#[tokio::test]
async fn upload_observer_does_not_infer_completion_from_an_early_response() -> Result<()> {
for transport in ["buffered", "html", "raw"] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/reject", listener.local_addr()?);
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let head = read_http_request_head(&mut stream).await.unwrap();
assert!(
head.to_ascii_lowercase()
.contains("expect: 100-continue\r\n")
for browser in [false, true] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/reject", listener.local_addr()?);
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let head = read_http_request_head(&mut stream).await.unwrap();
assert!(
head.to_ascii_lowercase()
.contains("expect: 100-continue\r\n")
);
stream.write_all(b"HTTP/1.1 413 Content Too Large\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.unwrap();
});
let (event_tx, mut events) = mpsc::unbounded_channel();
let body = vec![b'x'; 1024 * 1024];
let observer = if browser {
UploadObserver::new_browser(body.len() as u64, move |report| {
let _ = event_tx.send(report.event);
})
} else {
UploadObserver::new(body.len() as u64, move |event| {
let _ = event_tx.send(event);
})
};
let request = Request::new_bytes(
"POST",
&url,
Some(body.into()),
vec![("Expect".into(), "100-continue".into())],
)?
.with_upload_observer(observer);
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
assert_eq!(
upload_response(&client, request, transport, FetchCancelHandle::new()).await?,
413
);
stream.write_all(b"HTTP/1.1 413 Content Too Large\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.unwrap();
});
let (event_tx, mut events) = mpsc::unbounded_channel();
let body = vec![b'x'; 1024 * 1024];
let observer = UploadObserver::new(body.len() as u64, move |event| {
let _ = event_tx.send(event);
});
let request = Request::new_bytes(
"POST",
&url,
Some(body.into()),
vec![("Expect".into(), "100-continue".into())],
)?
.with_upload_observer(observer);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
assert_eq!(
upload_response(&client, request, transport, FetchCancelHandle::new()).await?,
413
);
assert!(
events.try_recv().is_err(),
"unsent body reported as uploaded: {transport}"
);
server.await?;
assert!(
events.try_recv().is_err(),
"unsent body reported as uploaded: {transport}"
);
server.await?;
}
}
Ok(())
}
+174 -4
View File
@@ -1,7 +1,7 @@
use parking_lot::Mutex;
use std::{
fmt,
sync::Arc,
sync::{Arc, Weak},
time::{Duration, Instant},
};
@@ -20,14 +20,60 @@ pub struct UploadObserver(Arc<UploadObserverInner>);
struct UploadObserverInner {
total: u64,
callback: Box<dyn Fn(UploadEvent) + Send + Sync>,
callback: UploadCallback,
state: Mutex<UploadState>,
}
enum UploadCallback {
Transmission(Box<dyn Fn(UploadEvent) + Send + Sync>),
Browser(Box<dyn Fn(UploadReport) + Send + Sync>),
}
const BROWSER_PROGRESS_INTERVAL: Duration = Duration::from_millis(100);
#[derive(Clone, Copy)]
enum BrowserSchedule {
Dormant,
Running(Instant),
Finished,
}
struct UploadState {
loaded: u64,
last_progress: Instant,
complete: bool,
browser: Option<BrowserSchedule>,
reported: u64,
waiting_for_ack: bool,
generation: u64,
}
/// A browser upload notification, acknowledged when its script owner applies
/// or discards it. Dropping a stale report cannot release a newer report, and
/// a queued report does not retain its request.
pub struct UploadReport {
pub event: UploadEvent,
observer: Weak<UploadObserverInner>,
generation: u64,
}
impl fmt::Debug for UploadReport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("UploadReport")
.field("event", &self.event)
.finish_non_exhaustive()
}
}
impl Drop for UploadReport {
fn drop(&mut self) {
if let Some(observer) = self.observer.upgrade() {
let mut state = observer.state.lock();
if state.generation == self.generation {
state.waiting_for_ack = false;
}
}
}
}
impl fmt::Debug for UploadObserver {
@@ -42,15 +88,131 @@ impl UploadObserver {
pub fn new(total: u64, callback: impl Fn(UploadEvent) + Send + Sync + 'static) -> Self {
Self(Arc::new(UploadObserverInner {
total,
callback: Box::new(callback),
callback: UploadCallback::Transmission(Box::new(callback)),
state: Mutex::new(UploadState {
loaded: 0,
last_progress: Instant::now(),
complete: false,
browser: None,
reported: 0,
waiting_for_ack: false,
generation: 0,
}),
}))
}
/// Browser upload reports use the network service's periodic cadence,
/// owner acknowledgements and final response-start flush. Transmission
/// itself continues independently of reporting or the script owner.
pub fn new_browser(
total: u64,
callback: impl Fn(UploadReport) + Send + Sync + 'static,
) -> Self {
Self(Arc::new(UploadObserverInner {
total,
callback: UploadCallback::Browser(Box::new(callback)),
state: Mutex::new(UploadState {
loaded: 0,
last_progress: Instant::now(),
complete: false,
browser: Some(BrowserSchedule::Dormant),
reported: 0,
waiting_for_ack: false,
generation: 0,
}),
}))
}
pub(crate) fn begin_transfer(&self) {
let mut state = self.0.state.lock();
if state.browser.is_some() && !state.complete {
state.browser = Some(BrowserSchedule::Running(
Instant::now() + BROWSER_PROGRESS_INTERVAL,
));
state.waiting_for_ack = false;
state.generation = state.generation.wrapping_add(1);
}
}
pub(crate) fn report_if_due(&self) {
let now = Instant::now();
let report = {
let mut state = self.0.state.lock();
let Some(BrowserSchedule::Running(deadline)) = state.browser else {
return;
};
if now < deadline {
return;
}
state.browser = Some(BrowserSchedule::Running(now + BROWSER_PROGRESS_INTERVAL));
self.prepare_browser_report(&mut state, now, false)
};
self.publish_browser_report(report);
}
pub(crate) fn finish_reporting(&self) {
let report = {
let mut state = self.0.state.lock();
if !matches!(state.browser, Some(BrowserSchedule::Running(_))) {
return;
}
let report = self.prepare_browser_report(&mut state, Instant::now(), true);
state.browser = Some(BrowserSchedule::Finished);
report
};
self.publish_browser_report(report);
}
fn prepare_browser_report(
&self,
state: &mut UploadState,
now: Instant,
final_report: bool,
) -> Option<(UploadEvent, u64)> {
if final_report {
state.waiting_for_ack = false;
}
let total = self.0.total;
if state.waiting_for_ack || total == 0 || state.loaded <= state.reported {
return None;
}
let loaded = state.loaded;
let finished = loaded == total;
if !finished
&& state.reported != 0
&& loaded - state.reported <= total / 200
&& now.duration_since(state.last_progress) <= Duration::from_secs(1)
{
return None;
}
state.reported = loaded;
state.last_progress = now;
state.complete = finished;
state.waiting_for_ack = true;
state.generation = state.generation.wrapping_add(1);
if finished {
state.browser = Some(BrowserSchedule::Finished);
}
let event = if finished {
UploadEvent::Complete { loaded, total }
} else {
UploadEvent::Progress { loaded, total }
};
Some((event, state.generation))
}
fn publish_browser_report(&self, report: Option<(UploadEvent, u64)>) {
if let Some((event, generation)) = report
&& let UploadCallback::Browser(callback) = &self.0.callback
{
callback(UploadReport {
event,
observer: Arc::downgrade(&self.0),
generation,
});
}
}
pub(crate) fn request_headers_sent(&self) {
// There is no positive curl upload count for a present, empty body.
// Its request headers, unlike a response, establish that it was sent.
@@ -73,6 +235,12 @@ impl UploadObserver {
return;
}
let loaded = loaded.min(total);
if let Some(schedule) = &state.browser {
if !matches!(schedule, BrowserSchedule::Finished) {
state.loaded = state.loaded.max(loaded);
}
return;
}
let progress = if loaded > state.loaded {
state.loaded = loaded;
let now = Instant::now();
@@ -94,7 +262,9 @@ impl UploadObserver {
// Keep the byte-progress and end-of-body tasks distinct. In particular,
// a consumer can abort while processing the former before the latter.
for event in progress.into_iter().chain(completion) {
(self.0.callback)(event);
if let UploadCallback::Transmission(callback) = &self.0.callback {
callback(event);
}
}
}
}