diff --git a/Cargo.lock b/Cargo.lock index 8f1876a3b96..934f860d149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2987,6 +2987,7 @@ dependencies = [ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "rcgen", "serde", + "serde_json", "strum", "thiserror", "tokio", diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.toml index 09b77005663..a178bcdc557 100644 --- a/moli-fetch/Cargo.toml +++ b/moli-fetch/Cargo.toml @@ -41,6 +41,7 @@ url = "2.5.7" wildcard = "0.3.0" [dev-dependencies] +serde_json = "1" h2 = "0.4" moli-cookie-jar = { path = "../moli-cookie-jar", features = ["test-support"] } rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs", "pem"] } diff --git a/moli-fetch/src/blocking/cache.rs b/moli-fetch/src/blocking/cache.rs index 28b31dd26ad..95a4744b23e 100644 --- a/moli-fetch/src/blocking/cache.rs +++ b/moli-fetch/src/blocking/cache.rs @@ -89,6 +89,7 @@ pub(crate) struct CachedStreamingResponseLookup { pub(crate) body: std::io::BufReader, pub(crate) needs_validation: bool, pub(crate) cache_state: crate::ResponseCacheState, + pub(crate) body_sizes: crate::FetchResponseBodySizes, } pub(crate) fn cached_streaming_response_needs_validation( diff --git a/moli-fetch/src/blocking/cache/lookup.rs b/moli-fetch/src/blocking/cache/lookup.rs index f298daa540d..38161bc36c6 100644 --- a/moli-fetch/src/blocking/cache/lookup.rs +++ b/moli-fetch/src/blocking/cache/lookup.rs @@ -26,9 +26,21 @@ pub(crate) fn load_cached_streaming_response_lookup( else { return Ok(None); }; - let Some(cached) = store.load_reader(&key)? else { + let Some(mut cached) = store.load_reader(&key)? else { return Ok(None); }; + if cached.metadata.encoded_body_size.is_none() { + // A decoded cache file cannot recover an older compressed wire size. + // Identity entries can be upgraded from their actual body file length. + if cached.metadata.headers.iter().any(|(name, value)| { + name.eq_ignore_ascii_case("content-encoding") + && !value.is_empty() + && !value.eq_ignore_ascii_case(b"identity") + }) { + return Ok(None); + } + cached.metadata.encoded_body_size = Some(cached.body_len()?); + } let cache_request_url = normalized_cache_url_string(request_url); if cached.metadata.request_url != cache_request_url || !vary_headers_match(config, request, request_url, &cached.metadata.vary_headers) @@ -42,7 +54,6 @@ pub(crate) fn load_cached_streaming_response_lookup( ); return Ok(None); } - let mut cached = cached; // Normalized cache keys should not leak the first request's fragment into // later streaming hits. cached.metadata.final_url = request_url.to_string(); @@ -63,7 +74,7 @@ pub(crate) fn load_cached_streaming_response_lookup( key, cached, needs_validation, - ))) + )?)) } pub(crate) fn cached_streaming_response_body_exceeds_response_limit( @@ -135,13 +146,14 @@ pub(crate) fn finish_streaming_cached_response( status_text: Option<&str>, headers: &[(String, Vec)], redirected: bool, + body_sizes: Option, writer: HttpCacheBodyWriter, ) -> Result<()> { let Some((_, _)) = cache_store_and_key_for_request(config, request, request_url, cookie_header) else { return Ok(()); }; - let Some(metadata) = cache_metadata_for_response_parts( + let Some(mut metadata) = cache_metadata_for_response_parts( config, request, request_url, @@ -154,6 +166,7 @@ pub(crate) fn finish_streaming_cached_response( return Ok(()); }; + metadata.encoded_body_size = body_sizes.map(|sizes| sizes.encoded); writer.finish(metadata)?; tracing::debug!(url = %request_url, "stored streaming response in disk cache"); Ok(()) @@ -163,10 +176,18 @@ fn cached_streaming_record_from_entry( key: String, entry: HttpCachedEntryReader, needs_validation: bool, -) -> CachedStreamingResponseLookup { +) -> Result { + let body_sizes = crate::FetchResponseBodySizes { + encoded: entry + .metadata + .encoded_body_size + .expect("cache lookup resolved encoded size"), + decoded: entry.body_len()?, + }; let metadata = entry.metadata.clone(); - CachedStreamingResponseLookup { + Ok(CachedStreamingResponseLookup { key, + body_sizes, metadata, final_url: entry.metadata.final_url, status: entry.metadata.status, @@ -174,7 +195,7 @@ fn cached_streaming_record_from_entry( body: entry.body, needs_validation, cache_state: crate::ResponseCacheState::Local, - } + }) } fn entry_reader_body_exceeds_response_limit( diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rs index 6ad41f62f19..630d10fb504 100644 --- a/moli-fetch/src/cancellation.rs +++ b/moli-fetch/src/cancellation.rs @@ -26,7 +26,18 @@ struct ResponseProgressState { timing: Mutex, } +/// Native counts of encoded receipt and decoded representation, excluding framing. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct FetchResponseBodySizes { + pub encoded: u64, + pub decoded: u64, +} + impl ResponseProgress { + pub(crate) fn timing(&self) -> FetchResponseTiming { + *self.inner.timing.lock() + } + pub(crate) fn mark_request_started(&self) { self.inner .timing @@ -62,11 +73,30 @@ impl ResponseProgress { } pub(crate) fn mark_response_started(&self) { - self.inner - .timing - .lock() - .response_started_at - .get_or_insert_with(Instant::now); + let mut timing = self.inner.timing.lock(); + timing.response_started_at.get_or_insert_with(Instant::now); + timing.body_sizes.get_or_insert_with(Default::default); + } + + pub(crate) fn reset_response_body_sizes(&self) { + let mut timing = self.inner.timing.lock(); + timing.body_sizes = Some(FetchResponseBodySizes::default()); + timing.body_bytes_received = false; + } + + pub(crate) fn record_encoded_response_body_size(&self, bytes: u64) { + let mut timing = self.inner.timing.lock(); + timing + .body_sizes + .get_or_insert_with(Default::default) + .encoded = bytes; + timing.body_bytes_received |= bytes > 0; + } + + pub(crate) fn record_decoded_response_body_size(&self, bytes: usize) { + let mut timing = self.inner.timing.lock(); + let sizes = timing.body_sizes.get_or_insert_with(Default::default); + sizes.decoded = sizes.decoded.saturating_add(bytes as u64); } fn finish_timing(&self) { @@ -77,8 +107,12 @@ impl ResponseProgress { .get_or_insert_with(Instant::now); } - pub(crate) fn mark_cached_response_available(&self) -> CachedResponseBodyTiming { + pub(crate) fn mark_cached_response_available( + &self, + sizes: FetchResponseBodySizes, + ) -> CachedResponseBodyTiming { self.mark_response_started(); + self.inner.timing.lock().body_sizes = Some(sizes); self.commit_state(); CachedResponseBodyTiming(self.clone()) } @@ -100,6 +134,7 @@ impl ResponseProgress { /// Native response boundaries shared with observers of the selected transfer. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct FetchResponseTiming { + body_sizes: Option, request_started_at: Option, first_interim_response_started_at: Option, body_bytes_received: bool, @@ -108,6 +143,9 @@ pub struct FetchResponseTiming { } impl FetchResponseTiming { + pub fn body_sizes(self) -> Option { + self.body_sizes + } pub fn body_bytes_received(self) -> bool { self.body_bytes_received } @@ -255,6 +293,11 @@ impl FetchCancelHandle { progress } + /// Restore representation counts without changing this transfer's clock. + pub fn set_response_body_sizes(&self, sizes: FetchResponseBodySizes) { + self.response_progress().inner.timing.lock().body_sizes = Some(sizes); + } + pub fn response_timing(&self) -> FetchResponseTiming { *self.response_progress().inner.timing.lock() } @@ -270,8 +313,12 @@ impl FetchCancelHandle { } #[cfg(test)] - fn mark_cached_response_available(&self) -> CachedResponseBodyTiming { - self.response_progress().mark_cached_response_available() + fn mark_cached_response_available( + &self, + sizes: FetchResponseBodySizes, + ) -> CachedResponseBodyTiming { + self.response_progress() + .mark_cached_response_available(sizes) } #[cfg(test)] @@ -286,7 +333,7 @@ impl FetchCancelHandle { #[cfg(test)] mod tests { - use super::FetchCancelHandle; + use super::{FetchCancelHandle, FetchResponseBodySizes}; use std::time::Instant; #[test] @@ -651,7 +698,10 @@ mod tests { for reset_before_body_end in [false, true] { let parent = FetchCancelHandle::new(); let response = parent.child_for_subrequest(); - let recorder = response.mark_cached_response_available(); + let recorder = response.mark_cached_response_available(FetchResponseBodySizes { + encoded: 23, + decoded: 3, + }); parent.adopt_response_progress(&response); assert!(parent.response_completion_is_committed()); assert!(parent.response_timing().response_started_at().is_some()); @@ -712,3 +762,40 @@ mod tests { assert_eq!(parent.response_timing(), Default::default()); } } + +#[cfg(test)] +mod body_size_generation_tests { + use super::*; + #[test] + fn response_body_sizes_remain_with_their_transfer_writer() { + let handle = FetchCancelHandle::new(); + let old = handle.begin_response_progress(); + old.mark_response_started(); + old.record_encoded_response_body_size(23); + old.record_decoded_response_body_size(3); + assert_eq!( + handle.response_timing().body_sizes(), + Some(FetchResponseBodySizes { + encoded: 23, + decoded: 3 + }) + ); + let current = handle.begin_response_progress(); + current.mark_response_started(); + current.record_encoded_response_body_size(7); + current.record_decoded_response_body_size(5); + old.record_encoded_response_body_size(99); + old.record_decoded_response_body_size(88); + old.commit(); + assert_eq!( + handle.response_timing().body_sizes(), + Some(FetchResponseBodySizes { + encoded: 7, + decoded: 5 + }) + ); + assert!(!handle.response_completion_is_committed()); + current.commit(); + assert!(handle.response_completion_is_committed()); + } +} diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs index 54fb5268268..05393c29f5e 100644 --- a/moli-fetch/src/lib.rs +++ b/moli-fetch/src/lib.rs @@ -41,7 +41,7 @@ pub use blocking::{ clear_http_cache_root_for_origin, cookie_header_for_request, http_cache_stats, observe_cookie_access_report_for_request, trim_http_cache, }; -pub use cancellation::{FetchCancelHandle, FetchResponseTiming}; +pub use cancellation::{FetchCancelHandle, FetchResponseBodySizes, FetchResponseTiming}; pub use client::{FetchClient, FetchClientHandle}; pub use config::FetchConfig; pub use cors::validate_cors_response_for_origin; diff --git a/moli-fetch/src/response.rs b/moli-fetch/src/response.rs index cbb76748457..bc7ccf0b9b8 100644 --- a/moli-fetch/src/response.rs +++ b/moli-fetch/src/response.rs @@ -776,6 +776,7 @@ pub struct RawResponse { pub status_text: Option, pub headers: Vec<(String, Vec)>, body: ResponseBody, + body_sizes: Option, pub request_cookie_report: Option, pub cookie_set_reports: Vec, pub redirected: bool, @@ -790,6 +791,7 @@ pub struct RawResponse { impl Clone for RawResponse { fn clone(&self) -> Self { Self { + body_sizes: self.body_sizes, final_url: self.final_url.clone(), status: self.status, status_text: self.status_text.clone(), @@ -812,6 +814,15 @@ impl Clone for RawResponse { } impl RawResponse { + pub fn body_sizes(&self) -> Option { + self.body_sizes + } + + pub fn with_body_sizes(mut self, sizes: Option) -> Self { + self.body_sizes = sizes; + self + } + pub fn with_network_request_extra_info( mut self, network_request_extra_info: Option, @@ -874,6 +885,7 @@ impl RawResponse { status_text: head.status_text, headers: head.headers, body: ResponseBody::materialized_bytes(body), + body_sizes: None, request_cookie_report: head.request_cookie_report, cookie_set_reports: head.cookie_set_reports, redirected: head.redirected, @@ -894,6 +906,7 @@ impl RawResponse { status_text: head.status_text, headers: head.headers, body: ResponseBody::materialized_bytes(body), + body_sizes: None, request_cookie_report: head.request_cookie_report, cookie_set_reports: head.cookie_set_reports, redirected: head.redirected, diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs index 7b42eeb1fef..0fe0ec6074d 100644 --- a/moli-fetch/src/runtime.rs +++ b/moli-fetch/src/runtime.rs @@ -2037,6 +2037,7 @@ impl RuntimeOwner { status_text.as_deref(), &headers, false, + job.response_progress.timing().body_sizes(), cache_body_writer, ) { @@ -2100,6 +2101,7 @@ impl RuntimeOwner { status_text.as_deref(), &headers, false, + job.response_progress.timing().body_sizes(), cache_body_writer, ) { @@ -2431,6 +2433,7 @@ impl RuntimeOwner { status_text.as_deref(), &headers, false, + job.response_progress.timing().body_sizes(), cache_body_writer, ) { @@ -2494,6 +2497,7 @@ impl RuntimeOwner { status_text.as_deref(), &headers, false, + job.response_progress.timing().body_sizes(), cache_body_writer, ) { @@ -2541,6 +2545,7 @@ impl RuntimeOwner { status_text.as_deref(), &headers, false, + job.response_progress.timing().body_sizes(), cache_body_writer, ) { @@ -3030,6 +3035,9 @@ impl Handler for FetchTransferHandler { if !data.is_empty() { self.mark_response_body_received(); } + if let Some(timing) = self.response_progress() { + timing.record_decoded_response_body_size(data.len()); + } match &mut self.response { FetchResponseCollector::Buffered(collector) => collector.write(data), FetchResponseCollector::Streaming(collector) => collector.write(data), @@ -3038,6 +3046,12 @@ impl Handler for FetchTransferHandler { } fn header(&mut self, data: &[u8]) -> bool { + if crate::response::parse_http_response_status_line(data) + .is_some_and(|line| line.status >= 200) + && let Some(timing) = self.response_progress() + { + timing.reset_response_body_sizes(); + } if let Some(recorder) = self.network_observation_recorder.as_ref() { recorder.record_response_header_line(data); } @@ -3052,6 +3066,11 @@ impl Handler for FetchTransferHandler { if dlnow > 0.0 { self.mark_response_body_received(); } + if dlnow > 0.0 + && let Some(timing) = self.response_progress() + { + timing.record_encoded_response_body_size(dlnow as u64); + } let keep_going = match &mut self.response { FetchResponseCollector::Buffered(collector) => { collector.progress(dltotal, dlnow, ultotal, ulnow) @@ -3228,6 +3247,7 @@ fn complete_cached_streaming_html_job( let CachedStreamingResponseLookup { metadata, cache_state, + body_sizes, final_url, status, headers, @@ -3250,7 +3270,9 @@ fn complete_cached_streaming_html_job( } }; - let response_timing = job.response_progress.mark_cached_response_available(); + let response_timing = job + .response_progress + .mark_cached_response_available(body_sizes); if let Some(started_tx) = job.started_tx { let _ = started_tx.send(Ok(StreamingHtmlResponseStart { status_text: metadata.status_text, @@ -3355,6 +3377,7 @@ fn complete_cached_streaming_raw_job( let CachedStreamingResponseLookup { metadata, cache_state, + body_sizes, final_url, status, headers, @@ -3377,7 +3400,9 @@ fn complete_cached_streaming_raw_job( } }; - let response_timing = job.response_progress.mark_cached_response_available(); + let response_timing = job + .response_progress + .mark_cached_response_available(body_sizes); if let Some(started_tx) = job.started_tx { let _ = started_tx.send(Ok(StreamingHtmlResponseStart { diff --git a/moli-fetch/src/streaming_response.rs b/moli-fetch/src/streaming_response.rs index 0924a7e7670..39d367d2a20 100644 --- a/moli-fetch/src/streaming_response.rs +++ b/moli-fetch/src/streaming_response.rs @@ -193,6 +193,16 @@ pub struct StreamingRawResponse { } impl StreamingRawResponse { + pub fn response_timing(&self) -> crate::FetchResponseTiming { + self.cancel_handle.response_timing() + } + + /// Shares observations with a tee's separate cancellation owner. This + /// does not give that owner authority to cancel the source transfer. + pub fn share_response_progress(&self, owner: &FetchCancelHandle) { + owner.adopt_response_progress(&self.cancel_handle); + } + pub fn new( final_url: Url, status: u16, @@ -352,11 +362,16 @@ impl StreamingRawResponse { /// Prefer chunked consumption for large bodies. This helper exists for /// compatibility surfaces that still need a complete [`RawResponse`]. pub async fn into_materialized_raw_response(self) -> Result { + let progress = self.cancel_handle.clone(); let network_request_extra_info = self.network_request_extra_info.clone(); let (head, body) = self.into_body(); RawResponse::from_head_and_body_source(head, body) .await - .map(|response| response.with_network_request_extra_info(network_request_extra_info)) + .map(|response| { + response + .with_body_sizes(progress.response_timing().body_sizes()) + .with_network_request_extra_info(network_request_extra_info) + }) } pub async fn into_lossy_materialized_text_response(self) -> Result { diff --git a/moli-fetch/src/tests/response_timing.rs b/moli-fetch/src/tests/response_timing.rs index 68248382032..54e8fb554ae 100644 --- a/moli-fetch/src/tests/response_timing.rs +++ b/moli-fetch/src/tests/response_timing.rs @@ -81,6 +81,17 @@ async fn native_response_timing_records_encoded_bytes_before_decoding_failures() "{method}/{coding}/chunked={chunked}" ); assert!(timing.response_end_at().is_some()); + assert_eq!( + timing.body_sizes(), + Some(crate::FetchResponseBodySizes { + encoded: if method == "GET" { + body.len() as u64 + } else { + 0 + }, + decoded: 0, + }) + ); server.join().expect("encoded response server")?; } Ok(()) @@ -129,6 +140,17 @@ async fn native_response_timing_distinguishes_encoded_empty_bodies_from_head_hea cancel.response_timing().body_bytes_received(), method == "GET" ); + assert_eq!( + cancel.response_timing().body_sizes(), + Some(crate::FetchResponseBodySizes { + encoded: if method == "GET" { + GZIP.len() as u64 + } else { + 0 + }, + decoded: 0, + }) + ); server.join().expect("server thread")?; } Ok(()) @@ -373,3 +395,132 @@ async fn native_request_timing_waits_for_the_proxy_tunnel() -> Result<()> { server.shutdown(); Ok(()) } + +#[tokio::test] +async fn native_response_body_sizes_survive_cache_replay_revalidation_and_legacy_entries() +-> Result<()> { + const GZIP: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 75, 76, 74, 6, 0, 194, 65, 36, 53, 3, 0, 0, 0, + ]; + for coding in ["identity", "gzip"] { + for chunked in [false, true] { + for legacy in [false, true] { + let payload = if coding == "gzip" { + GZIP + } else { + b"abc".as_slice() + }; + let cache_dir = unique_test_cache_dir(); + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let url = format!("http://{}/compressed", listener.local_addr()?); + let expected_hits = if legacy && coding == "gzip" { 3 } else { 2 }; + let server = thread::spawn(move || -> std::io::Result<()> { + for _ in 0..expected_hits { + let (mut stream, _) = listener.accept()?; + let mut request = Vec::new(); + let mut byte = [0]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte)?; + request.push(byte[0]); + } + if String::from_utf8_lossy(&request) + .to_ascii_lowercase() + .contains("if-none-match: \"v1\"") + { + stream.write_all(b"HTTP/1.1 304 Not Modified\r\nCache-Control: max-age=600\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")?; + continue; + } + let framing = if chunked { + "Transfer-Encoding: chunked".to_owned() + } else { + format!("Content-Length: {}", payload.len()) + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Encoding: {coding}\r\nContent-Type: text/plain\r\nETag: \"v1\"\r\nCache-Control: max-age=600\r\n{framing}\r\nConnection: close\r\n\r\n" + )?; + if chunked { + write!(stream, "{:x};test=value\r\n", payload.len())?; + stream.write_all(payload)?; + stream.write_all(b"\r\n0\r\nX-Trailer: present\r\n\r\n")?; + } else { + stream.write_all(payload)?; + } + } + Ok(()) + }); + let mut config = FetchConfig::default(); + config.set_http_cache_dir(Some(cache_dir.display().to_string())); + for (index, mode) in [ + RequestCacheMode::Default, + RequestCacheMode::Default, + RequestCacheMode::Validate, + RequestCacheMode::Default, + ] + .into_iter() + .enumerate() + { + if legacy && index == 1 { + let mut pending = vec![cache_dir.clone()]; + let mut changed = 0; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(directory)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + pending.push(entry.path()); + } else if entry.file_name() == "meta.json" { + let mut metadata: serde_json::Value = + serde_json::from_slice(&fs::read(entry.path())?)?; + metadata + .as_object_mut() + .unwrap() + .remove("encoded_body_size"); + fs::write(entry.path(), serde_json::to_vec(&metadata)?)?; + changed += 1; + } + } + } + assert_eq!(changed, 1); + } + // A new client proves that the counts are stored in the cache, + // rather than retained only by the first transfer's owner. + let client = FetchClient::new(&config, new_shared_browser_cookie_store()); + let cancel = FetchCancelHandle::new(); + let mut response = client + .fetch_raw_stream_with_cancel( + Request::get(&url)?.with_cache_mode(mode), + cancel.clone(), + ) + .await?; + let expected_state = match index { + 0 => crate::ResponseCacheState::None, + 1 if legacy && coding == "gzip" => crate::ResponseCacheState::None, + 2 => crate::ResponseCacheState::Validated, + _ => crate::ResponseCacheState::Local, + }; + assert_eq!( + response.cache_state, expected_state, + "{coding}/chunked={chunked}/legacy={legacy}/index={index}" + ); + let mut decoded = Vec::new(); + while let Some(chunk) = response.next_chunk().await { + decoded.extend_from_slice(&chunk); + } + response.finish().await?; + assert_eq!(decoded, b"abc"); + assert_eq!( + cancel.response_timing().body_sizes(), + Some(crate::FetchResponseBodySizes { + encoded: payload.len() as u64, + decoded: 3 + }), + "{coding}/chunked={chunked}/legacy={legacy}/index={index}" + ); + } + server.join().expect("compressed cache server")?; + fs::remove_dir_all(cache_dir)?; + } + } + } + Ok(()) +} diff --git a/moli-http-cache/src/metadata.rs b/moli-http-cache/src/metadata.rs index be4ae609404..e4363ac44f4 100644 --- a/moli-http-cache/src/metadata.rs +++ b/moli-http-cache/src/metadata.rs @@ -29,6 +29,10 @@ pub struct HttpCacheEntryMetadata { pub status: u16, pub status_text: Option, pub headers: Vec<(String, Vec)>, + /// Original encoded size; cache body files contain decoded bytes. Older + /// entries lack this fact and cannot reconstruct compressed sizes. + #[serde(default)] + pub encoded_body_size: Option, pub stored_at_unix_ms: u64, pub last_used_at_unix_ms: u64, pub expires_at_unix_ms: Option, @@ -54,6 +58,7 @@ impl HttpCacheEntryMetadata { status, status_text: None, headers, + encoded_body_size: None, stored_at_unix_ms, last_used_at_unix_ms: stored_at_unix_ms, expires_at_unix_ms, diff --git a/moli-http-cache/src/tests.rs b/moli-http-cache/src/tests.rs index 3aebfe4eb23..888dd4d6511 100644 --- a/moli-http-cache/src/tests.rs +++ b/moli-http-cache/src/tests.rs @@ -681,6 +681,7 @@ fn missing_body_file_is_treated_as_incomplete_entry() -> Result<()> { status: 200, status_text: None, headers: Vec::new(), + encoded_body_size: None, stored_at_unix_ms: 1, last_used_at_unix_ms: 1, expires_at_unix_ms: Some(2), @@ -806,6 +807,7 @@ fn metadata_body_file_must_not_escape_entry_directory() -> Result<()> { status: 200, status_text: None, headers: Vec::new(), + encoded_body_size: None, stored_at_unix_ms: 1, last_used_at_unix_ms: 1, expires_at_unix_ms: Some(2), diff --git a/moli-renderer-v8/src/context_bootstrap/performance_runtime/fetch_timing.rs b/moli-renderer-v8/src/context_bootstrap/performance_runtime/fetch_timing.rs index 4e18757d706..0f2fb3b4eaa 100644 --- a/moli-renderer-v8/src/context_bootstrap/performance_runtime/fetch_timing.rs +++ b/moli-renderer-v8/src/context_bootstrap/performance_runtime/fetch_timing.rs @@ -18,7 +18,7 @@ pub(crate) struct FetchResourceTiming { #[derive(Clone, Default)] struct FetchResourceResponseTiming { request_start: Option, - body_bytes_received: bool, + body_sizes: Option, final_headers_start: Option, first_interim_start: Option, end: Option, @@ -106,13 +106,14 @@ impl FetchResourceTiming { && final_start.is_none() && first_interim.is_none() && end.is_none() + && native.body_sizes().is_none() { return; } let mut response_timing = self.response_timing.borrow_mut(); let timing = response_timing.get_or_insert_with(Default::default); timing.request_start = request_start; - timing.body_bytes_received = native.body_bytes_received(); + timing.body_sizes = native.body_sizes(); if final_start.is_some() { timing.final_headers_start = final_start; timing.first_interim_start = first_interim; @@ -151,16 +152,7 @@ impl FetchResourceTiming { if readable && head.cache_state == ResponseCacheState::None { entry.initiator_url.clone_from(&self.initiator_url); } - if readable - && self - .response_timing - .borrow() - .as_ref() - .is_some_and(|timing| timing.body_bytes_received) - { - entry.content_encoding = - moli_fetch::resource_timing_content_encoding(&head.headers).to_owned(); - } + self.apply_native_body_sizes(&mut entry, head); if !readable { entry.response_status = 0.0; entry.content_type.clear(); @@ -194,6 +186,28 @@ impl FetchResourceTiming { entry } + fn apply_native_body_sizes(&self, entry: &mut ResourcePerformanceEntry, head: &ResponseHead) { + if let Some(sizes) = self + .response_timing + .borrow() + .as_ref() + .and_then(|timing| timing.body_sizes) + { + entry.encoded_body_size = sizes.encoded as f64; + entry.decoded_body_size = sizes.decoded as f64; + entry.transfer_size = match head.cache_state { + ResponseCacheState::Local => 0.0, + ResponseCacheState::Validated => 300.0, + ResponseCacheState::None => entry.encoded_body_size + 300.0, + }; + entry.content_encoding = if sizes.encoded > 0 { + moli_fetch::resource_timing_content_encoding(&head.headers).to_owned() + } else { + String::new() + }; + } + } + pub(crate) fn failed(&self, url: &Url) -> ResourcePerformanceEntry { let mut entry = ResourcePerformanceEntry::from_network_failure( url.as_str(), diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rs index 1778a9867b9..c2236144a2d 100644 --- a/moli-renderer-v8/src/network/request_client.rs +++ b/moli-renderer-v8/src/network/request_client.rs @@ -1020,6 +1020,9 @@ impl ResourceRequestClient { let mut network_request = request.clone(); let cached = match lookup { Some(RawSubresourceCacheLookup::Local(cached)) => { + if let Some(sizes) = cached.body_sizes() { + cancel_handle.set_response_body_sizes(sizes); + } return Ok(NetworkFetchResult::without_request_observation( streaming_raw_response_from_cached_subresource(*cached)?, )); @@ -1043,13 +1046,16 @@ impl ResourceRequestClient { let observed = if observe_network { self.resource_runtime .client() - .fetch_raw_stream_with_cancel_and_network_metadata(network_request, cancel_handle) + .fetch_raw_stream_with_cancel_and_network_metadata( + network_request, + cancel_handle.clone(), + ) .await? } else { NetworkFetchResult::without_request_observation( self.resource_runtime .client() - .fetch_raw_stream_with_cancel(network_request, cancel_handle) + .fetch_raw_stream_with_cancel(network_request, cancel_handle.clone()) .await?, ) }; @@ -1079,7 +1085,11 @@ impl ResourceRequestClient { head.request_cookie_report = response.request_cookie_report.clone(); head.cookie_set_reports = response.cookie_set_reports.clone(); head.negotiated_http_version = response.negotiated_http_version; - let merged = RawResponse::from_head_and_body(head.clone(), cached.clone_body_bytes()); + if let Some(sizes) = cached.body_sizes() { + cancel_handle.set_response_body_sizes(sizes); + } + let merged = RawResponse::from_head_and_body(head.clone(), cached.clone_body_bytes()) + .with_body_sizes(cached.body_sizes()); if let Some(cache_key) = cache_key && let Some(expires_at) = raw_subresource_memory_cache_expiry(&request, &merged) { @@ -1088,11 +1098,12 @@ impl ResourceRequestClient { .lock() .insert_raw_subresource(cache_key, merged, expires_at, &cache_request_headers); } - let completed = - streaming_raw_response_from_head_and_body(head, cached.clone_body_bytes())? - .with_network_request_extra_info( - response.network_request_extra_info().cloned(), - ); + let completed = streaming_raw_response_from_head_and_body( + head, + cached.clone_body_bytes(), + cached.body_sizes(), + )? + .with_network_request_extra_info(response.network_request_extra_info().cloned()); return Ok(NetworkFetchResult::with_observation_journal( completed, observation_journal, @@ -1145,6 +1156,8 @@ impl ResourceRequestClient { let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel(); let (completion_tx, completion_rx) = tokio::sync::oneshot::channel(); let resource_runtime = self.resource_runtime.clone(); + let outward_cancel = FetchCancelHandle::new(); + response.share_response_progress(&outward_cancel); tokio::spawn(async move { let mut body = Vec::new(); let completion = 'forwarding: { @@ -1179,7 +1192,8 @@ impl ResourceRequestClient { }; if completion.as_ref().is_some_and(Result::is_ok) { - let mut materialized = RawResponse::from_head_and_body(response.head(), body); + let mut materialized = RawResponse::from_head_and_body(response.head(), body) + .with_body_sizes(response.response_timing().body_sizes()); if !request.follow_redirects { // The outer CORS loop owns the logical redirect history. // Retain only this hop's representation, then let each @@ -1213,7 +1227,7 @@ impl ResourceRequestClient { let _ = completion_tx.send(completion); } }); - StreamingRawResponse::new_with_head(head, body_rx, FetchCancelHandle::new(), completion_rx) + StreamingRawResponse::new_with_head(head, body_rx, outward_cancel, completion_rx) } pub(crate) fn fetch_text_callback(&self, request: Request, callback: F) -> Result<()> @@ -1593,7 +1607,11 @@ fn streaming_raw_response_from_cached_subresource( head.cache_state = moli_fetch::ResponseCacheState::Local; head.cookie_set_reports.clear(); mark_redirects_as_cached(&mut head.redirect_chain); - streaming_raw_response_from_head_and_body(head, response.clone_body_bytes()) + streaming_raw_response_from_head_and_body( + head, + response.clone_body_bytes(), + response.body_sizes(), + ) } fn local_text_response(request: &Request) -> Option> { @@ -1609,7 +1627,11 @@ pub(crate) fn streaming_raw_response_from_local_response( response: Response, ) -> Result { let raw_response = response.into_materialized_raw_response(); - streaming_raw_response_from_head_and_body(raw_response.head(), raw_response.clone_body_bytes()) + streaming_raw_response_from_head_and_body( + raw_response.head(), + raw_response.clone_body_bytes(), + raw_response.body_sizes(), + ) } fn response_with_memory_cache_hit(mut response: Response) -> Response { @@ -1634,6 +1656,7 @@ fn mark_redirects_as_cached(redirects: &mut [moli_fetch::RedirectInfo]) { fn streaming_raw_response_from_head_and_body( head: ResponseHead, body: Vec, + body_sizes: Option, ) -> Result { let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel(); let (completion_tx, completion_rx) = tokio::sync::oneshot::channel(); @@ -1644,10 +1667,14 @@ fn streaming_raw_response_from_head_and_body( } drop(body_tx); let _ = completion_tx.send(Ok(())); + let cancel = FetchCancelHandle::new(); + if let Some(sizes) = body_sizes { + cancel.set_response_body_sizes(sizes); + } Ok(StreamingRawResponse::new_with_head( head, body_rx, - FetchCancelHandle::new(), + cancel, completion_rx, )) } diff --git a/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs b/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs index 4ea6ccfb603..bb8e3e9fca6 100644 --- a/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs +++ b/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs @@ -57,6 +57,103 @@ async fn handle_timing_request( let url = url::Url::parse(&format!("http://fixture.test{target}")).unwrap(); let query: HashMap<_, _> = url.query_pairs().into_owned().collect(); let get = |name: &str| query.get(name).map(String::as_str).unwrap_or(""); + if url.path() == "/compressed-worker.js" { + let body = r#"self.onmessage = async event => { + const probe = (0, eval)('(' + event.data.source + ')'); + postMessage(await probe(event.data.origins, event.data.label)); + close(); + };"#; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/javascript\r\nCache-Control: no-store\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(body.as_bytes()).await; + return; + } + if url.path() == "/compressed" { + const GZIP: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 75, 76, 74, 6, 0, 194, 65, 36, 53, 3, 0, 0, 0, + ]; + const EMPTY: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const OPAQUE_PNG: &[u8] = &[ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, + 8, 4, 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 252, 255, 31, + 0, 3, 3, 2, 0, 239, 163, 47, 89, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ]; + const OPAQUE_GZIP: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 235, 12, 240, 115, 231, 229, 146, 226, 98, 96, 96, + 224, 245, 244, 112, 9, 2, 210, 140, 32, 204, 193, 2, 36, 183, 202, 240, 48, 1, 41, 110, + 79, 23, 199, 144, 138, 91, 201, 127, 254, 203, 51, 48, 51, 51, 49, 188, 95, 172, 31, 9, + 20, 102, 240, 116, 245, 115, 89, 231, 148, 208, 4, 0, 122, 254, 219, 225, 68, 0, 0, 0, + ]; + let body = match (get("opaque"), get("kind")) { + ("1", "gzip") => OPAQUE_GZIP, + ("1", "identity") => OPAQUE_PNG, + (_, "gzip") => GZIP, + (_, "empty") => EMPTY, + _ => b"abc", + }; + let validate = + get("cache") == "1" && request.to_ascii_lowercase().contains("if-none-match:"); + let status = if validate { 304 } else { 200 }; + let chunked = !validate && get("chunked") == "1"; + let length = if validate { 0 } else { body.len() }; + let framing = if chunked { + "Transfer-Encoding: chunked".to_owned() + } else { + format!("Content-Length: {length}") + }; + let mime = if get("opaque") == "1" { + "image/png" + } else { + "text/plain" + }; + let mut headers = format!( + "HTTP/1.1 {status} Fixture\r\nContent-Type: {mime}\r\n{framing}\r\nConnection: close\r\n" + ); + if get("kind") != "identity" && !validate { + headers.push_str("Content-Encoding: gzip\r\n"); + } + headers.push_str(if get("cache") == "1" { + "Cache-Control: max-age=600\r\nETag: \"v1\"\r\n" + } else { + "Cache-Control: no-store\r\n" + }); + for (query, header) in [ + ("cors", "Access-Control-Allow-Origin"), + ("tao", "Timing-Allow-Origin"), + ] { + if !get(query).is_empty() { + headers.push_str(&format!("{header}: {}\r\n", get(query))); + } + } + headers.push_str("\r\n"); + if socket.write_all(headers.as_bytes()).await.is_err() + || request.starts_with("HEAD ") + || validate + { + return; + } + if chunked { + if socket + .write_all(format!("{:x};test=value\r\n", body.len()).as_bytes()) + .await + .is_err() + { + return; + } + let _ = socket.write_all(body).await; + let _ = socket + .write_all(b"\r\n0\r\nX-Trailer: present\r\n\r\n") + .await; + } else { + let _ = socket.write_all(body).await; + } + return; + } let key = get("key").to_owned(); let gate = { let mut state = state.lock(); @@ -314,3 +411,12 @@ async fn run_resource_timing_fixture(fixture: &str, expected_total: usize) { assert_eq!(result["total"], expected_total); assert_eq!(result["failures"], serde_json::json!([])); } + +#[tokio::test(flavor = "current_thread")] +async fn compressed_resource_timing_sizes_survive_window_child_worker_and_memory_cache() { + run_resource_timing_fixture( + include_str!("../../../tests/fixtures/compressed-request-resource-timing.js"), + 300, + ) + .await; +} diff --git a/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js b/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js new file mode 100644 index 00000000000..6f2fc5c6450 --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js @@ -0,0 +1,103 @@ +(async () => { + const probe = async (origins, label) => { + const rows = []; + globalThis.__compressedTimingProgress = rows; + const checks = async (name, run) => { + try { await run(); rows.push({name: label + ': ' + name, passed: true}); } + catch (error) { rows.push({name: label + ': ' + name, passed: false, error: String(error)}); } + }; + const assert = (condition, message) => { if (!condition) throw Error(message); }; + const entryFor = url => new Promise((resolve, reject) => { + const existing = performance.getEntriesByName(url, 'resource'); + if (existing.length) { resolve(existing[0]); return; } + const observer = new PerformanceObserver(list => { + const entry = list.getEntries().find(value => value.name === url); + if (entry) { clearTimeout(timer); observer.disconnect(); resolve(entry); } + }); + observer.observe({type: 'resource', buffered: true}); + const timer = setTimeout(() => { observer.disconnect(); reject(Error('Missing entry: ' + url)); }, 2000); + }); + const request = async (api, url, method, opaque, validate) => { + if (api === 'fetch') return (await fetch(url, {method, mode: opaque ? 'no-cors' : 'cors', cache: validate ? 'no-cache' : 'default'})).text(); + const xhr = new XMLHttpRequest(); + xhr.open(method, url, api !== 'sync-xhr'); + if (validate) xhr.setRequestHeader('Cache-Control', 'max-age=0'); + if (api === 'sync-xhr') { xhr.send(); return xhr.responseText; } + return new Promise((resolve, reject) => { + xhr.onload = () => resolve(xhr.responseText); + xhr.onerror = () => reject(Error('XHR network error')); + xhr.send(); + }); + }; + const apis = typeof document === 'undefined' ? ['fetch', 'xhr', 'sync-xhr'] : ['fetch', 'xhr']; + const specs = [ + {kind: 'gzip', encoded: 23, decoded: 3}, + {kind: 'gzip', chunked: true, encoded: 23, decoded: 3}, + {kind: 'identity', encoded: 3, decoded: 3}, + {kind: 'empty', encoded: 20, decoded: 0}, + {kind: 'empty', chunked: true, encoded: 20, decoded: 0}, + {kind: 'bad', encoded: 3, decoded: 0, fails: true}, + {kind: 'bad', chunked: true, encoded: 3, decoded: 0, fails: true}, + {kind: 'gzip', method: 'HEAD', encoded: 0, decoded: 0} + ]; + let next = 0; + for (const api of apis) for (const mode of ['same', 'cors', 'cors-tao', 'denied', ...(api === 'fetch' ? ['opaque', 'opaque-tao'] : [])]) for (const spec of specs) { + await checks(`${api}/${mode}/${spec.kind}/${spec.chunked ? 'chunked' : 'length'}/${spec.method || 'GET'}`, async () => { + const url = new URL('/compressed', origins[mode === 'same' ? 0 : 1]); + url.searchParams.set('key', `${label}-${++next}`); + url.searchParams.set('kind', spec.kind); + if (spec.chunked) url.searchParams.set('chunked', '1'); + if (mode.startsWith('opaque')) url.searchParams.set('opaque', '1'); + if (mode.startsWith('cors')) url.searchParams.set('cors', '*'); + if (mode.endsWith('tao')) url.searchParams.set('tao', '*'); + const name = url.href; + const opaque = mode.startsWith('opaque'); + let failed = false; + try { await request(api, name, spec.method || 'GET', opaque, false); } catch { failed = true; } + const entry = await entryFor(name); + const hidden = opaque || mode === 'denied'; + assert(failed === (mode === 'denied' || !!spec.fails && !opaque), 'Unexpected body result'); + assert(entry.encodedBodySize === (hidden ? 0 : spec.encoded), `encoded=${entry.encodedBodySize}`); + assert(entry.decodedBodySize === (hidden ? 0 : spec.decoded), `decoded=${entry.decodedBodySize}`); + assert(entry.responseStatus === (hidden ? 0 : 200), `status=${entry.responseStatus}`); + const allowed = mode === 'same' || mode.endsWith('tao'); + assert(entry.transferSize === (allowed ? (hidden ? 0 : spec.encoded) + 300 : 0), `transfer=${entry.transferSize}`); + assert(performance.getEntriesByName(name, 'resource').length === 1, 'Duplicate timing'); + }); + } + for (const api of apis) { + const base = `${origins[0]}/compressed?kind=gzip&cache=1&key=${label}-${api}-cache`; + for (let phase = 0; phase < 4; ++phase) await checks(`${api}/cache/${phase}`, async () => { + const url = base + '#phase-' + phase; + assert(await request(api, url, 'GET', false, phase === 2) === 'abc', 'Wrong cached representation'); + const entry = await entryFor(url); + assert(entry.encodedBodySize === 23 && entry.decodedBodySize === 3, `cache sizes=${entry.encodedBodySize}/${entry.decodedBodySize}`); + assert(entry.transferSize === [323, 0, 300, 0][phase], `cache transfer=${entry.transferSize}`); + assert(performance.getEntriesByName(url, 'resource').length === 1, 'Duplicate cached timing'); + }); + } + return {total: rows.length, failures: rows.filter(row => !row.passed)}; + }; + const results = [await probe(__fetchOrigins, 'window')]; + const iframe = document.createElement('iframe'); + const child = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(Error('Child timing probe did not finish')), 30000); + const receive = event => { + if (event.source !== iframe.contentWindow || event.data?.kind !== 'compressed-timing') return; + clearTimeout(timer); removeEventListener('message', receive); resolve(event.data.result); + }; + addEventListener('message', receive); + }); + iframe.srcdoc = '