diff --git a/Cargo.lock b/Cargo.lock index 4ad1bb1b4a0..ad30863fb8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2953,9 +2953,9 @@ name = "moli-fetch" version = "0.1.0" dependencies = [ "anyhow", - "bytes", "base64 0.22.1", "brotlic-sys", + "bytes", "cidr", "crossbeam-channel", "curl", diff --git a/Cargo.toml b/Cargo.toml index 337af2f065d..3edd83cea93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -154,3 +154,6 @@ codegen-units = 1 [profile.test] debug = 1 + +[patch."https://github.com/lexmount/curl-rust"] +curl-sys = { path = "vendor/curl-sys-0.4.91" } diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs index 599e322786c..833eccdf67a 100644 --- a/moli-fetch/src/blocking/mod.rs +++ b/moli-fetch/src/blocking/mod.rs @@ -677,7 +677,7 @@ pub(crate) fn configure_easy( .context("failed to disable curl redirect following")?; easy.accept_encoding("") .context("failed to enable curl response decompression")?; - easy.http_content_decoding(request.compression_dictionary.is_none()) + easy.http_content_decoding(!crate::content_decoding::ContentDecoding::required(request)) .context("failed to configure response content decoding")?; easy.get_mut() .begin_dictionary_response(request, request_url); diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rs index 7a6bded926e..a87cbc2a274 100644 --- a/moli-fetch/src/cancellation.rs +++ b/moli-fetch/src/cancellation.rs @@ -28,7 +28,10 @@ struct ResponseProgressState { /// Native counts of encoded receipt and decoded representation, excluding framing. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct FetchResponseBodySizes { pub encoded: u64, pub decoded: u64 } +pub struct FetchResponseBodySizes { + pub encoded: u64, + pub decoded: u64, +} impl ResponseProgress { pub(crate) fn mark_request_started(&self) { @@ -79,7 +82,17 @@ impl ResponseProgress { pub(crate) fn record_encoded_response_body_size(&self, bytes: u64) { let mut timing = self.inner.timing.lock(); - timing.body_sizes.get_or_insert_with(Default::default).encoded = bytes; + timing + .body_sizes + .get_or_insert_with(Default::default) + .encoded = bytes; + timing.body_bytes_received |= bytes > 0; + } + + pub(crate) fn record_encoded_response_body_chunk(&self, bytes: usize) { + let mut timing = self.inner.timing.lock(); + let sizes = timing.body_sizes.get_or_insert_with(Default::default); + sizes.encoded = sizes.encoded.saturating_add(bytes as u64); timing.body_bytes_received |= bytes > 0; } @@ -97,7 +110,10 @@ impl ResponseProgress { .get_or_insert_with(Instant::now); } - pub(crate) fn mark_cached_response_available(&self, sizes: FetchResponseBodySizes) -> CachedResponseBodyTiming { + pub(crate) fn mark_cached_response_available( + &self, + sizes: FetchResponseBodySizes, + ) -> CachedResponseBodyTiming { self.mark_response_started(); self.inner.timing.lock().body_sizes = Some(sizes); self.commit_state(); @@ -130,7 +146,9 @@ pub struct FetchResponseTiming { } impl FetchResponseTiming { - pub fn body_sizes(self) -> Option { self.body_sizes } + pub fn body_sizes(self) -> Option { + self.body_sizes + } pub fn body_bytes_received(self) -> bool { self.body_bytes_received } @@ -293,8 +311,12 @@ impl FetchCancelHandle { } #[cfg(test)] - fn mark_cached_response_available(&self, sizes: FetchResponseBodySizes) -> CachedResponseBodyTiming { - self.response_progress().mark_cached_response_available(sizes) + fn mark_cached_response_available( + &self, + sizes: FetchResponseBodySizes, + ) -> CachedResponseBodyTiming { + self.response_progress() + .mark_cached_response_available(sizes) } #[cfg(test)] @@ -309,7 +331,7 @@ impl FetchCancelHandle { #[cfg(test)] mod tests { - use super::FetchCancelHandle; + use super::{FetchCancelHandle, FetchResponseBodySizes}; use std::time::Instant; #[test] @@ -674,7 +696,10 @@ mod tests { for reset_before_body_end in [false, true] { let parent = FetchCancelHandle::new(); let response = parent.child_for_subrequest(); - let recorder = response.mark_cached_response_available(FetchResponseBodySizes { encoded: 23, decoded: 3 }); + let recorder = response.mark_cached_response_available(FetchResponseBodySizes { + encoded: 23, + decoded: 3, + }); parent.adopt_response_progress(&response); assert!(parent.response_completion_is_committed()); assert!(parent.response_timing().response_started_at().is_some()); @@ -746,7 +771,13 @@ mod body_size_generation_tests { old.mark_response_started(); old.record_encoded_response_body_size(23); old.record_decoded_response_body_size(3); - assert_eq!(handle.response_timing().body_sizes(), Some(FetchResponseBodySizes {encoded:23,decoded:3})); + assert_eq!( + handle.response_timing().body_sizes(), + Some(FetchResponseBodySizes { + encoded: 23, + decoded: 3 + }) + ); let current = handle.begin_response_progress(); current.mark_response_started(); current.record_encoded_response_body_size(7); @@ -754,7 +785,13 @@ mod body_size_generation_tests { old.record_encoded_response_body_size(99); old.record_decoded_response_body_size(88); old.commit(); - assert_eq!(handle.response_timing().body_sizes(), Some(FetchResponseBodySizes {encoded:7,decoded:5})); + assert_eq!( + handle.response_timing().body_sizes(), + Some(FetchResponseBodySizes { + encoded: 7, + decoded: 5 + }) + ); assert!(!handle.response_completion_is_committed()); current.commit(); assert!(handle.response_completion_is_committed()); diff --git a/moli-fetch/src/compression_dictionary/mod.rs b/moli-fetch/src/compression_dictionary/mod.rs index 40944cce5f7..7ee4b0ceaef 100644 --- a/moli-fetch/src/compression_dictionary/mod.rs +++ b/moli-fetch/src/compression_dictionary/mod.rs @@ -2,7 +2,6 @@ //! requests which may use them. Bytes enter the store only after a readable //! response completes; a header alone cannot register a dictionary. -mod decoding; mod fields; use std::{borrow::Cow, collections::VecDeque, fmt, io::Read, sync::Arc}; @@ -17,7 +16,6 @@ use url::Url; use crate::{FetchConfig, Request, RequestMode}; -pub(crate) use decoding::DictionaryDecoding; pub(crate) use fields::RegistrationOptions; const MAX_DICTIONARY_BYTES: usize = 64 * 1024 * 1024; @@ -392,18 +390,6 @@ impl DictionaryResponse { } } - pub(crate) fn headers(&self) -> &[(String, Vec)] { - &self.headers - } - - pub(crate) fn headers_finalized(&self) -> bool { - self.headers_finalized - } - - pub(crate) fn bodyless(&self) -> bool { - self.request.head || matches!(self.status, 204 | 205 | 304) - } - pub(crate) fn from_cached( store: SharedDictionaryStore, request: &Request, diff --git a/moli-fetch/src/content_coding.rs b/moli-fetch/src/content_coding.rs index 6b778848289..adf78bcdff8 100644 --- a/moli-fetch/src/content_coding.rs +++ b/moli-fetch/src/content_coding.rs @@ -1,17 +1,9 @@ use moli_header_field::{HeaderFieldTokenMode, HeaderFieldTokenizer}; fn supported_content_codings() -> impl Iterator { - let version = curl::Version::get(); - [ - ("deflate", version.feature_libz()), - ("gzip", version.feature_libz()), - ("br", version.feature_brotli()), - ("zstd", version.feature_zstd()), - ("dcb", version.feature_brotli()), - ("dcz", version.feature_zstd()), - ] - .into_iter() - .filter_map(|(coding, supported)| supported.then_some(coding)) + // The bounded native decoder supplies the same coding set even when the + // linked libcurl lacks Zstandard or external-dictionary support. + ["deflate", "gzip", "br", "zstd", "dcb", "dcz"].into_iter() } /// Choose the browser's header before cache lookup and transport setup so diff --git a/moli-fetch/src/compression_dictionary/decoding.rs b/moli-fetch/src/content_decoding.rs similarity index 84% rename from moli-fetch/src/compression_dictionary/decoding.rs rename to moli-fetch/src/content_decoding.rs index 31ef71ba8d1..f3cfc4ba122 100644 --- a/moli-fetch/src/compression_dictionary/decoding.rs +++ b/moli-fetch/src/content_decoding.rs @@ -1,7 +1,6 @@ -//! The native HTTP backend does not expose an external-dictionary option. -//! Requests carrying a dictionary therefore decode their content-coding stack -//! here, using the same native Brotli/zlib/Zstandard libraries. Other requests -//! retain libcurl's content decoders and transfer accounting. +//! Bounded content decoding shared by ordinary and dictionary responses. +//! Native zlib, Brotli and Zstandard decoders are used when libcurl cannot +//! decode the complete negotiated coding set or an external dictionary is needed. use std::{ ffi::c_void, @@ -12,34 +11,72 @@ use std::{ use anyhow::{Result, anyhow, bail, ensure}; use flate2::{Decompress, FlushDecompress, Status}; -use super::Dictionary; +use crate::{Request, compression_dictionary::Dictionary}; const OUTPUT_SIZE: usize = 16 * 1024; type Output<'a> = dyn FnMut(&[u8]) -> Result<()> + 'a; -pub(crate) struct DictionaryDecoding { - dictionary: Arc, +pub(crate) struct ContentDecoding { + dictionary: Option>, stages: Option>, + head: bool, + status: u16, + headers: Vec<(String, Vec)>, + headers_finalized: bool, } -impl DictionaryDecoding { - pub(crate) fn new(dictionary: Arc) -> Self { +impl ContentDecoding { + pub(crate) fn required(request: &Request) -> bool { + request.compression_dictionary.is_some() || !curl::Version::get().feature_zstd() + } + + pub(crate) fn new(request: &Request) -> Self { Self { - dictionary, + dictionary: request.compression_dictionary.clone(), stages: None, + head: request.method == "HEAD", + status: 0, + headers: Vec::new(), + headers_finalized: false, } } - pub(crate) fn reset(&mut self) { - self.stages = None; + pub(crate) fn header(&mut self, data: &[u8]) -> Result<()> { + if let Some(line) = crate::response::parse_http_response_status_line(data) { + self.status = line.status; + self.headers.clear(); + self.stages = None; + self.headers_finalized = false; + return Ok(()); + } + if self.headers_finalized { + return Ok(()); + } + if let Some(header) = crate::headers::parse_http_response_header_line(data) { + self.headers.push(header); + return Ok(()); + } + if self.status >= 200 && data.iter().all(|byte| matches!(byte, b'\r' | b'\n')) { + self.headers_finalized = true; + } + Ok(()) } - pub(crate) fn initialize(&mut self, headers: &[(String, Vec)]) -> Result<()> { + pub(crate) fn needs_readable_dictionary_response(&self) -> bool { + self.dictionary.is_some() && self.headers_finalized && !self.bodyless() + } + + fn bodyless(&self) -> bool { + self.head || matches!(self.status, 204 | 205 | 304) + } + + fn initialize(&mut self) -> Result<()> { if self.stages.is_some() { return Ok(()); } let mut names = Vec::new(); - for (_, value) in headers + for (_, value) in self + .headers .iter() .filter(|(name, _)| name.eq_ignore_ascii_case("content-encoding")) { @@ -61,7 +98,9 @@ impl DictionaryDecoding { "zstd" => Stage::Zstd(Zstd::new(None)?), "dcb" | "dcz" => Stage::Dictionary { brotli: name == "dcb", - dictionary: Arc::clone(&self.dictionary), + dictionary: self.dictionary.clone().ok_or_else(|| { + anyhow!("dictionary content encoding without a selected dictionary") + })?, prefix: Vec::new(), decoder: None, }, @@ -75,9 +114,10 @@ impl DictionaryDecoding { pub(crate) fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> { ensure!( - self.stages.is_some(), + self.headers_finalized, "compressed response body arrived before headers" ); + self.initialize()?; feed( self.stages.as_mut().expect("initialized decoding stack"), input, @@ -85,11 +125,12 @@ impl DictionaryDecoding { ) } - pub(crate) fn finish(&mut self, headers: &[(String, Vec)], bodyless: bool) -> Result<()> { - if bodyless { + pub(crate) fn finish(&mut self) -> Result<()> { + // An empty encoded body never invokes a decoder, including for an + // unknown coding. A decoder that received bytes must finish cleanly. + if self.bodyless() || self.stages.is_none() { return Ok(()); } - self.initialize(headers)?; for stage in self.stages.as_ref().expect("initialized decoding stack") { ensure!(stage.finished(), "truncated compressed response"); } diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs index 3bb368a272f..88602274959 100644 --- a/moli-fetch/src/lib.rs +++ b/moli-fetch/src/lib.rs @@ -12,6 +12,7 @@ mod client_hints; mod compression_dictionary; mod config; mod content_coding; +mod content_decoding; mod cors; mod dns; mod error; @@ -82,9 +83,10 @@ pub use referrer_policy::{ pub use request::{ BrowserNavigationRequestKind, BrowserRequestMetadata, CookieHeaderPolicy, FetchPriorityHint, Request, RequestAuth, RequestAuthScheme, RequestAuthTarget, RequestCacheMode, - RequestCredentialsMode, RequestHeaderOverride, RequestMode, RequestPriorityHints, RequestRedirectCheck, - RequestRedirectHandling, RequestRedirectMode, RequestResourceType, ResourceLoadPriority, - ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, SubresourceRequestMetadata, + RequestCredentialsMode, RequestHeaderOverride, RequestMode, RequestPriorityHints, + RequestRedirectCheck, RequestRedirectHandling, RequestRedirectMode, RequestResourceType, + ResourceLoadPriority, ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, + SubresourceRequestMetadata, }; pub use request_policy::{is_bad_port, should_request_be_blocked_due_to_bad_port}; pub use response::{ diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs index 380e489bfd9..72654b4b5d9 100644 --- a/moli-fetch/src/runtime.rs +++ b/moli-fetch/src/runtime.rs @@ -910,7 +910,7 @@ impl RuntimeOwner { if request_cache_policy_for_url(&self.config, &job.request, &job.current_url).only_if_cached { let head = cache_only_miss_start(&job.request, &job.current_url).into_head(); - job.cancel_handle.mark_response_terminal(); + job.response_progress.commit(); return Ok(JobOutcome::Complete( job.response_tx, Box::new(CompletedBufferedResponse::Raw( @@ -1474,15 +1474,11 @@ impl RuntimeOwner { mut completion: RuntimeCurlCompletion, ) { if let Some(easy) = completion.easy.as_mut() { - if let Some(error) = easy.get_mut().dictionary_error.take() { + if let Some(error) = easy.get_mut().content_decoding_error.take() { completion.result = Err(anyhow!(error)); } else if completion.result.is_ok() && !self.shutdown_requested.load(Ordering::SeqCst) - && !easy - .get_ref() - .response_timing - .as_ref() - .is_some_and(FetchCancelHandle::is_cancelled) + && !completion.context.is_cancelled() { completion.result = easy.get_mut().finish_dictionary_response(); } @@ -2771,6 +2767,14 @@ enum ActiveTransferContext { } impl ActiveTransferContext { + fn is_cancelled(&self) -> bool { + match self { + Self::Buffered(context) => context.job.cancel_handle.is_cancelled(), + Self::Streaming(context) => context.job.cancel_handle.is_cancelled(), + Self::StreamingRaw(context) => context.job.cancel_handle.is_cancelled(), + } + } + fn into_buffered(self) -> Option { match self { Self::Buffered(context) => Some(*context), @@ -2825,8 +2829,8 @@ pub(crate) struct FetchTransferHandler { upload_observer: Option, proxy_connect_response_recorder: ProxyConnectResponseRecorder, dictionary_response: Option, - dictionary_decoding: Option, - dictionary_error: Option, + content_decoding: Option, + content_decoding_error: Option, } enum FetchResponseCollector { @@ -2873,8 +2877,8 @@ impl FetchTransferHandler { upload_observer: None, proxy_connect_response_recorder: ProxyConnectResponseRecorder::default(), dictionary_response: None, - dictionary_decoding: None, - dictionary_error: None, + content_decoding: None, + content_decoding_error: None, } } @@ -2941,20 +2945,18 @@ impl FetchTransferHandler { url.clone(), ) }); - self.dictionary_decoding = request.compression_dictionary.as_ref().map(|dictionary| { - crate::compression_dictionary::DictionaryDecoding::new(Arc::clone(dictionary)) - }); - self.dictionary_error = None; + self.content_decoding = crate::content_decoding::ContentDecoding::required(request) + .then(|| crate::content_decoding::ContentDecoding::new(request)); + self.content_decoding_error = None; } fn finish_dictionary_response(&mut self) -> Result<()> { - let Some(response) = self.dictionary_response.as_mut() else { - return Ok(()); - }; - if let Some(decoding) = self.dictionary_decoding.as_mut() { - decoding.finish(response.headers(), response.bodyless())?; + if let Some(decoding) = self.content_decoding.as_mut() { + decoding.finish()?; + } + if let Some(response) = self.dictionary_response.as_mut() { + response.finish(); } - response.finish(); Ok(()) } @@ -2989,8 +2991,10 @@ impl Handler for FetchTransferHandler { } fn write(&mut self, data: &[u8]) -> std::result::Result { - if !data.is_empty() { self.mark_response_body_received(); } - if self.dictionary_decoding.is_none() { + if !data.is_empty() { + self.mark_response_body_received(); + } + if self.content_decoding.is_none() { if let Some(timing) = self.response_progress() { timing.record_decoded_response_body_size(data.len()); } @@ -3008,12 +3012,17 @@ impl Handler for FetchTransferHandler { } return result; } - if self.dictionary_error.is_some() { + let timing = self.response_progress().cloned(); + // A decoding failure can abort write before curl reports progress. + // Count the wire bytes at this boundary when we own content decoding. + if let Some(timing) = &timing { + timing.record_encoded_response_body_chunk(data.len()); + } + if self.content_decoding_error.is_some() { return Ok(0); } let policy = &mut self.dictionary_response; let response = &mut self.response; - let timing = self.response_progress().cloned(); let mut output = |bytes: &[u8]| -> Result<()> { if let Some(timing) = &timing { timing.record_decoded_response_body_size(bytes.len()); @@ -3034,14 +3043,14 @@ impl Handler for FetchTransferHandler { Ok(()) }; let result = self - .dictionary_decoding + .content_decoding .as_mut() - .expect("dictionary decoding selected") + .expect("native content decoding selected") .push(data, &mut output); match result { Ok(()) => Ok(data.len()), Err(error) => { - self.dictionary_error = Some(error.to_string()); + self.content_decoding_error = Some(error.to_string()); Ok(0) } } @@ -3050,25 +3059,23 @@ impl Handler for FetchTransferHandler { fn header(&mut self, data: &[u8]) -> bool { if crate::response::parse_http_response_status_line(data) .is_some_and(|line| line.status >= 200) - && let Some(decoding) = self.dictionary_decoding.as_mut() { - decoding.reset(); - self.dictionary_error = None; + self.content_decoding_error = None; } if let Some(response) = self.dictionary_response.as_mut() { response.header(data); } - if data.iter().all(|byte| matches!(byte, b'\r' | b'\n')) - && let Some(response) = self.dictionary_response.as_ref() - && response.headers_finalized() - && !response.bodyless() - && let Some(decoding) = self.dictionary_decoding.as_mut() - { - if !response.readable() { - self.dictionary_error = + if let Some(decoding) = self.content_decoding.as_mut() { + if let Err(error) = decoding.header(data) { + self.content_decoding_error = Some(error.to_string()); + } else if decoding.needs_readable_dictionary_response() + && self + .dictionary_response + .as_ref() + .is_none_or(|response| !response.readable()) + { + self.content_decoding_error = Some("dictionary response failed its CORS readability check".to_owned()); - } else if let Err(error) = decoding.initialize(response.headers()) { - self.dictionary_error = Some(error.to_string()); } } if crate::response::parse_http_response_status_line(data) @@ -3091,7 +3098,10 @@ impl Handler for FetchTransferHandler { if dlnow > 0.0 { self.mark_response_body_received(); } - if dlnow > 0.0 && let Some(timing) = self.response_progress() { + if self.content_decoding.is_none() + && dlnow > 0.0 + && let Some(timing) = self.response_progress() + { timing.record_encoded_response_body_size(dlnow as u64); } let keep_going = match &mut self.response { @@ -3277,7 +3287,9 @@ fn complete_cached_streaming_html_job( capture, job.cancel_handle.clone(), ); - let response_timing = job.response_progress.mark_cached_response_available(body_sizes); + let response_timing = job + .response_progress + .mark_cached_response_available(body_sizes); if let Some(started_tx) = job.started_tx { let _ = started_tx.send(Ok(StreamingHtmlResponseStart { @@ -3425,7 +3437,9 @@ fn complete_cached_streaming_raw_job( capture, job.cancel_handle.clone(), ); - let response_timing = job.response_progress.mark_cached_response_available(body_sizes); + let response_timing = job + .response_progress + .mark_cached_response_available(body_sizes); if let Some(started_tx) = job.started_tx { let _ = started_tx.send(Ok(StreamingHtmlResponseStart { @@ -3906,7 +3920,7 @@ fn complete_cache_only_html_miss(mut job: StreamingRuntimeJob) { (job.started_tx.take(), job.body_tx.take()) }; drop(body_tx); - job.cancel_handle.mark_response_terminal(); + job.response_progress.commit(); if let Some(started_tx) = started_tx { let _ = started_tx.send(Ok(start)); } @@ -3924,7 +3938,7 @@ fn complete_cache_only_raw_miss(mut job: StreamingRawRuntimeJob) { (job.started_tx.take(), job.body_tx.take()) }; drop(body_tx); - job.cancel_handle.mark_response_terminal(); + job.response_progress.commit(); if let Some(started_tx) = started_tx { let _ = started_tx.send(Ok(start)); } diff --git a/moli-fetch/src/runtime/response_timing_tests.rs b/moli-fetch/src/runtime/response_timing_tests.rs index 98f6f3fe047..8621d15c14b 100644 --- a/moli-fetch/src/runtime/response_timing_tests.rs +++ b/moli-fetch/src/runtime/response_timing_tests.rs @@ -392,7 +392,7 @@ fn transfer_progress_preserves_cancellation_and_body_observations_on_reuse() { for _ in 0..2 { cancel.begin_response_progress(); handler.buffered_mut().unwrap().begin_request(None); - handler.begin_transfer(None, false); + handler.begin_transfer(None, false, None); assert!(handler.progress(4.0, 0.0, 0.0, 0.0)); assert!(!cancel.response_timing().body_bytes_received()); assert!(handler.progress(4.0, 1.0, 0.0, 0.0)); diff --git a/moli-fetch/src/tests/cache_request_directives.rs b/moli-fetch/src/tests/cache_request_directives.rs index 2690ba4a1d2..4a75c2bdce9 100644 --- a/moli-fetch/src/tests/cache_request_directives.rs +++ b/moli-fetch/src/tests/cache_request_directives.rs @@ -14,8 +14,12 @@ async fn cache_only_requests_preserve_cancellation_before_local_completion() -> for follow_redirects in [true, false] { let cancel = FetchCancelHandle::new(); cancel.cancel(); - let request = - controlled(&server.url(), "only-if-cached")?.with_follow_redirects(follow_redirects); + let request = controlled(&server.url(), "only-if-cached")?; + let request = if follow_redirects { + request + } else { + request.with_single_hop() + }; let error = client .fetch_with_cancel(request, cancel) .await @@ -51,7 +55,11 @@ async fn cache_only_misses_never_contact_origin_on_raw_text_html_or_buffered_pat response.head() } 1 | 2 => { - let request = request.with_follow_redirects(path == 1); + let request = if path == 1 { + request + } else { + request.with_single_hop() + }; let observed = client .fetch_with_cancel_and_network_metadata(request, cancel.clone()) .await?; @@ -240,7 +248,7 @@ async fn disk_cache_only_redirects_stop_at_uncached_hops_without_network() -> Re let mut config = FetchConfig::default(); config.set_http_cache_dir(Some(cache_dir.display().to_string())); let client = FetchClient::new(&config, new_shared_browser_cookie_store()); - let prime = Request::get(&server.url())?.with_follow_redirects(false); + let prime = Request::get(&server.url())?.with_single_hop(); assert_eq!(client.fetch_raw(prime).await?.status, 302); let request = controlled(&server.url(), "only-if-cached")?; let head = if html { diff --git a/moli-fetch/src/tests/upload.rs b/moli-fetch/src/tests/upload.rs index 8694c32f12f..19024f06602 100644 --- a/moli-fetch/src/tests/upload.rs +++ b/moli-fetch/src/tests/upload.rs @@ -3,15 +3,15 @@ use crate::{UploadEvent, UploadObserver}; async fn upload_response( client: &FetchClient, - mut request: Request, + request: Request, transport: &str, cancel: FetchCancelHandle, ) -> Result { match transport { - "buffered" => { - request.follow_redirects = false; - Ok(client.fetch_with_cancel(request, cancel).await?.status) - } + "buffered" => Ok(client + .fetch_with_cancel(request.with_single_hop(), cancel) + .await? + .status), "html" => { let mut response = client.fetch_html_stream(request).await?; while response.next_chunk().await.is_some() {} @@ -59,8 +59,9 @@ async fn upload_observer_finishes_before_response_and_distinguishes_empty_from_a let observer = UploadObserver::new(total, move |event| { let _ = event_tx.send(event); }); - let request = Request::new_bytes("POST", &url, body.clone(), Vec::new())? - .with_upload_observer(observer); + let request = + Request::new_bytes("POST", &url, body.clone().map(Into::into), Vec::new())? + .with_upload_observer(observer); let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); let fetch = upload_response(&client, request, transport, FetchCancelHandle::new()); @@ -120,7 +121,7 @@ async fn upload_observer_does_not_infer_completion_from_an_early_response() -> R let request = Request::new_bytes( "POST", &url, - Some(body), + Some(body.into()), vec![("Expect".into(), "100-continue".into())], )? .with_upload_observer(observer); @@ -173,7 +174,7 @@ async fn upload_observer_reports_partial_bytes_and_stops_after_cancellation() -> let request = Request::new_bytes( "POST", &url, - Some(vec![b'x'; total as usize]), + Some(vec![b'x'; total as usize].into()), vec![("Expect".into(), String::new())], )? .with_upload_observer(observer); diff --git a/moli-renderer-v8/src/network/backend/memory_cache.rs b/moli-renderer-v8/src/network/backend/memory_cache.rs index c331a20eb4c..89d48824c06 100644 --- a/moli-renderer-v8/src/network/backend/memory_cache.rs +++ b/moli-renderer-v8/src/network/backend/memory_cache.rs @@ -911,7 +911,7 @@ fn raw_subresource_request_is_memory_cacheable( // A CORS-managed hop carries the logical request's earlier URLs. It // can cache this hop without treating the prefix as a followed // transport redirect; its origin/credential taint is part of the key. - && (request.redirect_chain().is_empty() || !request.follow_redirects) + && (request.redirect_chain().is_empty() || !request.transport_follows_redirects()) && raw_subresource_memory_cacheable_resource_type(request) && request.method.eq_ignore_ascii_case("GET") && request.body.is_none() @@ -1179,7 +1179,7 @@ mod tests { .with_request_origin((&initiator).into()) .with_initiator_url(&initiator); let per_hop = automatic.clone().with_single_hop(); - let key = raw_subresource_memory_cache_key(&per_hop, per_hop.headers()) + let key = raw_subresource_memory_cache_key(&per_hop, &per_hop.request_headers) .expect("cacheable first CORS hop"); let mut cache = SharedMemoryResourceCache::with_limits(usize::MAX, usize::MAX); cache.insert_raw_subresource( diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rs index 394b5e69d93..87dbd1eb2fc 100644 --- a/moli-renderer-v8/src/network/request_client.rs +++ b/moli-renderer-v8/src/network/request_client.rs @@ -1048,8 +1048,8 @@ impl ResourceRequestClient { ) -> Result { // These buffered transport modes have no distinct header boundary. if request.auth_requires_buffered_transport() - || (!request.follow_redirects - && request.redirect_mode != moli_fetch::RequestRedirectMode::Error) + || (!request.transport_follows_redirects() + && request.redirect_mode() != moli_fetch::RequestRedirectMode::Error) { return self .fetch_text_stream_with_cancel_after_policy(request, cancel_handle) @@ -1252,7 +1252,7 @@ impl ResourceRequestClient { )); } if response.status == 304 - && (!response.redirected || !request.follow_redirects) + && (!response.redirected || !request.transport_follows_redirects()) && response.final_url == request.url && let Some(cached) = cached { @@ -1340,7 +1340,7 @@ impl ResourceRequestClient { let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel(); let (completion_tx, completion_rx) = tokio::sync::oneshot::channel(); let resource_runtime = self.resource_runtime.clone(); - let outward_cancel = FetchCancelHandle::new(); + let outward_cancel = response.cancellation_handle().child_for_subrequest(); response.share_response_progress(&outward_cancel); tokio::spawn(async move { let mut body = Vec::new(); @@ -1378,7 +1378,7 @@ impl ResourceRequestClient { if completion.as_ref().is_some_and(Result::is_ok) { let mut materialized = RawResponse::from_head_and_body(response.head(), body) .with_body_sizes(response.response_timing().body_sizes()); - if !request.follow_redirects { + if !request.transport_follows_redirects() { // The outer CORS loop owns the logical redirect history. // Retain only this hop's representation, then let each // consumer attach its own already-validated URL list. diff --git a/moli-renderer-v8/src/network/request_client/tests/raw_cache_revalidation/directives.rs b/moli-renderer-v8/src/network/request_client/tests/raw_cache_revalidation/directives.rs index 3494571700c..f9236ca0de7 100644 --- a/moli-renderer-v8/src/network/request_client/tests/raw_cache_revalidation/directives.rs +++ b/moli-renderer-v8/src/network/request_client/tests/raw_cache_revalidation/directives.rs @@ -231,7 +231,7 @@ async fn raw_memory_cache_only_redirects_stop_at_uncached_targets() -> Result<() // Prime the complete redirect representation independently. The browser // redirect loop may drop an unread body before its cache tee completes. assert_eq!( - fetch(&client, request().with_follow_redirects(false), false) + fetch(&client, request().with_single_hop(), false) .await? .0 .status, diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rs index 3b6abfa90e2..cfa30acae15 100644 --- a/moli-renderer-v8/src/network_host.rs +++ b/moli-renderer-v8/src/network_host.rs @@ -79,7 +79,7 @@ pub(crate) use self::body_source::{ }; #[cfg(test)] pub(crate) use self::body_source::{ - consume_native_body_text_for_teardown_test, pending_network_body_source_buffered_len_for_test, + consume_native_body_for_teardown_test, pending_network_body_source_buffered_len_for_test, }; pub(crate) use self::browser_response::{ LocalUrlError, local_url_response, local_url_response_result, @@ -211,17 +211,18 @@ pub(crate) use self::xhr::{ PreparedXhrSendBody, XHR_ABORTED_SLOT, XHR_ACTIVE_INTERNAL_ID_SLOT, XHR_ASYNC_SLOT, XHR_METHOD_SLOT, XHR_OPEN_GENERATION_SLOT, XHR_SEND_FLAG_SLOT, XHR_TIMEOUT_SLOT, XHR_TIMEOUT_START_MS_SLOT, XHR_TIMEOUT_TIMER_SLOT, XHR_URL_SLOT, XHR_WITH_CREDENTIALS_SLOT, - apply_xhr_document_abort, apply_xhr_failure, apply_xhr_response, apply_xhr_response_body_source, - apply_xhr_response_body_source_with_status_text, apply_xhr_streaming_failure, apply_xhr_streaming_response_body_source, - apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout, apply_xhr_upload_event, - begin_xhr_send, capture_xhr_upload_listener_flag, convert_xhr_send_body_from_args, dispatch_xhr_loadstart, - finish_xhr_abort, install_progress_event_template_bindings, + apply_xhr_document_abort, apply_xhr_failure, apply_xhr_response, + apply_xhr_response_body_source, apply_xhr_response_body_source_with_status_text, + apply_xhr_streaming_failure, apply_xhr_streaming_response_body_source, + apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout, + apply_xhr_upload_event, begin_xhr_send, capture_xhr_upload_listener_flag, convert_xhr_send_body_from_args, + dispatch_xhr_loadstart, finish_xhr_abort, install_progress_event_template_bindings, install_window_xml_http_request_template_bindings, install_xml_http_request_bindings, install_xml_http_request_event_target_bindings, progress_event_constructor_callback, - set_xhr_state_number, throw_synchronous_xhr_failure, xhr_author_request_headers, - xhr_constructor_callback, xhr_ensure_send_allowed, - xhr_response_text_decoder, - xhr_state_bool_property, xhr_state_number_property, xhr_state_string_property, + set_xhr_state_number, throw_synchronous_xhr_failure, + xhr_author_request_headers, xhr_constructor_callback, xhr_ensure_send_allowed, + xhr_response_text_decoder, xhr_state_bool_property, xhr_state_number_property, + xhr_state_string_property, }; pub(in crate::network_host) const NETWORK_BODY_SLOT: &str = "__lmBody"; pub(in crate::network_host) const NETWORK_BODY_BYTES_SLOT: &str = "__lmBodyBytes"; diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rs index 32be1791290..8e0cc24701a 100644 --- a/moli-renderer-v8/src/network_host/async_fetch.rs +++ b/moli-renderer-v8/src/network_host/async_fetch.rs @@ -714,7 +714,8 @@ pub(crate) fn spawn_async_subresource_fetch( | BrowserRequestMetadata::Xhr, ) ) && request.transport_follows_redirects(); - let can_collect_image_body = parkable_image_manager.is_some() && request.transport_follows_redirects(); + let can_collect_image_body = + parkable_image_manager.is_some() && request.transport_follows_redirects(); if moli_trace::cdp_runtime_trace_enabled() { tracing::info!( target: "moli_cdp_nav_timing", @@ -998,7 +999,6 @@ async fn fetch_response_head_once( .map(|response| response.head()); } - let cancel_handle = cancel_handle.unwrap_or_default(); let response = loader .fetch_raw_stream_with_cancel(request, cancel_handle) .await @@ -1185,9 +1185,11 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, false, + None, ); match mode { RequestRedirectMode::Follow => { diff --git a/moli-renderer-v8/src/network_host/body_source.rs b/moli-renderer-v8/src/network_host/body_source.rs index 8c97b11d577..8d0d3285fc2 100644 --- a/moli-renderer-v8/src/network_host/body_source.rs +++ b/moli-renderer-v8/src/network_host/body_source.rs @@ -1482,12 +1482,18 @@ pub(in crate::network_host) fn consume_network_body_value_from_object<'s>( } #[cfg(test)] -pub(crate) fn consume_native_body_text_for_teardown_test<'s>( +pub(crate) fn consume_native_body_for_teardown_test<'s>( scope: &mut v8::PinScope<'s, '_>, response: v8::Local<'s, v8::Object>, + method: &str, ) -> v8::Local<'s, v8::Promise> { - let result = - consume_network_body_value_from_object(scope, response, NetworkBodyConsumptionKind::Text); + let kind = match method { + "text" => NetworkBodyConsumptionKind::Text, + "json" => NetworkBodyConsumptionKind::Json, + "arrayBuffer" => NetworkBodyConsumptionKind::ArrayBuffer, + _ => panic!("unsupported native body teardown fixture: {method}"), + }; + let result = consume_network_body_value_from_object(scope, response, kind); if let NetworkBodyConsumption::Pending(promise) = result { return promise; } diff --git a/moli-renderer-v8/src/network_host/fetch_surface.rs b/moli-renderer-v8/src/network_host/fetch_surface.rs index 351a179c078..584bab6f414 100644 --- a/moli-renderer-v8/src/network_host/fetch_surface.rs +++ b/moli-renderer-v8/src/network_host/fetch_surface.rs @@ -4,8 +4,6 @@ use super::headers::{ }; use super::response::{ParsedResponseInit, install_response_body_methods}; use super::*; -use crate::context_bootstrap::readable_stream_disturbed; -pub(in crate::network_host) use crate::util::constructor_prototype; use crate::util::{ callback_data_index_value, callback_data_item, get_private_value, set_private_value, throw_range_error, v8_string, diff --git a/moli-renderer-v8/src/network_host/request/init.rs b/moli-renderer-v8/src/network_host/request/init.rs index 490fac921ee..41779e0ac87 100644 --- a/moli-renderer-v8/src/network_host/request/init.rs +++ b/moli-renderer-v8/src/network_host/request/init.rs @@ -296,9 +296,12 @@ pub(crate) fn parse_fetch_init<'s>( Some(PreparedBodySource::Bytes(bytes)) => (Some(bytes), None), Some(PreparedBodySource::Blob(blob)) => { let blob = v8::Local::new(scope, blob); - (blob::blob_bytes_from_object(scope, blob), None) + ( + blob::blob_bytes_from_object(scope, blob).map(bytes::Bytes::from), + None, + ) } - Some(PreparedBodySource::Stream(stream)) => (Some(Vec::new()), Some(stream)), + Some(PreparedBodySource::Stream(stream)) => (Some(bytes::Bytes::new()), Some(stream)), None => (None, None), }; diff --git a/moli-renderer-v8/src/network_host/request/input.rs b/moli-renderer-v8/src/network_host/request/input.rs index f8f6552fb8d..16546fad4c8 100644 --- a/moli-renderer-v8/src/network_host/request/input.rs +++ b/moli-renderer-v8/src/network_host/request/input.rs @@ -292,9 +292,9 @@ fn request_input_snapshot_from_private_slots<'s>( let body = if body_unusable || for_constructor { None } else { - try_network_body_bytes_from_object(scope, object).map_err(|_| { - webidl::WebIdlError::custom_message("Failed to materialize request body") - })?.map(bytes::Bytes::from) + try_network_body_bytes_from_object(scope, object) + .map_err(|_| webidl::WebIdlError::custom_message("Failed to materialize request body"))? + .map(bytes::Bytes::from) }; // A Request copies its internal list, without consulting a script-defined // headers getter or the Headers object's public iterator. diff --git a/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs b/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs index e15a4412bb4..37348468df9 100644 --- a/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs +++ b/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs @@ -125,7 +125,13 @@ fn read_native_body_for_teardown_test<'s>( // retired. Public Body methods instead require the receiver's live fetch // task source, whose lifetime is covered by the body_completion tests. let response = v8::Local::::try_from(args.get(0)).unwrap(); - let promise = crate::network_host::consume_native_body_text_for_teardown_test(scope, response); + let method = if args.get(1).is_undefined() { + "text".to_owned() + } else { + args.get(1).to_rust_string_lossy(scope) + }; + let promise = + crate::network_host::consume_native_body_for_teardown_test(scope, response, &method); rv.set(promise.into()); } @@ -348,7 +354,7 @@ fn assert_body_result_is_collectible_in_retained_realm( ) { let (mut vm, body_source_id) = new_vm_with_pending_response_for_teardown_test(); vm.eval(&format!( - "globalThis.bodyPromise = response.{method}();\n\ + "globalThis.bodyPromise = readNativeBodyForTeardown(response, '{method}');\n\ bodyPromise.catch(() => {{}});\n\ globalThis.keepRealmAlive = () => document.URL;" )) diff --git a/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs b/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs index bd3a697694a..9a21812fc3c 100644 --- a/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs +++ b/moli-renderer-v8/src/script_vm/tests/fetch_resource_timing.rs @@ -60,7 +60,7 @@ async fn handle_timing_request( if url.path() == "/compressed-worker.js" { let body = r#"self.onmessage = async event => { const probe = (0, eval)('(' + event.data.source + ')'); - postMessage(await probe(event.data.origins, event.data.label)); + postMessage(await probe(event.data.origins, event.data.label, event.data.filter)); close(); };"#; let head = format!( @@ -398,14 +398,23 @@ fn xhr_resource_timing_reads_internal_headers_before_cors_exposure() { } async fn run_resource_timing_fixture(fixture: &str, expected_total: usize) { + run_resource_timing_fixture_scenario(fixture, expected_total, None).await; +} + +async fn run_resource_timing_fixture_scenario( + fixture: &str, + expected_total: usize, + scenario: Option<&serde_json::Value>, +) -> serde_json::Value { let first = timing_server().await; let second = timing_server().await; let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap(); let mut page = new_storage_page_task_executor_test_vm_with_loader(&format!("{}/", first.origin), &loader); page.eval(&format!( - "globalThis.__fetchOrigins = {}; globalThis.__fetchTimingResult = null;", + "globalThis.__fetchOrigins = {}; globalThis.__fetchTimingResult = null;globalThis.__compressedTimingScenario = {};", serde_json::to_string(&[&first.origin, &second.origin]).unwrap(), + serde_json::to_string(&scenario).unwrap(), )) .unwrap(); page.eval(&format!( @@ -425,13 +434,58 @@ async fn run_resource_timing_fixture(fixture: &str, expected_total: usize) { .unwrap_or_else(|error| panic!("fetch timing probe returned {result:?}: {error}")); assert_eq!(result["total"], expected_total); assert_eq!(result["failures"], serde_json::json!([])); + result } #[tokio::test(flavor = "current_thread")] async fn compressed_resource_timing_sizes_survive_window_child_worker_and_memory_cache() { - run_resource_timing_fixture( - include_str!("../../../tests/fixtures/compressed-request-resource-timing.js"), + let mut names = std::collections::BTreeSet::new(); + for realm in ["window", "child", "worker"] { + let apis = if realm == "worker" { + ["fetch", "xhr", "sync-xhr"].as_slice() + } else { + ["fetch", "xhr"].as_slice() + }; + for api in apis { + let modes = if *api == "fetch" { + [ + "same", + "cors", + "cors-tao", + "denied", + "opaque", + "opaque-tao", + "cache", + ] + .as_slice() + } else { + ["same", "cors", "cors-tao", "denied", "cache"].as_slice() + }; + for mode in modes { + let scenario = serde_json::json!({"realm": realm, "api": api, "mode": mode}); + let total = if *mode == "cache" { 8 } else { 15 }; + let result = run_resource_timing_fixture_scenario( + include_str!("../../../tests/fixtures/compressed-request-resource-timing.js"), + total, + Some(&scenario), + ) + .await; + let checks = result["names"].as_array().unwrap(); + assert_eq!(checks.len(), total, "{scenario}"); + for check in checks { + let name = check.as_str().unwrap(); + assert!( + name.starts_with(&format!("{realm}: {api}/")), + "{scenario}: {name}" + ); + assert!(names.insert(name.to_owned()), "duplicate check: {name}"); + } + } + } + } + assert_eq!( + names.len(), 566, - ) - .await; + "all Window, child, worker, coding, API and cache cases must run" + ); } diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs index 4f23ba7fab8..340a30ff61d 100644 --- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs +++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs @@ -401,7 +401,7 @@ impl ServiceWorkerRuntimeService { &started.response_head.response_type, started.response_head.cors_exposed_header_names.as_deref(), started.response_head.status, - job.request.redirect_mode, + job.request.redirect_mode(), ), skip_fetch_security_validation: true, internal_id: job.internal_id, diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rs index 4c4f975bba4..2960401aa4e 100644 --- a/moli-renderer-v8/src/worker/global_scope/fetch.rs +++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs @@ -286,7 +286,9 @@ pub(in crate::worker) fn spawn_worker_fetch_network( if let Some(check) = redirect_check { request = request.with_redirect_check(check); } - if request.auth_requires_buffered_transport() || !request.transport_follows_redirects() { + if request.auth_requires_buffered_transport() + || !request.transport_follows_redirects() + { match fetch_browser_subresource_with_preflight_headers_and_network_metadata( loader.clone(), request, diff --git a/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js b/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js index 8a8eed1c764..cd42b4e129d 100644 --- a/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js +++ b/moli-renderer-v8/tests/fixtures/compressed-request-resource-timing.js @@ -1,5 +1,5 @@ (async () => { - const probe = async (origins, label) => { + const probe = async (origins, label, filter) => { const rows = []; globalThis.__compressedTimingProgress = rows; const checks = async (name, run) => { @@ -49,6 +49,7 @@ ]; let next = 0; for (const api of apis) for (const mode of ['same', 'cors', 'cors-tao', 'denied', ...(api === 'fetch' ? ['opaque', 'opaque-tao'] : [])]) for (const spec of specs) { + if (filter && (filter.api !== api || filter.mode !== mode)) continue; await checks(`${api}/${mode}/${spec.kind}/${spec.chunked ? 'chunked' : 'length'}/${spec.method || 'GET'}`, async () => { const url = new URL('/compressed', origins[mode === 'same' ? 0 : 1]); url.searchParams.set('key', `${label}-${++next}`); @@ -75,6 +76,7 @@ }); } for (const api of apis) for (const {kind, encoded} of [{kind: 'gzip', encoded: 23}, {kind: 'zstd', encoded: 12}]) { + if (filter && (filter.api !== api || filter.mode !== 'cache')) continue; const base = `${origins[0]}/compressed?kind=${kind}&cache=1&key=${label}-${api}-cache`; for (let phase = 0; phase < 4; ++phase) await checks(`${api}/${kind}/cache/${phase}`, async () => { const url = base + '#phase-' + phase; @@ -86,28 +88,36 @@ assert(performance.getEntriesByName(url, 'resource').length === 1, 'Duplicate cached timing'); }); } - return {total: rows.length, failures: rows.filter(row => !row.passed)}; + return {total: rows.length, failures: rows.filter(row => !row.passed), names: rows.map(row => row.name)}; }; - const results = [await probe(__fetchOrigins, 'window')]; - const iframe = document.createElement('iframe'); - const child = new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(Error('Child timing probe did not finish')), 30000); - const receive = event => { - if (event.source !== iframe.contentWindow || event.data?.kind !== 'compressed-timing') return; - clearTimeout(timer); removeEventListener('message', receive); resolve(event.data.result); - }; - addEventListener('message', receive); - }); - iframe.srcdoc = '