From bf47da3d8d664cc5deda2e81c7e4d8b68dbbf495 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Sun, 18 Oct 2026 19:10:44 +0800 Subject: [PATCH] feat(fetch): support partitioned compression dictionaries --- Cargo.lock | 51 ++ moli-fetch/Cargo.toml | 8 + moli-fetch/src/blocking/cache/vary.rs | 6 +- moli-fetch/src/blocking/mod.rs | 13 +- moli-fetch/src/client.rs | 47 +- .../src/compression_dictionary/decoding.rs | 448 +++++++++++++++ .../src/compression_dictionary/fields.rs | 117 ++++ moli-fetch/src/compression_dictionary/mod.rs | 531 ++++++++++++++++++ moli-fetch/src/content_coding.rs | 19 +- moli-fetch/src/lib.rs | 2 + moli-fetch/src/request.rs | 22 +- moli-fetch/src/response.rs | 2 + moli-fetch/src/runtime.rs | 240 +++++++- .../src/tests/compression_dictionary.rs | 434 ++++++++++++++ moli-fetch/src/tests/mod.rs | 1 + .../compression-dictionary/.gitattributes | 7 + .../fixtures/compression-dictionary/README.md | 5 + .../compression-dictionary/dictionary.br | Bin 0 -> 29 bytes .../compression-dictionary/dictionary.dcb | Bin 0 -> 47 bytes .../compression-dictionary/dictionary.dcz | Bin 0 -> 59 bytes .../compression-dictionary/dictionary.deflate | Bin 0 -> 33 bytes .../compression-dictionary/dictionary.gzip | Bin 0 -> 45 bytes .../dictionary.raw-deflate | Bin 0 -> 27 bytes .../compression-dictionary/dictionary.zstd | Bin 0 -> 40 bytes .../compression-dictionary/payload.dcb | 1 + .../compression-dictionary/payload.dcz | Bin 0 -> 83 bytes .../src/network/backend/memory_cache.rs | 21 +- .../src/network/request_client.rs | 10 +- .../src/stylesheet_runtime/connected.rs | 35 +- .../moli-fetch-types/src/header_policy.rs | 2 + 30 files changed, 1974 insertions(+), 48 deletions(-) create mode 100644 moli-fetch/src/compression_dictionary/decoding.rs create mode 100644 moli-fetch/src/compression_dictionary/fields.rs create mode 100644 moli-fetch/src/compression_dictionary/mod.rs create mode 100644 moli-fetch/src/tests/compression_dictionary.rs create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/.gitattributes create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/README.md create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.br create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.dcb create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.dcz create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.deflate create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.gzip create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.raw-deflate create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/dictionary.zstd create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/payload.dcb create mode 100644 moli-fetch/tests/fixtures/compression-dictionary/payload.dcz diff --git a/Cargo.lock b/Cargo.lock index d25c9eee3bb..d81dfe3f3cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,10 +2964,13 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", + "base64 0.22.1", + "brotlic-sys", "cidr", "crossbeam-channel", "curl", "curl-sys", + "flate2", "h2", "http", "moli-browser-profile", @@ -2986,8 +2989,11 @@ dependencies = [ "parking_lot", "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "rcgen", + "regex", "serde", "serde_json", + "sfv", + "sha2", "strum", "thiserror", "tokio", @@ -2996,7 +3002,9 @@ dependencies = [ "tokio-util", "tracing", "url", + "urlpattern", "wildcard", + "zstd-sys", ] [[package]] @@ -5028,6 +5036,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "regex" version = "1.12.3" @@ -5423,6 +5451,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sfv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890ef0e5532daf21fdaa3dbaf991417187f3a87ab69231fe69be942495746480" +dependencies = [ + "base64 0.22.1", + "indexmap", + "ref-cast", +] + [[package]] name = "sha1" version = "0.10.6" @@ -6569,6 +6608,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "urlpattern" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df16f50ef4cc145211879a3867ba757076b25dfee812040dcb0658bd9ae7904b" +dependencies = [ + "icu_properties", + "regex", + "serde", + "url", +] + [[package]] name = "usvg" version = "0.48.1" diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.toml index a178bcdc557..231ff328cf2 100644 --- a/moli-fetch/Cargo.toml +++ b/moli-fetch/Cargo.toml @@ -17,6 +17,9 @@ cidr = "0.3.2" crossbeam-channel = "0.5.15" curl = { version = "0.4.51", default-features = false, features = ["brotli", "browser-response-headers", "http2", "poll_7_68_0", "ssl", "static-curl"] } curl-sys = { version = "0.4.91", default-features = false } +base64 = "0.22.1" +brotlic-sys = "0.2.2" +flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] } http = "1" moli-browser-profile = { path = "../moli-browser-profile" } moli-cookie-jar = { path = "../moli-cookie-jar" } @@ -31,14 +34,19 @@ moli-web-bot-auth = { path = "../moli-web-bot-auth" } moli-web-mime = { path = "../moli-web-mime" } parking_lot = "0.12" percent-encoding = "2" +regex = "1" serde = { version = "1.0.228", features = ["derive"] } +sfv = "0.15.0" +sha2 = "0.11.0" strum = { version = "0.28.0", features = ["derive"] } thiserror = "2.0.17" tokio = { version = "1.51.0", features = ["rt", "sync"] } tokio-util = { version = "0.7.18", features = ["rt"] } tracing = "0.1.41" url = "2.5.7" +urlpattern = "0.6.0" wildcard = "0.3.0" +zstd-sys = { version = "2.1.2", default-features = false, features = ["vendored"] } [dev-dependencies] serde_json = "1" diff --git a/moli-fetch/src/blocking/cache/vary.rs b/moli-fetch/src/blocking/cache/vary.rs index 2da560fc7e8..4308bdcd367 100644 --- a/moli-fetch/src/blocking/cache/vary.rs +++ b/moli-fetch/src/blocking/cache/vary.rs @@ -1,6 +1,6 @@ use std::str; -use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader, response_vary_header_names}; +use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader}; use url::Url; use crate::{FetchConfig, Request}; @@ -11,6 +11,8 @@ const SUPPORTED_CACHE_REQUEST_HEADERS: &[&str] = &[ "accept", "accept-encoding", "accept-language", + "available-dictionary", + "dictionary-id", "referer", "sec-ch-ua", "sec-ch-ua-arch", @@ -38,7 +40,7 @@ pub(super) fn vary_headers_for_response( response_headers: &[(String, Vec)], ) -> Option> { let mut out = Vec::new(); - for normalized_name in response_vary_header_names(response_headers)? { + for normalized_name in crate::cache_vary_header_names(request_url, response_headers)? { if !supported_vary_header(&normalized_name) { return None; } diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs index b765ad86194..e77cda485cd 100644 --- a/moli-fetch/src/blocking/mod.rs +++ b/moli-fetch/src/blocking/mod.rs @@ -282,10 +282,17 @@ pub(crate) fn outgoing_request_header_bytes_for_url( let value = if header_present(&outgoing, "range") { "identity".to_owned() } else { - crate::content_coding::default_accept_encoding() + if request.compression_dictionary.is_some() { + crate::content_coding::dictionary_accept_encoding() + } else { + crate::content_coding::default_accept_encoding() + } }; outgoing.push(("Accept-Encoding".to_owned(), value.into_bytes())); } + if let Some(dictionary) = &request.compression_dictionary { + crate::compression_dictionary::append_request_headers(&mut outgoing, dictionary); + } if request.has_browser_identity_override() { append_encoded_header_if_missing( @@ -649,6 +656,10 @@ pub(crate) fn configure_easy( .context("failed to disable curl redirect following")?; easy.accept_encoding("") .context("failed to enable curl response decompression")?; + easy.http_content_decoding(false) + .context("failed to configure response content decoding")?; + easy.get_mut() + .begin_dictionary_response(request, request_url); // TLS client identities are Fetch credentials. Check each redirect target; // curl also uses the client certificate configuration to match reusable diff --git a/moli-fetch/src/client.rs b/moli-fetch/src/client.rs index cbe4ee6e198..436e945f5fe 100644 --- a/moli-fetch/src/client.rs +++ b/moli-fetch/src/client.rs @@ -20,6 +20,7 @@ pub struct FetchClientHandle { config: FetchConfig, cookie_store: SharedBrowserCookieStore, runtime: FetchRuntimeHandle, + dictionaries: crate::compression_dictionary::SharedDictionaryStore, } /// Unique structured-concurrency owner for one fetch client. @@ -39,6 +40,17 @@ impl FetchClientHandle { /// network admission, after the caller has selected the resource kind. pub fn prepare_request(&self, request: &mut Request) { request.prepare_accept_header(&self.config); + let url = request.url.clone(); + if let std::borrow::Cow::Owned(prepared) = + crate::compression_dictionary::DictionaryStore::prepare( + &self.dictionaries, + &self.config, + request, + &url, + ) + { + *request = prepared; + } } /// Creates native WebSockets on this client's HTTP owner and Multi. The @@ -207,14 +219,42 @@ impl FetchClientHandle { /// Cookie-store and other mutable transport-only fields are not included; /// renderer caches must not admit Vary on those fields from this view. pub fn cache_request_headers(&self, request: &Request) -> crate::RequestHeaders { - crate::blocking::outgoing_request_header_bytes_for_url( + let prepared = crate::compression_dictionary::DictionaryStore::prepare( + &self.dictionaries, &self.config, request, &request.url, + ); + crate::blocking::outgoing_request_header_bytes_for_url( + &self.config, + &prepared, + &request.url, None, ) } + /// A complete cached representation can register its dictionary again. + /// Its retained freshness deadline must not be extended by this read. + pub fn observe_cached_response( + &self, + request: &Request, + response: &RawResponse, + fresh_until_unix_ms: u64, + ) { + if response + .headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary")) + { + crate::compression_dictionary::DictionaryResponse::from_cached( + Arc::clone(&self.dictionaries), + request, + response, + fresh_until_unix_ms, + ); + } + } + pub fn matches_config(&self, config: &FetchConfig) -> bool { &self.config == config } @@ -274,15 +314,18 @@ impl FetchClient { } let client_hint_preferences = Arc::new(parking_lot::Mutex::new(ClientHintPreferences::default())); - let runtime_owner = FetchRuntimeOwner::new_with_client_hint_preferences( + let dictionaries = Arc::default(); + let runtime_owner = FetchRuntimeOwner::new_with_network_state( config, Arc::clone(&cookie_store), Arc::clone(&client_hint_preferences), + Arc::clone(&dictionaries), ); let handle = FetchClientHandle { config: config.clone(), runtime: runtime_owner.handle(), cookie_store, + dictionaries, }; Self { handle, diff --git a/moli-fetch/src/compression_dictionary/decoding.rs b/moli-fetch/src/compression_dictionary/decoding.rs new file mode 100644 index 00000000000..87c0da92369 --- /dev/null +++ b/moli-fetch/src/compression_dictionary/decoding.rs @@ -0,0 +1,448 @@ +//! Decode response content with the linked Brotli/zlib/Zstandard libraries. +//! The same stack validates completion and counts wire bytes for ordinary and +//! dictionary-compressed responses, independently of libcurl's decoder build. + +use std::{ + ffi::c_void, + ptr::{self, NonNull}, + sync::Arc, +}; + +use anyhow::{Result, anyhow, bail, ensure}; +use flate2::{Decompress, FlushDecompress, Status}; + +use super::Dictionary; + +const OUTPUT_SIZE: usize = 16 * 1024; +type Output<'a> = dyn FnMut(&[u8]) -> Result<()> + 'a; + +pub(crate) struct DictionaryDecoding { + dictionary: Option>, + stages: Option>, + pub(crate) encoded_bytes: u64, +} + +impl DictionaryDecoding { + pub(crate) fn new(dictionary: Option>) -> Self { + Self { + dictionary, + stages: None, + encoded_bytes: 0, + } + } + + pub(crate) fn reset(&mut self) { + self.stages = None; + self.encoded_bytes = 0; + } + + pub(crate) fn uses_dictionary(&self) -> bool { + self.dictionary.is_some() + } + + pub(crate) fn initialize(&mut self, headers: &[(String, Vec)]) -> Result<()> { + if self.stages.is_some() { + return Ok(()); + } + let mut names = Vec::new(); + for (_, value) in headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("content-encoding")) + { + let value = crate::decode_header_value(value); + names.extend( + crate::split_http_header_list(&value) + .filter(|value| !value.is_empty()) + .map(str::to_ascii_lowercase), + ); + } + ensure!(names.len() <= 5, "too many content encodings"); + let mut stages = Vec::new(); + for name in names.into_iter().rev() { + let stage = match name.as_str() { + "identity" => continue, + "gzip" | "x-gzip" => Stage::Flate(Flate::gzip()), + "deflate" => Stage::Flate(Flate::deflate()), + "br" => Stage::Brotli(Brotli::new(None)?), + "zstd" => Stage::Zstd(Zstd::new(None)?), + "dcb" | "dcz" => Stage::Dictionary { + brotli: name == "dcb", + dictionary: Arc::clone(self.dictionary.as_ref().ok_or_else(|| { + anyhow!("dictionary compression without a negotiated dictionary") + })?), + prefix: Vec::new(), + decoder: None, + }, + _ => bail!("unsupported content encoding {name}"), + }; + stages.push(stage); + } + self.stages = Some(stages); + Ok(()) + } + + pub(crate) fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> { + ensure!( + self.stages.is_some(), + "compressed response body arrived before headers" + ); + feed( + self.stages.as_mut().expect("initialized decoding stack"), + input, + output, + ) + } + + pub(crate) fn finish(&mut self, headers: &[(String, Vec)], bodyless: bool) -> Result<()> { + if bodyless { + return Ok(()); + } + // An empty payload never enters a content decoder. HEAD and null-body + // responses likewise do not validate an unused coding declaration. + if self.encoded_bytes == 0 { + return Ok(()); + } + self.initialize(headers)?; + for stage in self.stages.as_ref().expect("initialized decoding stack") { + ensure!(stage.finished(), "truncated compressed response"); + } + Ok(()) + } +} + +fn feed(stages: &mut [Stage], input: &[u8], output: &mut Output<'_>) -> Result<()> { + let Some((stage, remaining)) = stages.split_first_mut() else { + return output(input); + }; + stage.push(input, &mut |bytes| feed(remaining, bytes, output)) +} + +enum Stage { + Flate(Flate), + Brotli(Brotli), + Zstd(Zstd), + Dictionary { + brotli: bool, + dictionary: Arc, + prefix: Vec, + decoder: Option>, + }, +} + +impl Stage { + fn push(&mut self, mut input: &[u8], output: &mut Output<'_>) -> Result<()> { + match self { + Self::Flate(decoder) => decoder.push(input, output), + Self::Brotli(decoder) => decoder.push(input, output), + Self::Zstd(decoder) => decoder.push(input, output), + Self::Dictionary { + brotli, + dictionary, + prefix, + decoder, + } => { + if decoder.is_none() { + let magic: &[u8] = if *brotli { + b"\xffDCB" + } else { + b"\x5e\x2a\x4d\x18\x20\0\0\0" + }; + let length = magic.len() + 32; + let count = input.len().min(length - prefix.len()); + prefix.extend_from_slice(&input[..count]); + input = &input[count..]; + if prefix.len() < length { + return Ok(()); + } + ensure!( + &prefix[..magic.len()] == magic, + "invalid dictionary compression header" + ); + ensure!( + prefix[magic.len()..] == dictionary.hash, + "compression dictionary hash mismatch" + ); + *decoder = Some(Box::new(if *brotli { + Self::Brotli(Brotli::new(Some(Arc::clone(dictionary)))?) + } else { + Self::Zstd(Zstd::new(Some(Arc::clone(dictionary)))?) + })); + } + decoder + .as_mut() + .expect("validated dictionary header") + .push(input, output) + } + } + } + + fn finished(&self) -> bool { + match self { + Self::Flate(decoder) => decoder.finished, + Self::Brotli(decoder) => decoder.finished, + Self::Zstd(decoder) => decoder.finished, + Self::Dictionary { decoder, .. } => { + decoder.as_ref().is_some_and(|decoder| decoder.finished()) + } + } + } +} + +struct Flate { + state: Option, + gzip: bool, + prefix: Vec, + finished: bool, +} + +impl Flate { + fn gzip() -> Self { + Self { + state: Some(Decompress::new_gzip(15)), + gzip: true, + prefix: Vec::new(), + finished: false, + } + } + fn deflate() -> Self { + Self { + state: None, + gzip: false, + prefix: Vec::new(), + finished: false, + } + } + + fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> { + if self.state.is_none() { + self.prefix.extend_from_slice(input); + if self.prefix.len() < 2 { + return Ok(()); + } + let wrapped = self.prefix[0] & 15 == 8 + && u16::from_be_bytes([self.prefix[0], self.prefix[1]]).is_multiple_of(31); + self.state = Some(Decompress::new(wrapped)); + let input = std::mem::take(&mut self.prefix); + return self.push(&input, output); + } + let mut remaining = input; + loop { + if self.finished { + if remaining.is_empty() { + return Ok(()); + } + ensure!(self.gzip, "bytes after a deflate stream"); + self.state = Some(Decompress::new_gzip(15)); + self.finished = false; + } + let decoder = self.state.as_mut().expect("initialized zlib decoder"); + let mut buffer = [0; OUTPUT_SIZE]; + let before_in = decoder.total_in(); + let before_out = decoder.total_out(); + let status = decoder.decompress(remaining, &mut buffer, FlushDecompress::None)?; + let read = (decoder.total_in() - before_in) as usize; + let written = (decoder.total_out() - before_out) as usize; + remaining = &remaining[read..]; + if written != 0 { + output(&buffer[..written])?; + } + self.finished = status == Status::StreamEnd; + if self.finished && !remaining.is_empty() { + continue; + } + if (read == 0 && written == 0) || (remaining.is_empty() && written < buffer.len()) { + return Ok(()); + } + } + } +} + +struct Brotli { + state: NonNull, + _dictionary: Option>, + finished: bool, +} + +// The decoder has one owner and is accessed only through &mut self. Moving it +// to the curl owner thread does not create concurrent access to native state. +unsafe impl Send for Brotli {} + +impl Brotli { + fn new(dictionary: Option>) -> Result { + let state = NonNull::new(unsafe { + brotlic_sys::BrotliDecoderCreateInstance(None, None, ptr::null_mut()) + }) + .ok_or_else(|| anyhow!("failed to allocate Brotli decoder"))?; + let decoder = Self { + state, + _dictionary: dictionary, + finished: false, + }; + if let Some(dictionary) = &decoder._dictionary { + let attached = unsafe { + brotlic_sys::BrotliDecoderSetParameter( + state.as_ptr(), + brotlic_sys::BrotliDecoderParameter_BROTLI_DECODER_PARAM_LARGE_WINDOW, + 1, + ); + brotlic_sys::BrotliDecoderAttachDictionary( + state.as_ptr(), + brotlic_sys::BrotliSharedDictionaryType_BROTLI_SHARED_DICTIONARY_RAW, + dictionary.bytes.len(), + dictionary.bytes.as_ptr(), + ) + }; + ensure!(attached != 0, "failed to attach Brotli dictionary"); + } + Ok(decoder) + } + + fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> { + if input.is_empty() && self.finished { + return Ok(()); + } + ensure!(!self.finished, "bytes after a Brotli stream"); + let mut available_in = input.len(); + let mut next_in = input.as_ptr(); + loop { + let mut buffer = [0; OUTPUT_SIZE]; + let mut available_out = buffer.len(); + let mut next_out = buffer.as_mut_ptr(); + let result = unsafe { + brotlic_sys::BrotliDecoderDecompressStream( + self.state.as_ptr(), + &mut available_in, + &mut next_in, + &mut available_out, + &mut next_out, + ptr::null_mut(), + ) + }; + let written = buffer.len() - available_out; + if written != 0 { + output(&buffer[..written])?; + } + match result { + brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_ERROR => { + bail!("invalid Brotli response") + } + brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_SUCCESS => { + ensure!(available_in == 0, "bytes after a Brotli stream"); + self.finished = true; + return Ok(()); + } + brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT => { + return Ok(()); + } + brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT => {} + _ => bail!("invalid Brotli decoder status"), + } + } + } +} + +impl Drop for Brotli { + fn drop(&mut self) { + unsafe { + brotlic_sys::BrotliDecoderDestroyInstance(self.state.as_ptr()); + } + } +} + +struct Zstd { + state: NonNull, + dictionary: Option>, + finished: bool, +} + +// Like Brotli, this context has a unique owner and no concurrent native calls. +unsafe impl Send for Zstd {} + +impl Zstd { + fn new(dictionary: Option>) -> Result { + let state = NonNull::new(unsafe { zstd_sys::ZSTD_createDStream() }) + .ok_or_else(|| anyhow!("failed to allocate Zstandard decoder"))?; + let decoder = Self { + state, + dictionary, + finished: false, + }; + if let Some(dictionary) = &decoder.dictionary { + let result = unsafe { + zstd_sys::ZSTD_DCtx_refPrefix( + state.as_ptr(), + dictionary.bytes.as_ptr().cast(), + dictionary.bytes.len(), + ) + }; + ensure!( + unsafe { zstd_sys::ZSTD_isError(result) } == 0, + "failed to attach Zstandard dictionary" + ); + } + Ok(decoder) + } + + fn push(&mut self, bytes: &[u8], output: &mut Output<'_>) -> Result<()> { + if bytes.is_empty() && self.finished { + return Ok(()); + } + let mut input = zstd_sys::ZSTD_inBuffer { + src: bytes.as_ptr().cast::(), + size: bytes.len(), + pos: 0, + }; + loop { + if self.finished { + if let Some(dictionary) = &self.dictionary { + let result = unsafe { + zstd_sys::ZSTD_DCtx_refPrefix( + self.state.as_ptr(), + dictionary.bytes.as_ptr().cast(), + dictionary.bytes.len(), + ) + }; + ensure!( + unsafe { zstd_sys::ZSTD_isError(result) } == 0, + "failed to reattach Zstandard dictionary" + ); + } + self.finished = false; + } + let mut buffer = [0; OUTPUT_SIZE]; + let mut decoded = zstd_sys::ZSTD_outBuffer { + dst: buffer.as_mut_ptr().cast::(), + size: buffer.len(), + pos: 0, + }; + let before = input.pos; + let result = unsafe { + zstd_sys::ZSTD_decompressStream(self.state.as_ptr(), &mut decoded, &mut input) + }; + ensure!( + unsafe { zstd_sys::ZSTD_isError(result) } == 0, + "invalid Zstandard response" + ); + if decoded.pos != 0 { + output(&buffer[..decoded.pos])?; + } + self.finished = result == 0; + if self.finished && input.pos < input.size { + continue; + } + if (input.pos == input.size && decoded.pos < buffer.len()) + || (input.pos == before && decoded.pos == 0) + { + return Ok(()); + } + } + } +} + +impl Drop for Zstd { + fn drop(&mut self) { + unsafe { + zstd_sys::ZSTD_freeDStream(self.state.as_ptr()); + } + } +} diff --git a/moli-fetch/src/compression_dictionary/fields.rs b/moli-fetch/src/compression_dictionary/fields.rs new file mode 100644 index 00000000000..53a5a181025 --- /dev/null +++ b/moli-fetch/src/compression_dictionary/fields.rs @@ -0,0 +1,117 @@ +use sfv::{BareItem, Dictionary, List, ListEntry, Parser}; +use url::Url; +use urlpattern::{UrlPattern, UrlPatternInit, UrlPatternMatchInput}; + +#[derive(Debug)] +pub(crate) struct RegistrationOptions { + pattern: UrlPattern, + match_text: String, + destinations: Vec, + id: String, +} + +impl RegistrationOptions { + pub(crate) fn parse(url: &Url, headers: &[(String, Vec)]) -> Option { + let values = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary")) + .map(|(_, value)| std::str::from_utf8(value).ok()) + .collect::>>()?; + if values.is_empty() { + return None; + } + let combined = values.join(", "); + if combined.len() > 16 * 1024 { + return None; + } + let dictionary: Dictionary = Parser::new(&combined).parse().ok()?; + let ListEntry::Item(item) = dictionary.get("match")? else { + return None; + }; + let BareItem::String(value) = &item.bare_item else { + return None; + }; + let match_text = value.as_str().to_owned(); + if match_text.len() > 4096 { + return None; + } + let init = UrlPatternInit::parse_constructor_string::( + &match_text, + Some(url.clone()), + ) + .ok()?; + let pattern = UrlPattern::parse(init, Default::default()).ok()?; + if pattern.has_regexp_groups() { + return None; + } + let id = match dictionary.get("id") { + None => String::new(), + Some(ListEntry::Item(item)) => item.bare_item.as_string()?.as_str().to_owned(), + Some(ListEntry::InnerList(_)) => return None, + }; + if id.len() > 1024 { + return None; + } + if let Some(kind) = dictionary.get("type") { + let ListEntry::Item(item) = kind else { + return None; + }; + if item.bare_item.as_token()?.as_str() != "raw" { + return None; + } + } + let destinations = match dictionary.get("match-dest") { + None => Vec::new(), + Some(ListEntry::InnerList(list)) => list + .items + .iter() + .map(|item| Some(item.bare_item.as_string()?.as_str().to_owned())) + .collect::>>()?, + Some(ListEntry::Item(_)) => return None, + }; + Some(Self { + pattern, + match_text, + destinations, + id, + }) + } + + pub(crate) fn matches(&self, url: &Url, destination: &str) -> bool { + (self.destinations.is_empty() || self.destinations.iter().any(|value| value == destination)) + && self + .pattern + .test(UrlPatternMatchInput::Url(url.clone())) + .unwrap_or(false) + } + + pub(crate) fn match_length(&self) -> usize { + self.match_text.len() + } + + pub(crate) fn same_scope(&self, other: &Self) -> bool { + self.match_text == other.match_text && self.destinations == other.destinations + } + + pub(crate) fn id_header(&self) -> Option> { + (!self.id.is_empty()).then(|| { + format!("\"{}\"", self.id.replace('\\', "\\\\").replace('"', "\\\"")).into_bytes() + }) + } +} + +pub(super) fn clears_dictionaries(headers: &[(String, Vec)]) -> bool { + headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("clear-site-data")) + .filter_map(|(_, value)| std::str::from_utf8(value).ok()) + .filter_map(|value| Parser::new(value).parse::().ok()) + .flatten() + .any(|entry| match entry { + ListEntry::Item(item) => item + .bare_item + .as_string() + .is_some_and(|value| matches!(value.as_str(), "cache" | "cookies" | "*")), + ListEntry::InnerList(_) => false, + }) +} diff --git a/moli-fetch/src/compression_dictionary/mod.rs b/moli-fetch/src/compression_dictionary/mod.rs new file mode 100644 index 00000000000..47ff8875d18 --- /dev/null +++ b/moli-fetch/src/compression_dictionary/mod.rs @@ -0,0 +1,531 @@ +//! Compression dictionaries are network-context state, partitioned like the +//! requests which may use them. Bytes enter the store only after a readable +//! response completes; a header alone cannot register a dictionary. + +mod decoding; +mod fields; + +use std::{borrow::Cow, collections::VecDeque, fmt, io::Read, sync::Arc}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use moli_http_cache::{response_cache_policy, unix_now_ms}; +use moli_url::{is_potentially_trustworthy_url, origin_ascii_serialization}; +use parking_lot::Mutex; +use sha2::{Digest, Sha256}; +use url::Url; + +use crate::{FetchConfig, Request, RequestMode}; + +pub(crate) use decoding::DictionaryDecoding; +pub(crate) use fields::RegistrationOptions; + +const MAX_DICTIONARY_BYTES: usize = 64 * 1024 * 1024; +const MAX_STORE_BYTES: usize = 128 * 1024 * 1024; +const MAX_STORE_ENTRIES: usize = 256; + +pub(crate) type SharedDictionaryStore = Arc>; + +pub(crate) struct Dictionary { + pub(crate) bytes: Arc<[u8]>, + pub(crate) hash: [u8; 32], + available: Vec, + id: Option>, +} + +impl fmt::Debug for Dictionary { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Dictionary") + .field("byte_length", &self.bytes.len()) + .field("available", &self.available) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DictionaryRequestSnapshot { + url: Url, + partition: Option, + destination: String, + permitted: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Partition { + top_frame: String, + frame: String, + network: Option, +} + +impl Partition { + fn for_request(request: &Request, url: &Url) -> Option { + let frame = match request.request_origin() { + Some(origin) if origin.is_opaque() => return None, + Some(origin) => origin.ascii_serialization().to_owned(), + None => origin_ascii_serialization(url), + }; + Some(Self { + top_frame: request + .cookie_context + .browser_context + .top_frame_origin_url + .as_ref() + .or(request + .cookie_context + .browser_context + .site_for_cookies_url + .as_ref()) + .map_or_else(|| frame.clone(), origin_ascii_serialization), + frame, + network: request.network_partition_key().map(str::to_owned), + }) + } +} + +#[derive(Debug)] +struct Entry { + partition: Partition, + origin: String, + options: RegistrationOptions, + dictionary: Arc, + expires: u64, +} + +#[derive(Debug, Default)] +pub(crate) struct DictionaryStore { + entries: VecDeque, + bytes: usize, + generation: u64, +} + +impl DictionaryStore { + pub(crate) fn prepare<'a>( + store: &SharedDictionaryStore, + config: &FetchConfig, + request: &'a Request, + url: &Url, + ) -> Cow<'a, Request> { + Self::prepare_inner(store, config, Cow::Borrowed(request), url) + } + + pub(crate) fn prepare_owned( + store: &SharedDictionaryStore, + config: &FetchConfig, + request: Request, + url: &Url, + ) -> Request { + Self::prepare_inner(store, config, Cow::Owned(request), url).into_owned() + } + + fn prepare_inner<'a>( + store: &SharedDictionaryStore, + config: &FetchConfig, + mut prepared: Cow<'a, Request>, + url: &Url, + ) -> Cow<'a, Request> { + let headers = + crate::blocking::outgoing_request_header_bytes_for_url(config, &prepared, url, None); + let destination = headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("sec-fetch-dest")) + .and_then(|(_, value)| std::str::from_utf8(value).ok()) + .unwrap_or(""); + let key = DictionaryRequestSnapshot { + url: url.clone(), + partition: Partition::for_request(&prepared, url), + destination: if destination == "empty" { + "" + } else { + destination + } + .to_owned(), + permitted: is_potentially_trustworthy_url(url) + && matches!(url.scheme(), "http" | "https") + && !(prepared.request_mode == RequestMode::NoCors + && prepared.has_cross_origin_url(url)) + && !prepared + .request_headers + .iter() + .any(|(name, _)| overrides_dictionary_negotiation(name)) + && !config + .default_request_headers() + .iter() + .any(|(name, _)| overrides_dictionary_negotiation(name)), + }; + if prepared.compression_dictionary_snapshot.as_ref() == Some(&key) + && prepared + .compression_dictionary_store + .as_ref() + .is_some_and(|previous| Arc::ptr_eq(previous, store)) + { + return prepared; + } + let request = prepared.to_mut(); + request.compression_dictionary = None; + request.compression_dictionary_store = Some(Arc::clone(store)); + request.compression_dictionary_snapshot = Some(key.clone()); + if !key.permitted { + return prepared; + } + let Some(partition) = key.partition else { + return prepared; + }; + let origin = origin_ascii_serialization(url); + let now = unix_now_ms(); + let mut locked = store.lock(); + locked.retain(|entry| entry.expires > now); + let selected = locked + .entries + .iter() + .enumerate() + .filter(|(_, entry)| { + entry.partition == partition + && entry.origin == origin + && entry.options.matches(url, &key.destination) + }) + .max_by_key(|(index, entry)| (entry.options.match_length(), *index)) + .map(|(index, _)| index); + if let Some(index) = selected { + let entry = locked + .entries + .remove(index) + .expect("selected dictionary exists"); + request.compression_dictionary = Some(Arc::clone(&entry.dictionary)); + locked.entries.push_back(entry); + } + prepared + } + + fn retain(&mut self, mut predicate: impl FnMut(&Entry) -> bool) { + self.entries.retain(|entry| predicate(entry)); + self.bytes = self + .entries + .iter() + .map(|entry| entry.dictionary.bytes.len()) + .sum(); + } + + fn clear_origin(&mut self, origin: &str) { + self.retain(|entry| entry.origin != origin); + // Cancel captures already in progress as well as retained entries. + self.generation = self.generation.wrapping_add(1); + } + + fn insert(&mut self, entry: Entry) { + self.retain(|old| { + !(old.partition == entry.partition + && old.origin == entry.origin + && old.options.same_scope(&entry.options)) + }); + self.bytes += entry.dictionary.bytes.len(); + self.entries.push_back(entry); + while self.bytes > MAX_STORE_BYTES || self.entries.len() > MAX_STORE_ENTRIES { + if let Some(old) = self.entries.pop_front() { + self.bytes -= old.dictionary.bytes.len(); + } + } + } +} + +/// Owns one response capture, including the clear-data generation at request +/// admission. Interrupted, opaque, stale and partial responses never commit. +pub(crate) struct DictionaryResponse { + store: SharedDictionaryStore, + request: ResponsePolicy, + url: Url, + generation: u64, + status: u16, + headers: Vec<(String, Vec)>, + capture: Option, + headers_finalized: bool, +} + +struct ResponsePolicy { + partition: Option, + head: bool, + credentials: bool, + opaque: bool, + cors: Option<(String, crate::RequestCredentialsMode)>, +} + +impl ResponsePolicy { + fn new(request: &Request, url: &Url) -> Self { + Self { + partition: Partition::for_request(request, url), + head: request.method == "HEAD", + credentials: request.allows_credentials_for_url(url), + opaque: request.request_mode == RequestMode::NoCors + && request.has_cross_origin_url(url), + cors: request + .request_origin() + .filter(|_| { + request.request_mode == RequestMode::Cors && request.has_cross_origin_url(url) + }) + .map(|origin| { + ( + request.url_list(url).serialized_origin(origin), + request.credentials_mode, + ) + }), + } + } + + fn readable(&self, headers: &[(String, Vec)]) -> bool { + !self.opaque + && self.cors.as_ref().is_none_or(|(origin, mode)| { + crate::validate_cors_response_for_origin(origin, headers, *mode).is_ok() + }) + } +} + +struct Capture { + options: RegistrationOptions, + partition: Partition, + expires: u64, + bytes: Vec, +} + +impl DictionaryResponse { + pub(crate) fn new(store: SharedDictionaryStore, request: &Request, url: Url) -> Self { + let generation = store.lock().generation; + Self { + store, + request: ResponsePolicy::new(request, &url), + url, + generation, + status: 0, + headers: Vec::new(), + capture: None, + headers_finalized: false, + } + } + + pub(crate) fn header(&mut self, data: &[u8]) { + if let Some(status) = crate::response::parse_http_response_status_line(data) { + self.status = status.status; + self.headers.clear(); + self.capture = None; + self.headers_finalized = false; + return; + } + if self.headers_finalized { + return; + } + if let Some(header) = crate::headers::parse_http_response_header_line(data) { + self.headers.push(header); + return; + } + if data.iter().any(|byte| !matches!(byte, b'\r' | b'\n')) || self.status < 200 { + return; + } + self.headers_finalized = true; + if !is_potentially_trustworthy_url(&self.url) { + return; + } + if fields::clears_dictionaries(&self.headers) && self.request.credentials { + self.store + .lock() + .clear_origin(&origin_ascii_serialization(&self.url)); + } + self.begin_capture(None); + } + + fn begin_capture(&mut self, fresh_until: Option) { + if self.status != 200 || self.request.head || !self.request.readable(&self.headers) { + return; + } + let Some(partition) = self.request.partition.clone() else { + return; + }; + let Some(options) = RegistrationOptions::parse(&self.url, &self.headers) else { + return; + }; + let policy = response_cache_policy(&self.headers); + let Some(mut expires) = policy + .expires_at_unix_ms + .filter(|expires| *expires > unix_now_ms()) + else { + return; + }; + if let Some(deadline) = fresh_until { + expires = expires.min(deadline); + } + if expires <= unix_now_ms() { + return; + } + if !policy.store { + return; + } + self.capture = Some(Capture { + options, + partition, + expires, + bytes: Vec::new(), + }); + } + + pub(crate) fn decoded(&mut self, bytes: &[u8]) { + let Some(capture) = self.capture.as_mut() else { + return; + }; + if bytes.len() > MAX_DICTIONARY_BYTES.saturating_sub(capture.bytes.len()) { + self.capture = None; + } else { + capture.bytes.extend_from_slice(bytes); + } + } + + pub(crate) fn headers(&self) -> &[(String, Vec)] { + &self.headers + } + + pub(crate) fn headers_finalized(&self) -> bool { + self.headers_finalized + } + + pub(crate) fn bodyless(&self) -> bool { + self.request.head || matches!(self.status, 204 | 205 | 304) + } + + pub(crate) fn from_cached( + store: SharedDictionaryStore, + request: &Request, + response: &crate::RawResponse, + fresh_until: u64, + ) { + let Some(mut capture) = Self::cached( + store, + request, + &response.final_url, + response.status, + &response.headers, + Some(fresh_until), + ) else { + return; + }; + capture.decoded(response.body_bytes()); + capture.finish(); + } + + pub(crate) fn cached( + store: SharedDictionaryStore, + request: &Request, + url: &Url, + status: u16, + headers: &[(String, Vec)], + fresh_until: Option, + ) -> Option { + if !is_potentially_trustworthy_url(url) + || !headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary")) + { + return None; + } + let deadline = fresh_until.filter(|deadline| *deadline > unix_now_ms())?; + let mut capture = Self::new(store, request, url.clone()); + capture.status = status; + capture.headers = headers.to_vec(); + capture.begin_capture(Some(deadline)); + capture.capture.as_ref()?; + Some(capture) + } + + pub(crate) fn readable(&self) -> bool { + self.request.readable(&self.headers) + } + + pub(crate) fn finish(&mut self) { + let Some(capture) = self.capture.take() else { + return; + }; + if capture.expires <= unix_now_ms() { + return; + } + let hash: [u8; 32] = Sha256::digest(&capture.bytes).into(); + let available = format!(":{}:", STANDARD.encode(hash)).into_bytes(); + let dictionary = Arc::new(Dictionary { + bytes: capture.bytes.into(), + hash, + available, + id: capture.options.id_header(), + }); + let mut store = self.store.lock(); + if store.generation != self.generation { + return; + } + store.insert(Entry { + partition: capture.partition, + origin: origin_ascii_serialization(&self.url), + options: capture.options, + dictionary, + expires: capture.expires, + }); + } +} + +pub(crate) struct DictionaryCachedReader { + inner: R, + capture: Option, + cancel: crate::FetchCancelHandle, +} + +impl DictionaryCachedReader { + pub(crate) fn new( + inner: R, + capture: Option, + cancel: crate::FetchCancelHandle, + ) -> Self { + Self { + inner, + capture, + cancel, + } + } +} + +impl Read for DictionaryCachedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.inner.read(buffer)?; + if buffer.is_empty() { + return Ok(count); + } + if let Some(capture) = self.capture.as_mut() { + if count == 0 { + if !self.cancel.is_cancelled() { + capture.finish(); + } + self.capture = None; + } else { + capture.decoded(&buffer[..count]); + } + } + Ok(count) + } +} + +fn overrides_dictionary_negotiation(name: &str) -> bool { + ["range", "available-dictionary", "dictionary-id"] + .iter() + .any(|value| name.eq_ignore_ascii_case(value)) +} + +pub(crate) fn append_request_headers(headers: &mut crate::RequestHeaders, dictionary: &Dictionary) { + headers.push(( + "Available-Dictionary".to_owned(), + dictionary.available.clone(), + )); + if let Some(id) = &dictionary.id { + headers.push(("Dictionary-ID".to_owned(), id.clone())); + } +} + +/// Dictionary representations remain reusable after registration changes the +/// browser-generated negotiation fields. Other Vary fields remain in force. +pub fn cache_vary_header_names(url: &Url, headers: &[(String, Vec)]) -> Option> { + let mut names = moli_http_cache::response_vary_header_names(headers)?; + if is_potentially_trustworthy_url(url) && RegistrationOptions::parse(url, headers).is_some() { + names.retain(|name| name != "available-dictionary" && name != "accept-encoding"); + } + Some(names) +} diff --git a/moli-fetch/src/content_coding.rs b/moli-fetch/src/content_coding.rs index a4d2fbc5687..e8bbb2d5722 100644 --- a/moli-fetch/src/content_coding.rs +++ b/moli-fetch/src/content_coding.rs @@ -1,21 +1,16 @@ use moli_header_field::{HeaderFieldTokenMode, HeaderFieldTokenizer}; fn supported_content_codings() -> impl Iterator { - let version = curl::Version::get(); - [ - ("deflate", version.feature_libz()), - ("gzip", version.feature_libz()), - ("br", version.feature_brotli()), - ("zstd", version.feature_zstd()), - ] - .into_iter() - .filter_map(|(coding, supported)| supported.then_some(coding)) + ["deflate", "gzip", "br", "zstd", "dcb", "dcz"].into_iter() } /// Choose the browser's header before cache lookup and transport setup so /// negotiation and Vary matching use the same actual value. pub(crate) fn default_accept_encoding() -> String { - let value = supported_content_codings().collect::>().join(", "); + let value = supported_content_codings() + .filter(|coding| !matches!(*coding, "dcb" | "dcz")) + .collect::>() + .join(", "); if value.is_empty() { "identity".to_owned() } else { @@ -23,6 +18,10 @@ pub(crate) fn default_accept_encoding() -> String { } } +pub(crate) fn dictionary_accept_encoding() -> String { + supported_content_codings().collect::>().join(", ") +} + /// Resource Timing exposes registered, supported content codings rather than /// reflecting an arbitrary response header into the timing entry. pub fn resource_timing_content_encoding(headers: &[(String, Vec)]) -> &'static str { diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs index fff5f2c2bab..4c39359aa83 100644 --- a/moli-fetch/src/lib.rs +++ b/moli-fetch/src/lib.rs @@ -9,6 +9,7 @@ mod blocking; mod cancellation; mod client; mod client_hints; +mod compression_dictionary; mod config; mod content_coding; mod cors; @@ -43,6 +44,7 @@ pub use blocking::{ }; pub use cancellation::{FetchCancelHandle, FetchResponseBodySizes, FetchResponseTiming}; pub use client::{FetchClient, FetchClientHandle}; +pub use compression_dictionary::cache_vary_header_names; pub use config::FetchConfig; pub use content_coding::resource_timing_content_encoding; pub use cors::validate_cors_response_for_origin; diff --git a/moli-fetch/src/request.rs b/moli-fetch/src/request.rs index 346d91c3e84..24cdd79e048 100644 --- a/moli-fetch/src/request.rs +++ b/moli-fetch/src/request.rs @@ -112,6 +112,11 @@ pub struct Request { redirect_policy: Option, upload_observer: Option, redirect_check: Option, + pub(crate) compression_dictionary: Option>, + pub(crate) compression_dictionary_store: + Option, + pub(crate) compression_dictionary_snapshot: + Option, } /// Request-owned admission check, run before following each redirect target. @@ -272,6 +277,7 @@ pub enum BrowserRequestMetadata { Audio, AudioWorklet, Beacon, + Dictionary, EventSource, Fetch, Font, @@ -309,7 +315,12 @@ impl BrowserRequestMetadata { Self::Style | Self::StyleModule => "style", Self::TextTrack => "track", Self::Video => "video", - Self::Beacon | Self::EventSource | Self::Fetch | Self::Ping | Self::Xhr => "empty", + Self::Beacon + | Self::Dictionary + | Self::EventSource + | Self::Fetch + | Self::Ping + | Self::Xhr => "empty", } } } @@ -665,6 +676,9 @@ impl Request { redirect_policy: None, upload_observer: None, redirect_check: None, + compression_dictionary: None, + compression_dictionary_store: None, + compression_dictionary_snapshot: None, }) } @@ -707,6 +721,9 @@ impl Request { redirect_policy: None, upload_observer: None, redirect_check: None, + compression_dictionary: None, + compression_dictionary_store: None, + compression_dictionary_snapshot: None, } } @@ -782,6 +799,9 @@ impl Request { redirect_policy: None, upload_observer: None, redirect_check: None, + compression_dictionary: None, + compression_dictionary_store: None, + compression_dictionary_snapshot: None, } } diff --git a/moli-fetch/src/response.rs b/moli-fetch/src/response.rs index 16c07e49e36..b3ada0015b2 100644 --- a/moli-fetch/src/response.rs +++ b/moli-fetch/src/response.rs @@ -995,6 +995,8 @@ mod resource_timing_tests { (vec!["deflate"], "deflate"), (vec!["br"], "br"), (vec!["zStd "], "zstd"), + (vec!["dcb"], "dcb"), + (vec!["dcz"], "dcz"), (vec!["gzip, deflate, Apple"], "multiple"), (vec!["gzip, "], "multiple"), (vec!["gzip", "deflate"], "multiple"), diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs index 0fe0ec6074d..fde29fe7501 100644 --- a/moli-fetch/src/runtime.rs +++ b/moli-fetch/src/runtime.rs @@ -370,19 +370,21 @@ impl PendingStreamingRawResponse { impl FetchRuntimeOwner { #[cfg(test)] pub(crate) fn new(config: &FetchConfig, cookie_store: SharedBrowserCookieStore) -> Self { - Self::new_with_client_hint_preferences( + Self::new_with_network_state( config, cookie_store, Arc::new(Mutex::new( crate::client_hints::ClientHintPreferences::default(), )), + Arc::default(), ) } - pub(crate) fn new_with_client_hint_preferences( + pub(crate) fn new_with_network_state( config: &FetchConfig, cookie_store: SharedBrowserCookieStore, client_hint_preferences: SharedClientHintPreferences, + dictionaries: crate::compression_dictionary::SharedDictionaryStore, ) -> Self { let runtime_id = NEXT_FETCH_RUNTIME_ID .fetch_add(1, Ordering::Relaxed) @@ -401,6 +403,7 @@ impl FetchRuntimeOwner { config: config.clone(), cookie_store, client_hint_preferences, + dictionaries, request_rx, curl_runtime, curl_http, @@ -711,6 +714,7 @@ struct RuntimeOwner { config: FetchConfig, cookie_store: SharedBrowserCookieStore, client_hint_preferences: SharedClientHintPreferences, + dictionaries: crate::compression_dictionary::SharedDictionaryStore, request_rx: Receiver, curl_runtime: CurlMultiRuntime, curl_http: CurlHttpSender, @@ -721,6 +725,27 @@ struct RuntimeOwner { } impl RuntimeOwner { + fn prepare_request( + &self, + restarts: &SharedNavigationClientHintRestarts, + request: &Request, + url: &Url, + ) -> crate::client_hints::PreparedClientHintRequest { + let mut prepared = prepare_client_hint_request( + &self.client_hint_preferences, + restarts, + &self.config, + request, + url, + ); + prepared.request = crate::compression_dictionary::DictionaryStore::prepare_owned( + &self.dictionaries, + &self.config, + prepared.request, + url, + ); + prepared + } fn run(self) { #[cfg(test)] self.owner_started.store(true, Ordering::SeqCst); @@ -900,10 +925,8 @@ impl RuntimeOwner { None }; let cookie_header = cookie_header_from_report(request_cookie_report.as_ref()); - let prepared_request = prepare_client_hint_request( - &self.client_hint_preferences, + let prepared_request = self.prepare_request( &job.client_hint_navigation_restarts, - &self.config, &job.request, &job.current_url, ); @@ -1105,10 +1128,8 @@ impl RuntimeOwner { None }; let cookie_header = cookie_header_from_report(request_cookie_report.as_ref()); - let prepared_request = prepare_client_hint_request( - &self.client_hint_preferences, + let prepared_request = self.prepare_request( &job.client_hint_navigation_restarts, - &self.config, &job.request, &job.current_url, ); @@ -1349,10 +1370,8 @@ impl RuntimeOwner { None }; let cookie_header = cookie_header_from_report(request_cookie_report.as_ref()); - let prepared_request = prepare_client_hint_request( - &self.client_hint_preferences, + let prepared_request = self.prepare_request( &job.client_hint_navigation_restarts, - &self.config, &job.request, &job.current_url, ); @@ -1535,7 +1554,28 @@ impl RuntimeOwner { } } - fn finish_active_transfer(&self, state: &mut OwnerState, completion: RuntimeCurlCompletion) { + fn finish_active_transfer( + &self, + state: &mut OwnerState, + mut completion: RuntimeCurlCompletion, + ) { + let cancelled = match &completion.context { + ActiveTransferContext::Buffered(context) => context.job.cancel_handle.is_cancelled(), + ActiveTransferContext::Streaming(context) => context.job.cancel_handle.is_cancelled(), + ActiveTransferContext::StreamingRaw(context) => { + context.job.cancel_handle.is_cancelled() + } + }; + if let Some(easy) = completion.easy.as_mut() { + if let Some(error) = easy.get_mut().dictionary_error.take() { + completion.result = Err(anyhow!(error)); + } else if completion.result.is_ok() + && !self.shutdown_requested.load(Ordering::SeqCst) + && !cancelled + { + completion.result = easy.get_mut().finish_dictionary_response(); + } + } state.active_transfers = state.active_transfers.saturating_sub(1); match completion.context { ActiveTransferContext::Buffered(context) => { @@ -2612,6 +2652,7 @@ impl RuntimeOwner { return Ok(()); } + job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries)); complete_cached_streaming_html_job(job, cached, request_cookie_report); Ok(()) } @@ -2674,6 +2715,7 @@ impl RuntimeOwner { return Ok(()); } + job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries)); complete_cached_streaming_raw_job(job, cached, request_cookie_report); Ok(()) } @@ -2898,6 +2940,9 @@ pub(crate) struct FetchTransferHandler { network_observation_recorder: Option, upload_observer: Option, proxy_connect_response_recorder: ProxyConnectResponseRecorder, + dictionary_response: Option, + dictionary_decoding: Option, + dictionary_error: Option, } enum FetchResponseCollector { @@ -2943,6 +2988,9 @@ impl FetchTransferHandler { network_observation_recorder: None, upload_observer: None, proxy_connect_response_recorder: ProxyConnectResponseRecorder::default(), + dictionary_response: None, + dictionary_decoding: None, + dictionary_error: None, } } @@ -3001,6 +3049,31 @@ impl FetchTransferHandler { .begin_transfer(capture_proxy_connect_response); } + pub(crate) fn begin_dictionary_response(&mut self, request: &Request, url: &Url) { + self.dictionary_response = request.compression_dictionary_store.as_ref().map(|store| { + crate::compression_dictionary::DictionaryResponse::new( + Arc::clone(store), + request, + url.clone(), + ) + }); + self.dictionary_decoding = Some(crate::compression_dictionary::DictionaryDecoding::new( + request.compression_dictionary.clone(), + )); + self.dictionary_error = None; + } + + fn finish_dictionary_response(&mut self) -> Result<()> { + let Some(response) = self.dictionary_response.as_mut() else { + return Ok(()); + }; + if let Some(decoding) = self.dictionary_decoding.as_mut() { + decoding.finish(response.headers(), response.bodyless())?; + } + response.finish(); + Ok(()) + } + fn take_failed_proxy_connect_response( &mut self, connect_status: u32, @@ -3035,17 +3108,101 @@ impl Handler for FetchTransferHandler { if !data.is_empty() { self.mark_response_body_received(); } - if let Some(timing) = self.response_progress() { - timing.record_decoded_response_body_size(data.len()); + if self.dictionary_decoding.is_none() { + if let Some(timing) = self.response_progress() { + timing.record_decoded_response_body_size(data.len()); + } + let result = match &mut self.response { + FetchResponseCollector::Buffered(collector) => collector.write(data), + FetchResponseCollector::Streaming(collector) => collector.write(data), + FetchResponseCollector::StreamingRaw(collector) => collector.write(data), + }; + if result + .as_ref() + .is_ok_and(|accepted| *accepted == data.len()) + && let Some(policy) = self.dictionary_response.as_mut() + { + policy.decoded(data); + } + return result; } - match &mut self.response { - FetchResponseCollector::Buffered(collector) => collector.write(data), - FetchResponseCollector::Streaming(collector) => collector.write(data), - FetchResponseCollector::StreamingRaw(collector) => collector.write(data), + if self.dictionary_error.is_some() { + return Ok(0); + } + let timing = self.response_progress().cloned(); + let decoding = self + .dictionary_decoding + .as_mut() + .expect("content decoding selected"); + decoding.encoded_bytes = decoding.encoded_bytes.saturating_add(data.len() as u64); + if let Some(timing) = &timing { + timing.record_encoded_response_body_size(decoding.encoded_bytes); + } + if let Err(error) = decoding.initialize( + self.dictionary_response + .as_ref() + .map_or(&[], |response| response.headers()), + ) { + self.dictionary_error = Some(error.to_string()); + return Ok(0); + } + let policy = &mut self.dictionary_response; + let response = &mut self.response; + let mut output = |bytes: &[u8]| -> Result<()> { + if let Some(timing) = &timing { + timing.record_decoded_response_body_size(bytes.len()); + } + let accepted = match response { + FetchResponseCollector::Buffered(collector) => collector.write(bytes), + FetchResponseCollector::Streaming(collector) => collector.write(bytes), + FetchResponseCollector::StreamingRaw(collector) => collector.write(bytes), + } + .map_err(|_| anyhow!("response collector rejected decoded bytes"))?; + anyhow::ensure!( + accepted == bytes.len(), + "response collector rejected decoded bytes" + ); + if let Some(policy) = policy.as_mut() { + policy.decoded(bytes); + } + Ok(()) + }; + let result = self + .dictionary_decoding + .as_mut() + .expect("dictionary decoding selected") + .push(data, &mut output); + match result { + Ok(()) => Ok(data.len()), + Err(error) => { + self.dictionary_error = Some(error.to_string()); + Ok(0) + } } } fn header(&mut self, data: &[u8]) -> bool { + if crate::response::parse_http_response_status_line(data) + .is_some_and(|line| line.status >= 200) + && let Some(decoding) = self.dictionary_decoding.as_mut() + { + decoding.reset(); + self.dictionary_error = None; + } + if let Some(response) = self.dictionary_response.as_mut() { + response.header(data); + } + if data.iter().all(|byte| matches!(byte, b'\r' | b'\n')) + && let Some(response) = self.dictionary_response.as_ref() + && response.headers_finalized() + && !response.bodyless() + && let Some(decoding) = self.dictionary_decoding.as_mut() + && decoding.uses_dictionary() + && !response.readable() + { + self.dictionary_error = + Some("dictionary response failed its CORS readability check".to_owned()); + } if crate::response::parse_http_response_status_line(data) .is_some_and(|line| line.status >= 200) && let Some(timing) = self.response_progress() @@ -3251,7 +3408,7 @@ fn complete_cached_streaming_html_job( final_url, status, headers, - mut body, + body, .. } = cached; let final_url = match Url::parse(&final_url) { @@ -3270,9 +3427,29 @@ fn complete_cached_streaming_html_job( } }; + let capture = job + .request + .compression_dictionary_store + .as_ref() + .and_then(|store| { + crate::compression_dictionary::DictionaryResponse::cached( + Arc::clone(store), + &job.request, + &final_url, + status, + &headers, + metadata.expires_at_unix_ms, + ) + }); + let mut body = crate::compression_dictionary::DictionaryCachedReader::new( + body, + capture, + job.cancel_handle.clone(), + ); let response_timing = job .response_progress .mark_cached_response_available(body_sizes); + if let Some(started_tx) = job.started_tx { let _ = started_tx.send(Ok(StreamingHtmlResponseStart { status_text: metadata.status_text, @@ -3381,7 +3558,7 @@ fn complete_cached_streaming_raw_job( final_url, status, headers, - mut body, + body, .. } = cached; let final_url = match Url::parse(&final_url) { @@ -3400,6 +3577,25 @@ fn complete_cached_streaming_raw_job( } }; + let capture = job + .request + .compression_dictionary_store + .as_ref() + .and_then(|store| { + crate::compression_dictionary::DictionaryResponse::cached( + Arc::clone(store), + &job.request, + &final_url, + status, + &headers, + metadata.expires_at_unix_ms, + ) + }); + let mut body = crate::compression_dictionary::DictionaryCachedReader::new( + body, + capture, + job.cancel_handle.clone(), + ); let response_timing = job .response_progress .mark_cached_response_available(body_sizes); @@ -3474,6 +3670,7 @@ fn request_fetch_load_priority(request: &Request) -> crate::ResourceLoadPriority Some( crate::BrowserRequestMetadata::Audio | crate::BrowserRequestMetadata::Beacon + | crate::BrowserRequestMetadata::Dictionary | crate::BrowserRequestMetadata::Font | crate::BrowserRequestMetadata::Image | crate::BrowserRequestMetadata::Ping @@ -4430,7 +4627,8 @@ mod tests { .with_resource_type(crate::RequestResourceType::LinkPrefetch); let dictionary = Request::new("GET", "https://example.test/dict.bin", None, Vec::new()) .expect("test request should parse") - .with_resource_type(crate::RequestResourceType::Dictionary); + .with_resource_type(crate::RequestResourceType::Dictionary) + .with_browser_request_metadata(crate::BrowserRequestMetadata::Dictionary); assert_eq!( request_fetch_load_priority(&stylesheet), diff --git a/moli-fetch/src/tests/compression_dictionary.rs b/moli-fetch/src/tests/compression_dictionary.rs new file mode 100644 index 00000000000..da323b810d4 --- /dev/null +++ b/moli-fetch/src/tests/compression_dictionary.rs @@ -0,0 +1,434 @@ +use super::*; + +const DICTIONARY: &[u8] = b"This is a test dictionary.\n"; +const PAYLOAD: &[u8] = b"This is compressed test data using a test dictionary"; +const HASH: &str = ":U5abz16WDg7b8KS93msLPpOB4Vbef1uRzoORYkJw9BY=:"; +const DCB: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcb"); +const DCZ: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcz"); + +fn reply(headers: &[(&str, &str)], body: &[u8], chunked: bool) -> Vec { + let mut head = + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain\r\n".to_owned(); + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + if chunked { + head.push_str("Transfer-Encoding: chunked\r\n\r\n"); + } else { + head.push_str(&format!("Content-Length: {}\r\n\r\n", body.len())); + } + let mut response = head.into_bytes(); + if chunked { + for byte in body { + response.extend_from_slice(&[b'1', b'\r', b'\n', *byte, b'\r', b'\n']); + } + response.extend_from_slice(b"0\r\nX-Trailer: retained\r\n\r\n"); + } else { + response.extend_from_slice(body); + } + response +} + +async fn server( + responses: Vec>, +) -> Result<(String, tokio::task::JoinHandle>>>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}/data", listener.local_addr()?); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = + tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??; + let mut request = Vec::new(); + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let mut buffer = [0; 4096]; + let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer)) + .await??; + anyhow::ensure!(count != 0 && request.len() < 65536, "incomplete request"); + request.extend_from_slice(&buffer[..count]); + } + requests.push(request); + // Error responses may make the client close the socket early. + let _ = stream.write_all(&response).await; + let _ = stream.shutdown().await; + } + Ok(requests) + }); + Ok((url, task)) +} + +fn header<'a>(request: &'a [u8], name: &str) -> Option<&'a str> { + std::str::from_utf8(request) + .unwrap() + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(field, _)| field.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.trim()) +} + +async fn fetch(client: &FetchClientHandle, url: &str, mode: &str) -> Result> { + let request = Request::new("GET", url, None, crate::RequestHeaders::default())?; + let (_, body) = match mode { + "buffered" => client + .fetch(request.with_auth(crate::RequestAuth { + target: crate::RequestAuthTarget::Server, + scheme: crate::RequestAuthScheme::Digest, + username: "user".to_owned(), + password: "password".to_owned(), + })) + .await? + .into_body(), + "html" => client.fetch_html_stream(request).await?.into_body(), + "raw" => client + .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new()) + .await? + .into_body(), + _ => unreachable!(), + }; + body.into_materialized_bytes().await +} + +#[tokio::test] +async fn compression_dictionary_registration_and_decoding_cover_every_transport() -> Result<()> { + for mode in ["raw", "html", "buffered"] { + for chunked in [false, true] { + let coding = [ + ("dcb", DCB, PAYLOAD), + ("dcz", DCZ, PAYLOAD), + ( + "gzip", + include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.gzip") + .as_slice(), + DICTIONARY, + ), + ( + "deflate", + include_bytes!( + "../../tests/fixtures/compression-dictionary/dictionary.deflate" + ) + .as_slice(), + DICTIONARY, + ), + ( + "deflate", + include_bytes!( + "../../tests/fixtures/compression-dictionary/dictionary.raw-deflate" + ) + .as_slice(), + DICTIONARY, + ), + ( + "br", + include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.br") + .as_slice(), + DICTIONARY, + ), + ( + "zstd", + include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.zstd") + .as_slice(), + DICTIONARY, + ), + ]; + let mut responses = vec![reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\", id=\"first\""), + ], + DICTIONARY, + chunked, + )]; + responses.extend( + coding + .iter() + .map(|(name, body, _)| reply(&[("Content-Encoding", name)], body, chunked)), + ); + let (url, task) = server(responses).await?; + let client = + FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + assert_eq!( + fetch(&client, &url, mode).await?, + DICTIONARY, + "{mode}/{chunked}" + ); + for (name, _, expected) in coding { + assert_eq!( + fetch(&client, &url, mode).await?, + expected, + "{mode}/{chunked}/{name}" + ); + } + let requests = task.await??; + assert_eq!(header(&requests[0], "available-dictionary"), None); + assert!( + !header(&requests[0], "accept-encoding") + .unwrap() + .contains("dcb") + ); + for request in &requests[1..] { + assert_eq!(header(request, "available-dictionary"), Some(HASH)); + assert_eq!(header(request, "dictionary-id"), Some("\"first\"")); + let encoding = header(request, "accept-encoding").unwrap(); + assert!(encoding.contains("dcb") && encoding.contains("dcz")); + } + } + } + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_clear_site_data_observes_cache_and_cookie_directives() -> Result<()> +{ + let registered = reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\""), + ], + DICTIONARY, + false, + ); + for directive in ["storage", "cache", "cookies", "*"] { + let field = format!("\"{directive}\""); + let (url, task) = server(vec![ + registered.clone(), + reply(&[("Clear-Site-Data", &field)], b"OK", false), + reply(&[], b"OK", false), + ]) + .await?; + let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + for _ in 0..3 { + fetch(&client, &url, "raw").await?; + } + let requests = task.await??; + assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH)); + assert_eq!( + header(&requests[2], "available-dictionary"), + if directive == "storage" { + Some(HASH) + } else { + None + } + ); + } + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_bad_hash_and_truncated_streams_do_not_register() -> Result<()> { + let registered = reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\", id=\"good\""), + ], + DICTIONARY, + false, + ); + for (coding, original) in [("dcb", DCB), ("dcz", DCZ)] { + let mut bad_hash = original.to_vec(); + bad_hash[12] ^= 1; + for body in [bad_hash.as_slice(), &original[..original.len() - 1]] { + let invalid = reply( + &[ + ("Content-Encoding", coding), + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\", id=\"bad\""), + ], + body, + true, + ); + let (url, task) = + server(vec![registered.clone(), invalid, reply(&[], b"OK", false)]).await?; + let client = + FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + fetch(&client, &url, "raw").await?; + assert!(fetch(&client, &url, "raw").await.is_err(), "{coding}"); + fetch(&client, &url, "raw").await?; + assert_eq!(header(&task.await??[2], "dictionary-id"), Some("\"good\"")); + } + } + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_request_snapshot_survives_later_registration() -> Result<()> { + let (url, task) = server(vec![ + reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\""), + ], + DICTIONARY, + false, + ), + reply(&[], b"OK", false), + reply(&[], b"OK", false), + reply(&[], b"OK", false), + ]) + .await?; + let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + let mut early = Request::get(&url)?; + client.prepare_request(&mut early); + fetch(&client, &url, "raw").await?; + assert!( + !client + .cache_request_headers(&early) + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("available-dictionary")) + ); + client.fetch_raw(early.clone()).await?; + fetch(&client, &url, "raw").await?; + client + .fetch_raw(early.with_network_partition_key(Some("https://other.test".to_owned()))) + .await?; + let requests = task.await??; + assert_eq!(header(&requests[1], "available-dictionary"), None); + assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH)); + assert_eq!(header(&requests[3], "available-dictionary"), None); + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_cached_body_registration_keeps_original_freshness() -> Result<()> { + let (url, task) = server(vec![ + reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\""), + ], + DICTIONARY, + false, + ), + reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false), + reply(&[], b"OK", false), + reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false), + reply(&[], b"OK", false), + ]) + .await?; + let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + let request = Request::get(&url)?; + let cached = client.fetch_raw(request.clone()).await?; + fetch(&client, &url, "raw").await?; + client.observe_cached_response(&request, &cached, moli_http_cache::unix_now_ms() + 600_000); + fetch(&client, &url, "raw").await?; + fetch(&client, &url, "raw").await?; + client.observe_cached_response( + &request, + &cached, + moli_http_cache::unix_now_ms().saturating_sub(1), + ); + fetch(&client, &url, "raw").await?; + let requests = task.await??; + assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH)); + assert_eq!(header(&requests[4], "available-dictionary"), None); + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_disk_cache_hit_registers_decoded_dictionary() -> Result<()> { + let (url, task) = server(vec![ + reply( + &[ + ("Cache-Control", "max-age=600"), + ("Use-As-Dictionary", "match=\"/*\""), + ], + DICTIONARY, + false, + ), + reply(&[], b"probe", false), + ]) + .await?; + let cache_dir = unique_test_cache_dir(); + let mut config = FetchConfig::default(); + config.set_http_cache_dir(Some(cache_dir.display().to_string())); + let request = Request::get(&url)?; + { + let client = FetchClient::new(&config, new_shared_browser_cookie_store()); + assert_eq!( + client.fetch_raw(request.clone()).await?.body_bytes(), + DICTIONARY + ); + } + let client = FetchClient::new(&config, new_shared_browser_cookie_store()); + let hit = client.fetch_raw(request).await?; + assert_eq!(hit.cache_state, crate::ResponseCacheState::Local); + let probe = format!("{url}?probe"); + client.fetch_raw(Request::get(&probe)?).await?; + let requests = task.await??; + assert_eq!(requests.len(), 2); + assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH)); + fs::remove_dir_all(cache_dir)?; + Ok(()) +} + +#[tokio::test] +async fn compression_dictionary_registration_requires_readable_and_fresh_responses() -> Result<()> { + let cases = [ + ("match=\"/*\"", "max-age=600", "0", true), + ("match=42", "max-age=600", "0", false), + ("match=\"/:id(d.*)\"", "max-age=600", "0", false), + ("match=\"/*\", type=other", "max-age=600", "0", false), + ( + "match=\"/*\", match-dest=\"script\"", + "max-age=600", + "0", + false, + ), + ("match=\"/*\"", "max-age=600", "601", false), + ("match=\"/*\"", "no-store, max-age=600", "0", false), + ]; + for (options, cache, age, expected) in cases { + let (url, task) = server(vec![ + reply( + &[ + ("Use-As-Dictionary", options), + ("Cache-Control", cache), + ("Age", age), + ], + DICTIONARY, + false, + ), + reply(&[], b"OK", false), + ]) + .await?; + let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + fetch(&client, &url, "raw").await?; + fetch(&client, &url, "raw").await?; + assert_eq!( + header(&task.await??[1], "available-dictionary"), + expected.then_some(HASH), + "{options}/{cache}/{age}" + ); + } + for (mode, cors, expected) in [ + (crate::RequestMode::NoCors, true, false), + (crate::RequestMode::Cors, false, false), + (crate::RequestMode::Cors, true, true), + ] { + let mut headers = vec![ + ("Use-As-Dictionary", "match=\"/*\""), + ("Cache-Control", "max-age=600"), + ]; + if cors { + headers.push(("Access-Control-Allow-Origin", "*")); + } + let (url, task) = server(vec![ + reply(&headers, DICTIONARY, false), + reply(&[("Access-Control-Allow-Origin", "*")], b"OK", false), + ]) + .await?; + let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store()); + let mut request = Request::get(&url)? + .with_request_origin(moli_url::WebOrigin::from_serialized("https://frame.test")); + request.request_mode = mode; + request.credentials_mode = crate::RequestCredentialsMode::Omit; + let _ = client.fetch_raw(request.clone()).await; + request.request_mode = crate::RequestMode::Cors; + client.fetch_raw(request).await?; + assert_eq!( + header(&task.await??[1], "available-dictionary"), + expected.then_some(HASH), + "{mode:?}/{cors}" + ); + } + Ok(()) +} diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs index b865eded9a0..295bf316553 100644 --- a/moli-fetch/src/tests/mod.rs +++ b/moli-fetch/src/tests/mod.rs @@ -3,6 +3,7 @@ mod cache_request_controls_modes; mod cache_request_directives; mod cache_request_modes; mod cache_vary_encoding; +mod compression_dictionary; mod cookie_context; mod cookie_store; mod corp; diff --git a/moli-fetch/tests/fixtures/compression-dictionary/.gitattributes b/moli-fetch/tests/fixtures/compression-dictionary/.gitattributes new file mode 100644 index 00000000000..cf3b78c3783 --- /dev/null +++ b/moli-fetch/tests/fixtures/compression-dictionary/.gitattributes @@ -0,0 +1,7 @@ +*.br binary +*.dcb binary +*.dcz binary +*.deflate binary +*.gzip binary +*.raw-deflate binary +*.zstd binary diff --git a/moli-fetch/tests/fixtures/compression-dictionary/README.md b/moli-fetch/tests/fixtures/compression-dictionary/README.md new file mode 100644 index 00000000000..914c41fee7a --- /dev/null +++ b/moli-fetch/tests/fixtures/compression-dictionary/README.md @@ -0,0 +1,5 @@ +# Compression dictionary fixtures + +The `dictionary.*` and `payload.*` Brotli, gzip, Zstandard, dcb and dcz bytes are copied unchanged from the upstream WPT `fetch/compression-dictionary/resources/register-dictionary.py` and `compressed-data.py`, snapshot `db95fafd1fcef8428805e41eb5705d444e8c67ce`. The two deflate fixtures encode the same dictionary with Python zlib, with and without the zlib wrapper. + +Dictionary bytes are `This is a test dictionary.\n`; payload bytes are `This is compressed test data using a test dictionary`. The dcb/dcz headers and dictionary SHA-256 remain part of each fixture. diff --git a/moli-fetch/tests/fixtures/compression-dictionary/dictionary.br b/moli-fetch/tests/fixtures/compression-dictionary/dictionary.br new file mode 100644 index 0000000000000000000000000000000000000000..5b5ba67550021644ccb2d0301a2f43e0de96150f GIT binary patch literal 29 lcmZ2@f#E>@l9JD1TbVp0=4#D8?IhB;T;x`#=H%)VoB+@I4MP9` literal 0 HcmV?d00001 diff --git a/moli-fetch/tests/fixtures/compression-dictionary/dictionary.dcb b/moli-fetch/tests/fixtures/compression-dictionary/dictionary.dcb new file mode 100644 index 0000000000000000000000000000000000000000..0ff294c9f46654a1d0adfb50aec54928c95971e1 GIT binary patch literal 47 zcmV+~0MP&cL_6bzoZr~ literal 0 HcmV?d00001 diff --git a/moli-fetch/tests/fixtures/compression-dictionary/dictionary.deflate b/moli-fetch/tests/fixtures/compression-dictionary/dictionary.deflate new file mode 100644 index 0000000000000000000000000000000000000000..e602abcb363331994a1777254f1b528ac2a0095a GIT binary patch literal 33 rcmV++0N($2oD0dwELH%bM1_*n;u3|F%;b{H{Jg}XN-iwFP^F-K?s0}IK>ELH%bM1_*n;u3|F%;b{H{Jg}XNNlGNf7g_O+XlFa“�áVÞ[‘΃‘bBpô¡˜€b¤Lß„Œ®ÂÊ`"n�É·ÃD޼àìÁî43> \ No newline at end of file diff --git a/moli-fetch/tests/fixtures/compression-dictionary/payload.dcz b/moli-fetch/tests/fixtures/compression-dictionary/payload.dcz new file mode 100644 index 0000000000000000000000000000000000000000..43c0cfd8038ed09936db30548b6e669502b17752 GIT binary patch literal 83 zcmV-Z0IdIBDoq$50000}mYdIBmJSZv@T9%oYYRS;f#Fu(e_N5xgOOrFaP$@^wJ-f7 pH1z-gm}75kaB^jHb7f>&P^1QDPBL)Bf literal 0 HcmV?d00001 diff --git a/moli-renderer-v8/src/network/backend/memory_cache.rs b/moli-renderer-v8/src/network/backend/memory_cache.rs index 25902008c2a..1f90d7f2734 100644 --- a/moli-renderer-v8/src/network/backend/memory_cache.rs +++ b/moli-renderer-v8/src/network/backend/memory_cache.rs @@ -13,8 +13,8 @@ use moli_fetch::{ BrowserRequestMetadata, RawResponse, Request, RequestHeaders, RequestResourceType, Response, }; use moli_http_cache::{ - HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date, - response_vary_header_names, unix_now_ms, validation_headers_from_headers, + HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date, unix_now_ms, + validation_headers_from_headers, }; use parking_lot::Mutex; use url::Url; @@ -81,7 +81,10 @@ pub(in crate::network) struct RawSubresourceCacheKey { } pub(in crate::network) enum RawSubresourceCacheLookup { - Local(Box), + Local { + response: Box, + expires_at_unix_ms: u64, + }, Revalidate(Box), } @@ -724,9 +727,13 @@ impl SharedMemoryResourceCache { return Some(RawSubresourceCacheLookup::Revalidate(variant.response)); } let mut hit = variants[index].response.clone(); + let expires_at_unix_ms = variants[index].expires_at_unix_ms; hit.final_url.set_fragment(request.url.fragment()); self.insert_raw_subresource_variants(key.clone(), variants); - Some(RawSubresourceCacheLookup::Local(hit)) + Some(RawSubresourceCacheLookup::Local { + response: hit, + expires_at_unix_ms, + }) } pub(in crate::network) fn remove_raw_subresource( @@ -750,7 +757,9 @@ impl SharedMemoryResourceCache { if response.body_bytes().len() > self.resource_body_bytes_limit { return; } - let Some(vary_names) = response_vary_header_names(&response.headers) else { + let Some(vary_names) = + moli_fetch::cache_vary_header_names(&response.final_url, &response.headers) + else { return; }; let variant = RawSubresourceVariant { @@ -1089,7 +1098,7 @@ pub(in crate::network) fn raw_subresource_memory_cache_expiry( // Match ordinary fields against the same encoded builder as the native // request. Mutable transport-only fields need exact sent-value tracking // before a renderer representation can safely vary on them. - if !response_vary_header_names(&response.headers)? + if !moli_fetch::cache_vary_header_names(&response.final_url, &response.headers)? .iter() .all(|name| { name.bytes() diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rs index c2236144a2d..3c78f5d218b 100644 --- a/moli-renderer-v8/src/network/request_client.rs +++ b/moli-renderer-v8/src/network/request_client.rs @@ -1019,7 +1019,15 @@ impl ResourceRequestClient { }); let mut network_request = request.clone(); let cached = match lookup { - Some(RawSubresourceCacheLookup::Local(cached)) => { + Some(RawSubresourceCacheLookup::Local { + response: cached, + expires_at_unix_ms, + }) => { + self.resource_runtime.client().observe_cached_response( + &request, + &cached, + expires_at_unix_ms, + ); if let Some(sizes) = cached.body_sizes() { cancel_handle.set_response_body_sizes(sizes); } diff --git a/moli-renderer-v8/src/stylesheet_runtime/connected.rs b/moli-renderer-v8/src/stylesheet_runtime/connected.rs index 67445931518..c256a08c753 100644 --- a/moli-renderer-v8/src/stylesheet_runtime/connected.rs +++ b/moli-renderer-v8/src/stylesheet_runtime/connected.rs @@ -2490,10 +2490,8 @@ fn preload_like_link_readiness_fetch_options( }; let credentials_mode = if resource_type == SubresourceResourceType::Dictionary { // Blink sets compression dictionary requests to CORS mode with - // credentials omitted and no referrer. Request::new already uses CORS - // for subresources; Moli does not yet model per-request referrer - // policy for this loader path, so credentials and priority are the - // observable pieces preserved here. + // credentials omitted and no referrer. Browser metadata and referrer + // suppression are applied by connected_link_readiness_request. RequestCredentialsMode::Omit } else if cross_origin.is_some() { link_crossorigin_credentials_mode(cross_origin) @@ -2654,6 +2652,11 @@ fn connected_link_readiness_request( if let Some(resource_type) = options.request_resource_type { request = request.with_resource_type(resource_type); } + if options.resource_type == SubresourceResourceType::Dictionary { + request = request + .with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Dictionary) + .without_inferred_referrer(); + } if let Some(metadata) = options.script_fetch_metadata.as_ref() { request = request.with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Script); request = request.with_script_fetch_metadata(moli_fetch::ScriptFetchRequestMetadata { @@ -5297,6 +5300,30 @@ mod tests { Some(RequestResourceType::Dictionary) ); assert_eq!(options.credentials_mode, RequestCredentialsMode::Omit); + let document_url = document.final_url().expect("HTML document URL"); + let url = Url::parse("https://example.test/dict.bin").unwrap(); + let request = connected_link_readiness_request( + document_url, + &moli_url::WebOrigin::from_url(document_url), + &url, + &options, + ); + let headers = moli_fetch::outgoing_request_headers(&Default::default(), &request, None); + assert_eq!( + request.browser_request_metadata(), + Some(moli_fetch::BrowserRequestMetadata::Dictionary) + ); + assert!(headers.iter().any(|(name, value)| name.eq_ignore_ascii_case("sec-fetch-mode") && value == "cors")); + assert!( + headers.iter().any( + |(name, value)| name.eq_ignore_ascii_case("sec-fetch-dest") && value == "empty" + ) + ); + assert!( + !headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("referer")) + ); assert!(!options.link_preload); assert!(options.script_fetch_metadata.is_none()); } diff --git a/moli-types/moli-fetch-types/src/header_policy.rs b/moli-types/moli-fetch-types/src/header_policy.rs index 955600be1af..f0c9d6b3680 100644 --- a/moli-types/moli-fetch-types/src/header_policy.rs +++ b/moli-types/moli-fetch-types/src/header_policy.rs @@ -8,6 +8,8 @@ pub fn is_forbidden_request_header_name(name: &str) -> bool { name.as_str(), "accept-charset" | "accept-encoding" + | "available-dictionary" + | "dictionary-id" | "access-control-request-headers" | "access-control-request-method" | "connection"