feat(fetch): support partitioned compression dictionaries

This commit is contained in:
ldm0
2026-11-20 22:47:54 +08:00
parent a9da276061
commit bf47da3d8d
30 changed files with 1974 additions and 48 deletions
Generated
+51
View File
@@ -2964,10 +2964,13 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bytes",
"base64 0.22.1",
"brotlic-sys",
"cidr",
"crossbeam-channel",
"curl",
"curl-sys",
"flate2",
"h2",
"http",
"moli-browser-profile",
@@ -2986,8 +2989,11 @@ dependencies = [
"parking_lot",
"percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rcgen",
"regex",
"serde",
"serde_json",
"sfv",
"sha2",
"strum",
"thiserror",
"tokio",
@@ -2996,7 +3002,9 @@ dependencies = [
"tokio-util",
"tracing",
"url",
"urlpattern",
"wildcard",
"zstd-sys",
]
[[package]]
@@ -5028,6 +5036,26 @@ dependencies = [
"bitflags",
]
[[package]]
name = "ref-cast"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
dependencies = [
"ref-cast-impl",
]
[[package]]
name = "ref-cast-impl"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "regex"
version = "1.12.3"
@@ -5423,6 +5451,17 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "sfv"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "890ef0e5532daf21fdaa3dbaf991417187f3a87ab69231fe69be942495746480"
dependencies = [
"base64 0.22.1",
"indexmap",
"ref-cast",
]
[[package]]
name = "sha1"
version = "0.10.6"
@@ -6569,6 +6608,18 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "urlpattern"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df16f50ef4cc145211879a3867ba757076b25dfee812040dcb0658bd9ae7904b"
dependencies = [
"icu_properties",
"regex",
"serde",
"url",
]
[[package]]
name = "usvg"
version = "0.48.1"
+8
View File
@@ -17,6 +17,9 @@ cidr = "0.3.2"
crossbeam-channel = "0.5.15"
curl = { version = "0.4.51", default-features = false, features = ["brotli", "browser-response-headers", "http2", "poll_7_68_0", "ssl", "static-curl"] }
curl-sys = { version = "0.4.91", default-features = false }
base64 = "0.22.1"
brotlic-sys = "0.2.2"
flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] }
http = "1"
moli-browser-profile = { path = "../moli-browser-profile" }
moli-cookie-jar = { path = "../moli-cookie-jar" }
@@ -31,14 +34,19 @@ moli-web-bot-auth = { path = "../moli-web-bot-auth" }
moli-web-mime = { path = "../moli-web-mime" }
parking_lot = "0.12"
percent-encoding = "2"
regex = "1"
serde = { version = "1.0.228", features = ["derive"] }
sfv = "0.15.0"
sha2 = "0.11.0"
strum = { version = "0.28.0", features = ["derive"] }
thiserror = "2.0.17"
tokio = { version = "1.51.0", features = ["rt", "sync"] }
tokio-util = { version = "0.7.18", features = ["rt"] }
tracing = "0.1.41"
url = "2.5.7"
urlpattern = "0.6.0"
wildcard = "0.3.0"
zstd-sys = { version = "2.1.2", default-features = false, features = ["vendored"] }
[dev-dependencies]
serde_json = "1"
+4 -2
View File
@@ -1,6 +1,6 @@
use std::str;
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader, response_vary_header_names};
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader};
use url::Url;
use crate::{FetchConfig, Request};
@@ -11,6 +11,8 @@ const SUPPORTED_CACHE_REQUEST_HEADERS: &[&str] = &[
"accept",
"accept-encoding",
"accept-language",
"available-dictionary",
"dictionary-id",
"referer",
"sec-ch-ua",
"sec-ch-ua-arch",
@@ -38,7 +40,7 @@ pub(super) fn vary_headers_for_response(
response_headers: &[(String, Vec<u8>)],
) -> Option<Vec<HttpCacheVaryHeader>> {
let mut out = Vec::new();
for normalized_name in response_vary_header_names(response_headers)? {
for normalized_name in crate::cache_vary_header_names(request_url, response_headers)? {
if !supported_vary_header(&normalized_name) {
return None;
}
+12 -1
View File
@@ -282,10 +282,17 @@ pub(crate) fn outgoing_request_header_bytes_for_url(
let value = if header_present(&outgoing, "range") {
"identity".to_owned()
} else {
crate::content_coding::default_accept_encoding()
if request.compression_dictionary.is_some() {
crate::content_coding::dictionary_accept_encoding()
} else {
crate::content_coding::default_accept_encoding()
}
};
outgoing.push(("Accept-Encoding".to_owned(), value.into_bytes()));
}
if let Some(dictionary) = &request.compression_dictionary {
crate::compression_dictionary::append_request_headers(&mut outgoing, dictionary);
}
if request.has_browser_identity_override() {
append_encoded_header_if_missing(
@@ -649,6 +656,10 @@ pub(crate) fn configure_easy(
.context("failed to disable curl redirect following")?;
easy.accept_encoding("")
.context("failed to enable curl response decompression")?;
easy.http_content_decoding(false)
.context("failed to configure response content decoding")?;
easy.get_mut()
.begin_dictionary_response(request, request_url);
// TLS client identities are Fetch credentials. Check each redirect target;
// curl also uses the client certificate configuration to match reusable
+45 -2
View File
@@ -20,6 +20,7 @@ pub struct FetchClientHandle {
config: FetchConfig,
cookie_store: SharedBrowserCookieStore,
runtime: FetchRuntimeHandle,
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
}
/// Unique structured-concurrency owner for one fetch client.
@@ -39,6 +40,17 @@ impl FetchClientHandle {
/// network admission, after the caller has selected the resource kind.
pub fn prepare_request(&self, request: &mut Request) {
request.prepare_accept_header(&self.config);
let url = request.url.clone();
if let std::borrow::Cow::Owned(prepared) =
crate::compression_dictionary::DictionaryStore::prepare(
&self.dictionaries,
&self.config,
request,
&url,
)
{
*request = prepared;
}
}
/// Creates native WebSockets on this client's HTTP owner and Multi. The
@@ -207,14 +219,42 @@ impl FetchClientHandle {
/// Cookie-store and other mutable transport-only fields are not included;
/// renderer caches must not admit Vary on those fields from this view.
pub fn cache_request_headers(&self, request: &Request) -> crate::RequestHeaders {
crate::blocking::outgoing_request_header_bytes_for_url(
let prepared = crate::compression_dictionary::DictionaryStore::prepare(
&self.dictionaries,
&self.config,
request,
&request.url,
);
crate::blocking::outgoing_request_header_bytes_for_url(
&self.config,
&prepared,
&request.url,
None,
)
}
/// A complete cached representation can register its dictionary again.
/// Its retained freshness deadline must not be extended by this read.
pub fn observe_cached_response(
&self,
request: &Request,
response: &RawResponse,
fresh_until_unix_ms: u64,
) {
if response
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
{
crate::compression_dictionary::DictionaryResponse::from_cached(
Arc::clone(&self.dictionaries),
request,
response,
fresh_until_unix_ms,
);
}
}
pub fn matches_config(&self, config: &FetchConfig) -> bool {
&self.config == config
}
@@ -274,15 +314,18 @@ impl FetchClient {
}
let client_hint_preferences =
Arc::new(parking_lot::Mutex::new(ClientHintPreferences::default()));
let runtime_owner = FetchRuntimeOwner::new_with_client_hint_preferences(
let dictionaries = Arc::default();
let runtime_owner = FetchRuntimeOwner::new_with_network_state(
config,
Arc::clone(&cookie_store),
Arc::clone(&client_hint_preferences),
Arc::clone(&dictionaries),
);
let handle = FetchClientHandle {
config: config.clone(),
runtime: runtime_owner.handle(),
cookie_store,
dictionaries,
};
Self {
handle,
@@ -0,0 +1,448 @@
//! Decode response content with the linked Brotli/zlib/Zstandard libraries.
//! The same stack validates completion and counts wire bytes for ordinary and
//! dictionary-compressed responses, independently of libcurl's decoder build.
use std::{
ffi::c_void,
ptr::{self, NonNull},
sync::Arc,
};
use anyhow::{Result, anyhow, bail, ensure};
use flate2::{Decompress, FlushDecompress, Status};
use super::Dictionary;
const OUTPUT_SIZE: usize = 16 * 1024;
type Output<'a> = dyn FnMut(&[u8]) -> Result<()> + 'a;
pub(crate) struct DictionaryDecoding {
dictionary: Option<Arc<Dictionary>>,
stages: Option<Vec<Stage>>,
pub(crate) encoded_bytes: u64,
}
impl DictionaryDecoding {
pub(crate) fn new(dictionary: Option<Arc<Dictionary>>) -> Self {
Self {
dictionary,
stages: None,
encoded_bytes: 0,
}
}
pub(crate) fn reset(&mut self) {
self.stages = None;
self.encoded_bytes = 0;
}
pub(crate) fn uses_dictionary(&self) -> bool {
self.dictionary.is_some()
}
pub(crate) fn initialize(&mut self, headers: &[(String, Vec<u8>)]) -> Result<()> {
if self.stages.is_some() {
return Ok(());
}
let mut names = Vec::new();
for (_, value) in headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("content-encoding"))
{
let value = crate::decode_header_value(value);
names.extend(
crate::split_http_header_list(&value)
.filter(|value| !value.is_empty())
.map(str::to_ascii_lowercase),
);
}
ensure!(names.len() <= 5, "too many content encodings");
let mut stages = Vec::new();
for name in names.into_iter().rev() {
let stage = match name.as_str() {
"identity" => continue,
"gzip" | "x-gzip" => Stage::Flate(Flate::gzip()),
"deflate" => Stage::Flate(Flate::deflate()),
"br" => Stage::Brotli(Brotli::new(None)?),
"zstd" => Stage::Zstd(Zstd::new(None)?),
"dcb" | "dcz" => Stage::Dictionary {
brotli: name == "dcb",
dictionary: Arc::clone(self.dictionary.as_ref().ok_or_else(|| {
anyhow!("dictionary compression without a negotiated dictionary")
})?),
prefix: Vec::new(),
decoder: None,
},
_ => bail!("unsupported content encoding {name}"),
};
stages.push(stage);
}
self.stages = Some(stages);
Ok(())
}
pub(crate) fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
ensure!(
self.stages.is_some(),
"compressed response body arrived before headers"
);
feed(
self.stages.as_mut().expect("initialized decoding stack"),
input,
output,
)
}
pub(crate) fn finish(&mut self, headers: &[(String, Vec<u8>)], bodyless: bool) -> Result<()> {
if bodyless {
return Ok(());
}
// An empty payload never enters a content decoder. HEAD and null-body
// responses likewise do not validate an unused coding declaration.
if self.encoded_bytes == 0 {
return Ok(());
}
self.initialize(headers)?;
for stage in self.stages.as_ref().expect("initialized decoding stack") {
ensure!(stage.finished(), "truncated compressed response");
}
Ok(())
}
}
fn feed(stages: &mut [Stage], input: &[u8], output: &mut Output<'_>) -> Result<()> {
let Some((stage, remaining)) = stages.split_first_mut() else {
return output(input);
};
stage.push(input, &mut |bytes| feed(remaining, bytes, output))
}
enum Stage {
Flate(Flate),
Brotli(Brotli),
Zstd(Zstd),
Dictionary {
brotli: bool,
dictionary: Arc<Dictionary>,
prefix: Vec<u8>,
decoder: Option<Box<Stage>>,
},
}
impl Stage {
fn push(&mut self, mut input: &[u8], output: &mut Output<'_>) -> Result<()> {
match self {
Self::Flate(decoder) => decoder.push(input, output),
Self::Brotli(decoder) => decoder.push(input, output),
Self::Zstd(decoder) => decoder.push(input, output),
Self::Dictionary {
brotli,
dictionary,
prefix,
decoder,
} => {
if decoder.is_none() {
let magic: &[u8] = if *brotli {
b"\xffDCB"
} else {
b"\x5e\x2a\x4d\x18\x20\0\0\0"
};
let length = magic.len() + 32;
let count = input.len().min(length - prefix.len());
prefix.extend_from_slice(&input[..count]);
input = &input[count..];
if prefix.len() < length {
return Ok(());
}
ensure!(
&prefix[..magic.len()] == magic,
"invalid dictionary compression header"
);
ensure!(
prefix[magic.len()..] == dictionary.hash,
"compression dictionary hash mismatch"
);
*decoder = Some(Box::new(if *brotli {
Self::Brotli(Brotli::new(Some(Arc::clone(dictionary)))?)
} else {
Self::Zstd(Zstd::new(Some(Arc::clone(dictionary)))?)
}));
}
decoder
.as_mut()
.expect("validated dictionary header")
.push(input, output)
}
}
}
fn finished(&self) -> bool {
match self {
Self::Flate(decoder) => decoder.finished,
Self::Brotli(decoder) => decoder.finished,
Self::Zstd(decoder) => decoder.finished,
Self::Dictionary { decoder, .. } => {
decoder.as_ref().is_some_and(|decoder| decoder.finished())
}
}
}
}
struct Flate {
state: Option<Decompress>,
gzip: bool,
prefix: Vec<u8>,
finished: bool,
}
impl Flate {
fn gzip() -> Self {
Self {
state: Some(Decompress::new_gzip(15)),
gzip: true,
prefix: Vec::new(),
finished: false,
}
}
fn deflate() -> Self {
Self {
state: None,
gzip: false,
prefix: Vec::new(),
finished: false,
}
}
fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
if self.state.is_none() {
self.prefix.extend_from_slice(input);
if self.prefix.len() < 2 {
return Ok(());
}
let wrapped = self.prefix[0] & 15 == 8
&& u16::from_be_bytes([self.prefix[0], self.prefix[1]]).is_multiple_of(31);
self.state = Some(Decompress::new(wrapped));
let input = std::mem::take(&mut self.prefix);
return self.push(&input, output);
}
let mut remaining = input;
loop {
if self.finished {
if remaining.is_empty() {
return Ok(());
}
ensure!(self.gzip, "bytes after a deflate stream");
self.state = Some(Decompress::new_gzip(15));
self.finished = false;
}
let decoder = self.state.as_mut().expect("initialized zlib decoder");
let mut buffer = [0; OUTPUT_SIZE];
let before_in = decoder.total_in();
let before_out = decoder.total_out();
let status = decoder.decompress(remaining, &mut buffer, FlushDecompress::None)?;
let read = (decoder.total_in() - before_in) as usize;
let written = (decoder.total_out() - before_out) as usize;
remaining = &remaining[read..];
if written != 0 {
output(&buffer[..written])?;
}
self.finished = status == Status::StreamEnd;
if self.finished && !remaining.is_empty() {
continue;
}
if (read == 0 && written == 0) || (remaining.is_empty() && written < buffer.len()) {
return Ok(());
}
}
}
}
struct Brotli {
state: NonNull<brotlic_sys::BrotliDecoderState>,
_dictionary: Option<Arc<Dictionary>>,
finished: bool,
}
// The decoder has one owner and is accessed only through &mut self. Moving it
// to the curl owner thread does not create concurrent access to native state.
unsafe impl Send for Brotli {}
impl Brotli {
fn new(dictionary: Option<Arc<Dictionary>>) -> Result<Self> {
let state = NonNull::new(unsafe {
brotlic_sys::BrotliDecoderCreateInstance(None, None, ptr::null_mut())
})
.ok_or_else(|| anyhow!("failed to allocate Brotli decoder"))?;
let decoder = Self {
state,
_dictionary: dictionary,
finished: false,
};
if let Some(dictionary) = &decoder._dictionary {
let attached = unsafe {
brotlic_sys::BrotliDecoderSetParameter(
state.as_ptr(),
brotlic_sys::BrotliDecoderParameter_BROTLI_DECODER_PARAM_LARGE_WINDOW,
1,
);
brotlic_sys::BrotliDecoderAttachDictionary(
state.as_ptr(),
brotlic_sys::BrotliSharedDictionaryType_BROTLI_SHARED_DICTIONARY_RAW,
dictionary.bytes.len(),
dictionary.bytes.as_ptr(),
)
};
ensure!(attached != 0, "failed to attach Brotli dictionary");
}
Ok(decoder)
}
fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
if input.is_empty() && self.finished {
return Ok(());
}
ensure!(!self.finished, "bytes after a Brotli stream");
let mut available_in = input.len();
let mut next_in = input.as_ptr();
loop {
let mut buffer = [0; OUTPUT_SIZE];
let mut available_out = buffer.len();
let mut next_out = buffer.as_mut_ptr();
let result = unsafe {
brotlic_sys::BrotliDecoderDecompressStream(
self.state.as_ptr(),
&mut available_in,
&mut next_in,
&mut available_out,
&mut next_out,
ptr::null_mut(),
)
};
let written = buffer.len() - available_out;
if written != 0 {
output(&buffer[..written])?;
}
match result {
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_ERROR => {
bail!("invalid Brotli response")
}
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_SUCCESS => {
ensure!(available_in == 0, "bytes after a Brotli stream");
self.finished = true;
return Ok(());
}
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT => {
return Ok(());
}
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT => {}
_ => bail!("invalid Brotli decoder status"),
}
}
}
}
impl Drop for Brotli {
fn drop(&mut self) {
unsafe {
brotlic_sys::BrotliDecoderDestroyInstance(self.state.as_ptr());
}
}
}
struct Zstd {
state: NonNull<zstd_sys::ZSTD_DCtx>,
dictionary: Option<Arc<Dictionary>>,
finished: bool,
}
// Like Brotli, this context has a unique owner and no concurrent native calls.
unsafe impl Send for Zstd {}
impl Zstd {
fn new(dictionary: Option<Arc<Dictionary>>) -> Result<Self> {
let state = NonNull::new(unsafe { zstd_sys::ZSTD_createDStream() })
.ok_or_else(|| anyhow!("failed to allocate Zstandard decoder"))?;
let decoder = Self {
state,
dictionary,
finished: false,
};
if let Some(dictionary) = &decoder.dictionary {
let result = unsafe {
zstd_sys::ZSTD_DCtx_refPrefix(
state.as_ptr(),
dictionary.bytes.as_ptr().cast(),
dictionary.bytes.len(),
)
};
ensure!(
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
"failed to attach Zstandard dictionary"
);
}
Ok(decoder)
}
fn push(&mut self, bytes: &[u8], output: &mut Output<'_>) -> Result<()> {
if bytes.is_empty() && self.finished {
return Ok(());
}
let mut input = zstd_sys::ZSTD_inBuffer {
src: bytes.as_ptr().cast::<c_void>(),
size: bytes.len(),
pos: 0,
};
loop {
if self.finished {
if let Some(dictionary) = &self.dictionary {
let result = unsafe {
zstd_sys::ZSTD_DCtx_refPrefix(
self.state.as_ptr(),
dictionary.bytes.as_ptr().cast(),
dictionary.bytes.len(),
)
};
ensure!(
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
"failed to reattach Zstandard dictionary"
);
}
self.finished = false;
}
let mut buffer = [0; OUTPUT_SIZE];
let mut decoded = zstd_sys::ZSTD_outBuffer {
dst: buffer.as_mut_ptr().cast::<c_void>(),
size: buffer.len(),
pos: 0,
};
let before = input.pos;
let result = unsafe {
zstd_sys::ZSTD_decompressStream(self.state.as_ptr(), &mut decoded, &mut input)
};
ensure!(
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
"invalid Zstandard response"
);
if decoded.pos != 0 {
output(&buffer[..decoded.pos])?;
}
self.finished = result == 0;
if self.finished && input.pos < input.size {
continue;
}
if (input.pos == input.size && decoded.pos < buffer.len())
|| (input.pos == before && decoded.pos == 0)
{
return Ok(());
}
}
}
}
impl Drop for Zstd {
fn drop(&mut self) {
unsafe {
zstd_sys::ZSTD_freeDStream(self.state.as_ptr());
}
}
}
@@ -0,0 +1,117 @@
use sfv::{BareItem, Dictionary, List, ListEntry, Parser};
use url::Url;
use urlpattern::{UrlPattern, UrlPatternInit, UrlPatternMatchInput};
#[derive(Debug)]
pub(crate) struct RegistrationOptions {
pattern: UrlPattern,
match_text: String,
destinations: Vec<String>,
id: String,
}
impl RegistrationOptions {
pub(crate) fn parse(url: &Url, headers: &[(String, Vec<u8>)]) -> Option<Self> {
let values = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
.map(|(_, value)| std::str::from_utf8(value).ok())
.collect::<Option<Vec<_>>>()?;
if values.is_empty() {
return None;
}
let combined = values.join(", ");
if combined.len() > 16 * 1024 {
return None;
}
let dictionary: Dictionary = Parser::new(&combined).parse().ok()?;
let ListEntry::Item(item) = dictionary.get("match")? else {
return None;
};
let BareItem::String(value) = &item.bare_item else {
return None;
};
let match_text = value.as_str().to_owned();
if match_text.len() > 4096 {
return None;
}
let init = UrlPatternInit::parse_constructor_string::<regex::Regex>(
&match_text,
Some(url.clone()),
)
.ok()?;
let pattern = UrlPattern::parse(init, Default::default()).ok()?;
if pattern.has_regexp_groups() {
return None;
}
let id = match dictionary.get("id") {
None => String::new(),
Some(ListEntry::Item(item)) => item.bare_item.as_string()?.as_str().to_owned(),
Some(ListEntry::InnerList(_)) => return None,
};
if id.len() > 1024 {
return None;
}
if let Some(kind) = dictionary.get("type") {
let ListEntry::Item(item) = kind else {
return None;
};
if item.bare_item.as_token()?.as_str() != "raw" {
return None;
}
}
let destinations = match dictionary.get("match-dest") {
None => Vec::new(),
Some(ListEntry::InnerList(list)) => list
.items
.iter()
.map(|item| Some(item.bare_item.as_string()?.as_str().to_owned()))
.collect::<Option<Vec<_>>>()?,
Some(ListEntry::Item(_)) => return None,
};
Some(Self {
pattern,
match_text,
destinations,
id,
})
}
pub(crate) fn matches(&self, url: &Url, destination: &str) -> bool {
(self.destinations.is_empty() || self.destinations.iter().any(|value| value == destination))
&& self
.pattern
.test(UrlPatternMatchInput::Url(url.clone()))
.unwrap_or(false)
}
pub(crate) fn match_length(&self) -> usize {
self.match_text.len()
}
pub(crate) fn same_scope(&self, other: &Self) -> bool {
self.match_text == other.match_text && self.destinations == other.destinations
}
pub(crate) fn id_header(&self) -> Option<Vec<u8>> {
(!self.id.is_empty()).then(|| {
format!("\"{}\"", self.id.replace('\\', "\\\\").replace('"', "\\\"")).into_bytes()
})
}
}
pub(super) fn clears_dictionaries(headers: &[(String, Vec<u8>)]) -> bool {
headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("clear-site-data"))
.filter_map(|(_, value)| std::str::from_utf8(value).ok())
.filter_map(|value| Parser::new(value).parse::<List>().ok())
.flatten()
.any(|entry| match entry {
ListEntry::Item(item) => item
.bare_item
.as_string()
.is_some_and(|value| matches!(value.as_str(), "cache" | "cookies" | "*")),
ListEntry::InnerList(_) => false,
})
}
@@ -0,0 +1,531 @@
//! Compression dictionaries are network-context state, partitioned like the
//! requests which may use them. Bytes enter the store only after a readable
//! response completes; a header alone cannot register a dictionary.
mod decoding;
mod fields;
use std::{borrow::Cow, collections::VecDeque, fmt, io::Read, sync::Arc};
use base64::{Engine, engine::general_purpose::STANDARD};
use moli_http_cache::{response_cache_policy, unix_now_ms};
use moli_url::{is_potentially_trustworthy_url, origin_ascii_serialization};
use parking_lot::Mutex;
use sha2::{Digest, Sha256};
use url::Url;
use crate::{FetchConfig, Request, RequestMode};
pub(crate) use decoding::DictionaryDecoding;
pub(crate) use fields::RegistrationOptions;
const MAX_DICTIONARY_BYTES: usize = 64 * 1024 * 1024;
const MAX_STORE_BYTES: usize = 128 * 1024 * 1024;
const MAX_STORE_ENTRIES: usize = 256;
pub(crate) type SharedDictionaryStore = Arc<Mutex<DictionaryStore>>;
pub(crate) struct Dictionary {
pub(crate) bytes: Arc<[u8]>,
pub(crate) hash: [u8; 32],
available: Vec<u8>,
id: Option<Vec<u8>>,
}
impl fmt::Debug for Dictionary {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Dictionary")
.field("byte_length", &self.bytes.len())
.field("available", &self.available)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DictionaryRequestSnapshot {
url: Url,
partition: Option<Partition>,
destination: String,
permitted: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Partition {
top_frame: String,
frame: String,
network: Option<String>,
}
impl Partition {
fn for_request(request: &Request, url: &Url) -> Option<Self> {
let frame = match request.request_origin() {
Some(origin) if origin.is_opaque() => return None,
Some(origin) => origin.ascii_serialization().to_owned(),
None => origin_ascii_serialization(url),
};
Some(Self {
top_frame: request
.cookie_context
.browser_context
.top_frame_origin_url
.as_ref()
.or(request
.cookie_context
.browser_context
.site_for_cookies_url
.as_ref())
.map_or_else(|| frame.clone(), origin_ascii_serialization),
frame,
network: request.network_partition_key().map(str::to_owned),
})
}
}
#[derive(Debug)]
struct Entry {
partition: Partition,
origin: String,
options: RegistrationOptions,
dictionary: Arc<Dictionary>,
expires: u64,
}
#[derive(Debug, Default)]
pub(crate) struct DictionaryStore {
entries: VecDeque<Entry>,
bytes: usize,
generation: u64,
}
impl DictionaryStore {
pub(crate) fn prepare<'a>(
store: &SharedDictionaryStore,
config: &FetchConfig,
request: &'a Request,
url: &Url,
) -> Cow<'a, Request> {
Self::prepare_inner(store, config, Cow::Borrowed(request), url)
}
pub(crate) fn prepare_owned(
store: &SharedDictionaryStore,
config: &FetchConfig,
request: Request,
url: &Url,
) -> Request {
Self::prepare_inner(store, config, Cow::Owned(request), url).into_owned()
}
fn prepare_inner<'a>(
store: &SharedDictionaryStore,
config: &FetchConfig,
mut prepared: Cow<'a, Request>,
url: &Url,
) -> Cow<'a, Request> {
let headers =
crate::blocking::outgoing_request_header_bytes_for_url(config, &prepared, url, None);
let destination = headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("sec-fetch-dest"))
.and_then(|(_, value)| std::str::from_utf8(value).ok())
.unwrap_or("");
let key = DictionaryRequestSnapshot {
url: url.clone(),
partition: Partition::for_request(&prepared, url),
destination: if destination == "empty" {
""
} else {
destination
}
.to_owned(),
permitted: is_potentially_trustworthy_url(url)
&& matches!(url.scheme(), "http" | "https")
&& !(prepared.request_mode == RequestMode::NoCors
&& prepared.has_cross_origin_url(url))
&& !prepared
.request_headers
.iter()
.any(|(name, _)| overrides_dictionary_negotiation(name))
&& !config
.default_request_headers()
.iter()
.any(|(name, _)| overrides_dictionary_negotiation(name)),
};
if prepared.compression_dictionary_snapshot.as_ref() == Some(&key)
&& prepared
.compression_dictionary_store
.as_ref()
.is_some_and(|previous| Arc::ptr_eq(previous, store))
{
return prepared;
}
let request = prepared.to_mut();
request.compression_dictionary = None;
request.compression_dictionary_store = Some(Arc::clone(store));
request.compression_dictionary_snapshot = Some(key.clone());
if !key.permitted {
return prepared;
}
let Some(partition) = key.partition else {
return prepared;
};
let origin = origin_ascii_serialization(url);
let now = unix_now_ms();
let mut locked = store.lock();
locked.retain(|entry| entry.expires > now);
let selected = locked
.entries
.iter()
.enumerate()
.filter(|(_, entry)| {
entry.partition == partition
&& entry.origin == origin
&& entry.options.matches(url, &key.destination)
})
.max_by_key(|(index, entry)| (entry.options.match_length(), *index))
.map(|(index, _)| index);
if let Some(index) = selected {
let entry = locked
.entries
.remove(index)
.expect("selected dictionary exists");
request.compression_dictionary = Some(Arc::clone(&entry.dictionary));
locked.entries.push_back(entry);
}
prepared
}
fn retain(&mut self, mut predicate: impl FnMut(&Entry) -> bool) {
self.entries.retain(|entry| predicate(entry));
self.bytes = self
.entries
.iter()
.map(|entry| entry.dictionary.bytes.len())
.sum();
}
fn clear_origin(&mut self, origin: &str) {
self.retain(|entry| entry.origin != origin);
// Cancel captures already in progress as well as retained entries.
self.generation = self.generation.wrapping_add(1);
}
fn insert(&mut self, entry: Entry) {
self.retain(|old| {
!(old.partition == entry.partition
&& old.origin == entry.origin
&& old.options.same_scope(&entry.options))
});
self.bytes += entry.dictionary.bytes.len();
self.entries.push_back(entry);
while self.bytes > MAX_STORE_BYTES || self.entries.len() > MAX_STORE_ENTRIES {
if let Some(old) = self.entries.pop_front() {
self.bytes -= old.dictionary.bytes.len();
}
}
}
}
/// Owns one response capture, including the clear-data generation at request
/// admission. Interrupted, opaque, stale and partial responses never commit.
pub(crate) struct DictionaryResponse {
store: SharedDictionaryStore,
request: ResponsePolicy,
url: Url,
generation: u64,
status: u16,
headers: Vec<(String, Vec<u8>)>,
capture: Option<Capture>,
headers_finalized: bool,
}
struct ResponsePolicy {
partition: Option<Partition>,
head: bool,
credentials: bool,
opaque: bool,
cors: Option<(String, crate::RequestCredentialsMode)>,
}
impl ResponsePolicy {
fn new(request: &Request, url: &Url) -> Self {
Self {
partition: Partition::for_request(request, url),
head: request.method == "HEAD",
credentials: request.allows_credentials_for_url(url),
opaque: request.request_mode == RequestMode::NoCors
&& request.has_cross_origin_url(url),
cors: request
.request_origin()
.filter(|_| {
request.request_mode == RequestMode::Cors && request.has_cross_origin_url(url)
})
.map(|origin| {
(
request.url_list(url).serialized_origin(origin),
request.credentials_mode,
)
}),
}
}
fn readable(&self, headers: &[(String, Vec<u8>)]) -> bool {
!self.opaque
&& self.cors.as_ref().is_none_or(|(origin, mode)| {
crate::validate_cors_response_for_origin(origin, headers, *mode).is_ok()
})
}
}
struct Capture {
options: RegistrationOptions,
partition: Partition,
expires: u64,
bytes: Vec<u8>,
}
impl DictionaryResponse {
pub(crate) fn new(store: SharedDictionaryStore, request: &Request, url: Url) -> Self {
let generation = store.lock().generation;
Self {
store,
request: ResponsePolicy::new(request, &url),
url,
generation,
status: 0,
headers: Vec::new(),
capture: None,
headers_finalized: false,
}
}
pub(crate) fn header(&mut self, data: &[u8]) {
if let Some(status) = crate::response::parse_http_response_status_line(data) {
self.status = status.status;
self.headers.clear();
self.capture = None;
self.headers_finalized = false;
return;
}
if self.headers_finalized {
return;
}
if let Some(header) = crate::headers::parse_http_response_header_line(data) {
self.headers.push(header);
return;
}
if data.iter().any(|byte| !matches!(byte, b'\r' | b'\n')) || self.status < 200 {
return;
}
self.headers_finalized = true;
if !is_potentially_trustworthy_url(&self.url) {
return;
}
if fields::clears_dictionaries(&self.headers) && self.request.credentials {
self.store
.lock()
.clear_origin(&origin_ascii_serialization(&self.url));
}
self.begin_capture(None);
}
fn begin_capture(&mut self, fresh_until: Option<u64>) {
if self.status != 200 || self.request.head || !self.request.readable(&self.headers) {
return;
}
let Some(partition) = self.request.partition.clone() else {
return;
};
let Some(options) = RegistrationOptions::parse(&self.url, &self.headers) else {
return;
};
let policy = response_cache_policy(&self.headers);
let Some(mut expires) = policy
.expires_at_unix_ms
.filter(|expires| *expires > unix_now_ms())
else {
return;
};
if let Some(deadline) = fresh_until {
expires = expires.min(deadline);
}
if expires <= unix_now_ms() {
return;
}
if !policy.store {
return;
}
self.capture = Some(Capture {
options,
partition,
expires,
bytes: Vec::new(),
});
}
pub(crate) fn decoded(&mut self, bytes: &[u8]) {
let Some(capture) = self.capture.as_mut() else {
return;
};
if bytes.len() > MAX_DICTIONARY_BYTES.saturating_sub(capture.bytes.len()) {
self.capture = None;
} else {
capture.bytes.extend_from_slice(bytes);
}
}
pub(crate) fn headers(&self) -> &[(String, Vec<u8>)] {
&self.headers
}
pub(crate) fn headers_finalized(&self) -> bool {
self.headers_finalized
}
pub(crate) fn bodyless(&self) -> bool {
self.request.head || matches!(self.status, 204 | 205 | 304)
}
pub(crate) fn from_cached(
store: SharedDictionaryStore,
request: &Request,
response: &crate::RawResponse,
fresh_until: u64,
) {
let Some(mut capture) = Self::cached(
store,
request,
&response.final_url,
response.status,
&response.headers,
Some(fresh_until),
) else {
return;
};
capture.decoded(response.body_bytes());
capture.finish();
}
pub(crate) fn cached(
store: SharedDictionaryStore,
request: &Request,
url: &Url,
status: u16,
headers: &[(String, Vec<u8>)],
fresh_until: Option<u64>,
) -> Option<Self> {
if !is_potentially_trustworthy_url(url)
|| !headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
{
return None;
}
let deadline = fresh_until.filter(|deadline| *deadline > unix_now_ms())?;
let mut capture = Self::new(store, request, url.clone());
capture.status = status;
capture.headers = headers.to_vec();
capture.begin_capture(Some(deadline));
capture.capture.as_ref()?;
Some(capture)
}
pub(crate) fn readable(&self) -> bool {
self.request.readable(&self.headers)
}
pub(crate) fn finish(&mut self) {
let Some(capture) = self.capture.take() else {
return;
};
if capture.expires <= unix_now_ms() {
return;
}
let hash: [u8; 32] = Sha256::digest(&capture.bytes).into();
let available = format!(":{}:", STANDARD.encode(hash)).into_bytes();
let dictionary = Arc::new(Dictionary {
bytes: capture.bytes.into(),
hash,
available,
id: capture.options.id_header(),
});
let mut store = self.store.lock();
if store.generation != self.generation {
return;
}
store.insert(Entry {
partition: capture.partition,
origin: origin_ascii_serialization(&self.url),
options: capture.options,
dictionary,
expires: capture.expires,
});
}
}
pub(crate) struct DictionaryCachedReader<R> {
inner: R,
capture: Option<DictionaryResponse>,
cancel: crate::FetchCancelHandle,
}
impl<R> DictionaryCachedReader<R> {
pub(crate) fn new(
inner: R,
capture: Option<DictionaryResponse>,
cancel: crate::FetchCancelHandle,
) -> Self {
Self {
inner,
capture,
cancel,
}
}
}
impl<R: Read> Read for DictionaryCachedReader<R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
let count = self.inner.read(buffer)?;
if buffer.is_empty() {
return Ok(count);
}
if let Some(capture) = self.capture.as_mut() {
if count == 0 {
if !self.cancel.is_cancelled() {
capture.finish();
}
self.capture = None;
} else {
capture.decoded(&buffer[..count]);
}
}
Ok(count)
}
}
fn overrides_dictionary_negotiation(name: &str) -> bool {
["range", "available-dictionary", "dictionary-id"]
.iter()
.any(|value| name.eq_ignore_ascii_case(value))
}
pub(crate) fn append_request_headers(headers: &mut crate::RequestHeaders, dictionary: &Dictionary) {
headers.push((
"Available-Dictionary".to_owned(),
dictionary.available.clone(),
));
if let Some(id) = &dictionary.id {
headers.push(("Dictionary-ID".to_owned(), id.clone()));
}
}
/// Dictionary representations remain reusable after registration changes the
/// browser-generated negotiation fields. Other Vary fields remain in force.
pub fn cache_vary_header_names(url: &Url, headers: &[(String, Vec<u8>)]) -> Option<Vec<String>> {
let mut names = moli_http_cache::response_vary_header_names(headers)?;
if is_potentially_trustworthy_url(url) && RegistrationOptions::parse(url, headers).is_some() {
names.retain(|name| name != "available-dictionary" && name != "accept-encoding");
}
Some(names)
}
+9 -10
View File
@@ -1,21 +1,16 @@
use moli_header_field::{HeaderFieldTokenMode, HeaderFieldTokenizer};
fn supported_content_codings() -> impl Iterator<Item = &'static str> {
let version = curl::Version::get();
[
("deflate", version.feature_libz()),
("gzip", version.feature_libz()),
("br", version.feature_brotli()),
("zstd", version.feature_zstd()),
]
.into_iter()
.filter_map(|(coding, supported)| supported.then_some(coding))
["deflate", "gzip", "br", "zstd", "dcb", "dcz"].into_iter()
}
/// Choose the browser's header before cache lookup and transport setup so
/// negotiation and Vary matching use the same actual value.
pub(crate) fn default_accept_encoding() -> String {
let value = supported_content_codings().collect::<Vec<_>>().join(", ");
let value = supported_content_codings()
.filter(|coding| !matches!(*coding, "dcb" | "dcz"))
.collect::<Vec<_>>()
.join(", ");
if value.is_empty() {
"identity".to_owned()
} else {
@@ -23,6 +18,10 @@ pub(crate) fn default_accept_encoding() -> String {
}
}
pub(crate) fn dictionary_accept_encoding() -> String {
supported_content_codings().collect::<Vec<_>>().join(", ")
}
/// Resource Timing exposes registered, supported content codings rather than
/// reflecting an arbitrary response header into the timing entry.
pub fn resource_timing_content_encoding(headers: &[(String, Vec<u8>)]) -> &'static str {
+2
View File
@@ -9,6 +9,7 @@ mod blocking;
mod cancellation;
mod client;
mod client_hints;
mod compression_dictionary;
mod config;
mod content_coding;
mod cors;
@@ -43,6 +44,7 @@ pub use blocking::{
};
pub use cancellation::{FetchCancelHandle, FetchResponseBodySizes, FetchResponseTiming};
pub use client::{FetchClient, FetchClientHandle};
pub use compression_dictionary::cache_vary_header_names;
pub use config::FetchConfig;
pub use content_coding::resource_timing_content_encoding;
pub use cors::validate_cors_response_for_origin;
+21 -1
View File
@@ -112,6 +112,11 @@ pub struct Request {
redirect_policy: Option<RequestRedirectPolicy>,
upload_observer: Option<crate::UploadObserver>,
redirect_check: Option<RequestRedirectCheck>,
pub(crate) compression_dictionary: Option<Arc<crate::compression_dictionary::Dictionary>>,
pub(crate) compression_dictionary_store:
Option<crate::compression_dictionary::SharedDictionaryStore>,
pub(crate) compression_dictionary_snapshot:
Option<crate::compression_dictionary::DictionaryRequestSnapshot>,
}
/// Request-owned admission check, run before following each redirect target.
@@ -272,6 +277,7 @@ pub enum BrowserRequestMetadata {
Audio,
AudioWorklet,
Beacon,
Dictionary,
EventSource,
Fetch,
Font,
@@ -309,7 +315,12 @@ impl BrowserRequestMetadata {
Self::Style | Self::StyleModule => "style",
Self::TextTrack => "track",
Self::Video => "video",
Self::Beacon | Self::EventSource | Self::Fetch | Self::Ping | Self::Xhr => "empty",
Self::Beacon
| Self::Dictionary
| Self::EventSource
| Self::Fetch
| Self::Ping
| Self::Xhr => "empty",
}
}
}
@@ -665,6 +676,9 @@ impl Request {
redirect_policy: None,
upload_observer: None,
redirect_check: None,
compression_dictionary: None,
compression_dictionary_store: None,
compression_dictionary_snapshot: None,
})
}
@@ -707,6 +721,9 @@ impl Request {
redirect_policy: None,
upload_observer: None,
redirect_check: None,
compression_dictionary: None,
compression_dictionary_store: None,
compression_dictionary_snapshot: None,
}
}
@@ -782,6 +799,9 @@ impl Request {
redirect_policy: None,
upload_observer: None,
redirect_check: None,
compression_dictionary: None,
compression_dictionary_store: None,
compression_dictionary_snapshot: None,
}
}
+2
View File
@@ -995,6 +995,8 @@ mod resource_timing_tests {
(vec!["deflate"], "deflate"),
(vec!["br"], "br"),
(vec!["zStd "], "zstd"),
(vec!["dcb"], "dcb"),
(vec!["dcz"], "dcz"),
(vec!["gzip, deflate, Apple"], "multiple"),
(vec!["gzip, "], "multiple"),
(vec!["gzip", "deflate"], "multiple"),
+219 -21
View File
@@ -370,19 +370,21 @@ impl PendingStreamingRawResponse {
impl FetchRuntimeOwner {
#[cfg(test)]
pub(crate) fn new(config: &FetchConfig, cookie_store: SharedBrowserCookieStore) -> Self {
Self::new_with_client_hint_preferences(
Self::new_with_network_state(
config,
cookie_store,
Arc::new(Mutex::new(
crate::client_hints::ClientHintPreferences::default(),
)),
Arc::default(),
)
}
pub(crate) fn new_with_client_hint_preferences(
pub(crate) fn new_with_network_state(
config: &FetchConfig,
cookie_store: SharedBrowserCookieStore,
client_hint_preferences: SharedClientHintPreferences,
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
) -> Self {
let runtime_id = NEXT_FETCH_RUNTIME_ID
.fetch_add(1, Ordering::Relaxed)
@@ -401,6 +403,7 @@ impl FetchRuntimeOwner {
config: config.clone(),
cookie_store,
client_hint_preferences,
dictionaries,
request_rx,
curl_runtime,
curl_http,
@@ -711,6 +714,7 @@ struct RuntimeOwner {
config: FetchConfig,
cookie_store: SharedBrowserCookieStore,
client_hint_preferences: SharedClientHintPreferences,
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
request_rx: Receiver<RuntimeCommand>,
curl_runtime: CurlMultiRuntime<FetchTransferHandler, ActiveTransferContext>,
curl_http: CurlHttpSender<FetchTransferHandler, ActiveTransferContext>,
@@ -721,6 +725,27 @@ struct RuntimeOwner {
}
impl RuntimeOwner {
fn prepare_request(
&self,
restarts: &SharedNavigationClientHintRestarts,
request: &Request,
url: &Url,
) -> crate::client_hints::PreparedClientHintRequest {
let mut prepared = prepare_client_hint_request(
&self.client_hint_preferences,
restarts,
&self.config,
request,
url,
);
prepared.request = crate::compression_dictionary::DictionaryStore::prepare_owned(
&self.dictionaries,
&self.config,
prepared.request,
url,
);
prepared
}
fn run(self) {
#[cfg(test)]
self.owner_started.store(true, Ordering::SeqCst);
@@ -900,10 +925,8 @@ impl RuntimeOwner {
None
};
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
let prepared_request = prepare_client_hint_request(
&self.client_hint_preferences,
let prepared_request = self.prepare_request(
&job.client_hint_navigation_restarts,
&self.config,
&job.request,
&job.current_url,
);
@@ -1105,10 +1128,8 @@ impl RuntimeOwner {
None
};
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
let prepared_request = prepare_client_hint_request(
&self.client_hint_preferences,
let prepared_request = self.prepare_request(
&job.client_hint_navigation_restarts,
&self.config,
&job.request,
&job.current_url,
);
@@ -1349,10 +1370,8 @@ impl RuntimeOwner {
None
};
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
let prepared_request = prepare_client_hint_request(
&self.client_hint_preferences,
let prepared_request = self.prepare_request(
&job.client_hint_navigation_restarts,
&self.config,
&job.request,
&job.current_url,
);
@@ -1535,7 +1554,28 @@ impl RuntimeOwner {
}
}
fn finish_active_transfer(&self, state: &mut OwnerState, completion: RuntimeCurlCompletion) {
fn finish_active_transfer(
&self,
state: &mut OwnerState,
mut completion: RuntimeCurlCompletion,
) {
let cancelled = match &completion.context {
ActiveTransferContext::Buffered(context) => context.job.cancel_handle.is_cancelled(),
ActiveTransferContext::Streaming(context) => context.job.cancel_handle.is_cancelled(),
ActiveTransferContext::StreamingRaw(context) => {
context.job.cancel_handle.is_cancelled()
}
};
if let Some(easy) = completion.easy.as_mut() {
if let Some(error) = easy.get_mut().dictionary_error.take() {
completion.result = Err(anyhow!(error));
} else if completion.result.is_ok()
&& !self.shutdown_requested.load(Ordering::SeqCst)
&& !cancelled
{
completion.result = easy.get_mut().finish_dictionary_response();
}
}
state.active_transfers = state.active_transfers.saturating_sub(1);
match completion.context {
ActiveTransferContext::Buffered(context) => {
@@ -2612,6 +2652,7 @@ impl RuntimeOwner {
return Ok(());
}
job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries));
complete_cached_streaming_html_job(job, cached, request_cookie_report);
Ok(())
}
@@ -2674,6 +2715,7 @@ impl RuntimeOwner {
return Ok(());
}
job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries));
complete_cached_streaming_raw_job(job, cached, request_cookie_report);
Ok(())
}
@@ -2898,6 +2940,9 @@ pub(crate) struct FetchTransferHandler {
network_observation_recorder: Option<NetworkObservationRecorder>,
upload_observer: Option<crate::UploadObserver>,
proxy_connect_response_recorder: ProxyConnectResponseRecorder,
dictionary_response: Option<crate::compression_dictionary::DictionaryResponse>,
dictionary_decoding: Option<crate::compression_dictionary::DictionaryDecoding>,
dictionary_error: Option<String>,
}
enum FetchResponseCollector {
@@ -2943,6 +2988,9 @@ impl FetchTransferHandler {
network_observation_recorder: None,
upload_observer: None,
proxy_connect_response_recorder: ProxyConnectResponseRecorder::default(),
dictionary_response: None,
dictionary_decoding: None,
dictionary_error: None,
}
}
@@ -3001,6 +3049,31 @@ impl FetchTransferHandler {
.begin_transfer(capture_proxy_connect_response);
}
pub(crate) fn begin_dictionary_response(&mut self, request: &Request, url: &Url) {
self.dictionary_response = request.compression_dictionary_store.as_ref().map(|store| {
crate::compression_dictionary::DictionaryResponse::new(
Arc::clone(store),
request,
url.clone(),
)
});
self.dictionary_decoding = Some(crate::compression_dictionary::DictionaryDecoding::new(
request.compression_dictionary.clone(),
));
self.dictionary_error = None;
}
fn finish_dictionary_response(&mut self) -> Result<()> {
let Some(response) = self.dictionary_response.as_mut() else {
return Ok(());
};
if let Some(decoding) = self.dictionary_decoding.as_mut() {
decoding.finish(response.headers(), response.bodyless())?;
}
response.finish();
Ok(())
}
fn take_failed_proxy_connect_response(
&mut self,
connect_status: u32,
@@ -3035,17 +3108,101 @@ impl Handler for FetchTransferHandler {
if !data.is_empty() {
self.mark_response_body_received();
}
if let Some(timing) = self.response_progress() {
timing.record_decoded_response_body_size(data.len());
if self.dictionary_decoding.is_none() {
if let Some(timing) = self.response_progress() {
timing.record_decoded_response_body_size(data.len());
}
let result = match &mut self.response {
FetchResponseCollector::Buffered(collector) => collector.write(data),
FetchResponseCollector::Streaming(collector) => collector.write(data),
FetchResponseCollector::StreamingRaw(collector) => collector.write(data),
};
if result
.as_ref()
.is_ok_and(|accepted| *accepted == data.len())
&& let Some(policy) = self.dictionary_response.as_mut()
{
policy.decoded(data);
}
return result;
}
match &mut self.response {
FetchResponseCollector::Buffered(collector) => collector.write(data),
FetchResponseCollector::Streaming(collector) => collector.write(data),
FetchResponseCollector::StreamingRaw(collector) => collector.write(data),
if self.dictionary_error.is_some() {
return Ok(0);
}
let timing = self.response_progress().cloned();
let decoding = self
.dictionary_decoding
.as_mut()
.expect("content decoding selected");
decoding.encoded_bytes = decoding.encoded_bytes.saturating_add(data.len() as u64);
if let Some(timing) = &timing {
timing.record_encoded_response_body_size(decoding.encoded_bytes);
}
if let Err(error) = decoding.initialize(
self.dictionary_response
.as_ref()
.map_or(&[], |response| response.headers()),
) {
self.dictionary_error = Some(error.to_string());
return Ok(0);
}
let policy = &mut self.dictionary_response;
let response = &mut self.response;
let mut output = |bytes: &[u8]| -> Result<()> {
if let Some(timing) = &timing {
timing.record_decoded_response_body_size(bytes.len());
}
let accepted = match response {
FetchResponseCollector::Buffered(collector) => collector.write(bytes),
FetchResponseCollector::Streaming(collector) => collector.write(bytes),
FetchResponseCollector::StreamingRaw(collector) => collector.write(bytes),
}
.map_err(|_| anyhow!("response collector rejected decoded bytes"))?;
anyhow::ensure!(
accepted == bytes.len(),
"response collector rejected decoded bytes"
);
if let Some(policy) = policy.as_mut() {
policy.decoded(bytes);
}
Ok(())
};
let result = self
.dictionary_decoding
.as_mut()
.expect("dictionary decoding selected")
.push(data, &mut output);
match result {
Ok(()) => Ok(data.len()),
Err(error) => {
self.dictionary_error = Some(error.to_string());
Ok(0)
}
}
}
fn header(&mut self, data: &[u8]) -> bool {
if crate::response::parse_http_response_status_line(data)
.is_some_and(|line| line.status >= 200)
&& let Some(decoding) = self.dictionary_decoding.as_mut()
{
decoding.reset();
self.dictionary_error = None;
}
if let Some(response) = self.dictionary_response.as_mut() {
response.header(data);
}
if data.iter().all(|byte| matches!(byte, b'\r' | b'\n'))
&& let Some(response) = self.dictionary_response.as_ref()
&& response.headers_finalized()
&& !response.bodyless()
&& let Some(decoding) = self.dictionary_decoding.as_mut()
&& decoding.uses_dictionary()
&& !response.readable()
{
self.dictionary_error =
Some("dictionary response failed its CORS readability check".to_owned());
}
if crate::response::parse_http_response_status_line(data)
.is_some_and(|line| line.status >= 200)
&& let Some(timing) = self.response_progress()
@@ -3251,7 +3408,7 @@ fn complete_cached_streaming_html_job(
final_url,
status,
headers,
mut body,
body,
..
} = cached;
let final_url = match Url::parse(&final_url) {
@@ -3270,9 +3427,29 @@ fn complete_cached_streaming_html_job(
}
};
let capture = job
.request
.compression_dictionary_store
.as_ref()
.and_then(|store| {
crate::compression_dictionary::DictionaryResponse::cached(
Arc::clone(store),
&job.request,
&final_url,
status,
&headers,
metadata.expires_at_unix_ms,
)
});
let mut body = crate::compression_dictionary::DictionaryCachedReader::new(
body,
capture,
job.cancel_handle.clone(),
);
let response_timing = job
.response_progress
.mark_cached_response_available(body_sizes);
if let Some(started_tx) = job.started_tx {
let _ = started_tx.send(Ok(StreamingHtmlResponseStart {
status_text: metadata.status_text,
@@ -3381,7 +3558,7 @@ fn complete_cached_streaming_raw_job(
final_url,
status,
headers,
mut body,
body,
..
} = cached;
let final_url = match Url::parse(&final_url) {
@@ -3400,6 +3577,25 @@ fn complete_cached_streaming_raw_job(
}
};
let capture = job
.request
.compression_dictionary_store
.as_ref()
.and_then(|store| {
crate::compression_dictionary::DictionaryResponse::cached(
Arc::clone(store),
&job.request,
&final_url,
status,
&headers,
metadata.expires_at_unix_ms,
)
});
let mut body = crate::compression_dictionary::DictionaryCachedReader::new(
body,
capture,
job.cancel_handle.clone(),
);
let response_timing = job
.response_progress
.mark_cached_response_available(body_sizes);
@@ -3474,6 +3670,7 @@ fn request_fetch_load_priority(request: &Request) -> crate::ResourceLoadPriority
Some(
crate::BrowserRequestMetadata::Audio
| crate::BrowserRequestMetadata::Beacon
| crate::BrowserRequestMetadata::Dictionary
| crate::BrowserRequestMetadata::Font
| crate::BrowserRequestMetadata::Image
| crate::BrowserRequestMetadata::Ping
@@ -4430,7 +4627,8 @@ mod tests {
.with_resource_type(crate::RequestResourceType::LinkPrefetch);
let dictionary = Request::new("GET", "https://example.test/dict.bin", None, Vec::new())
.expect("test request should parse")
.with_resource_type(crate::RequestResourceType::Dictionary);
.with_resource_type(crate::RequestResourceType::Dictionary)
.with_browser_request_metadata(crate::BrowserRequestMetadata::Dictionary);
assert_eq!(
request_fetch_load_priority(&stylesheet),
@@ -0,0 +1,434 @@
use super::*;
const DICTIONARY: &[u8] = b"This is a test dictionary.\n";
const PAYLOAD: &[u8] = b"This is compressed test data using a test dictionary";
const HASH: &str = ":U5abz16WDg7b8KS93msLPpOB4Vbef1uRzoORYkJw9BY=:";
const DCB: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcb");
const DCZ: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcz");
fn reply(headers: &[(&str, &str)], body: &[u8], chunked: bool) -> Vec<u8> {
let mut head =
"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain\r\n".to_owned();
for (name, value) in headers {
head.push_str(&format!("{name}: {value}\r\n"));
}
if chunked {
head.push_str("Transfer-Encoding: chunked\r\n\r\n");
} else {
head.push_str(&format!("Content-Length: {}\r\n\r\n", body.len()));
}
let mut response = head.into_bytes();
if chunked {
for byte in body {
response.extend_from_slice(&[b'1', b'\r', b'\n', *byte, b'\r', b'\n']);
}
response.extend_from_slice(b"0\r\nX-Trailer: retained\r\n\r\n");
} else {
response.extend_from_slice(body);
}
response
}
async fn server(
responses: Vec<Vec<u8>>,
) -> Result<(String, tokio::task::JoinHandle<Result<Vec<Vec<u8>>>>)> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/data", listener.local_addr()?);
let task = tokio::spawn(async move {
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) =
tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
let mut request = Vec::new();
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
let mut buffer = [0; 4096];
let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer))
.await??;
anyhow::ensure!(count != 0 && request.len() < 65536, "incomplete request");
request.extend_from_slice(&buffer[..count]);
}
requests.push(request);
// Error responses may make the client close the socket early.
let _ = stream.write_all(&response).await;
let _ = stream.shutdown().await;
}
Ok(requests)
});
Ok((url, task))
}
fn header<'a>(request: &'a [u8], name: &str) -> Option<&'a str> {
std::str::from_utf8(request)
.unwrap()
.lines()
.filter_map(|line| line.split_once(':'))
.find(|(field, _)| field.eq_ignore_ascii_case(name))
.map(|(_, value)| value.trim())
}
async fn fetch(client: &FetchClientHandle, url: &str, mode: &str) -> Result<Vec<u8>> {
let request = Request::new("GET", url, None, crate::RequestHeaders::default())?;
let (_, body) = match mode {
"buffered" => client
.fetch(request.with_auth(crate::RequestAuth {
target: crate::RequestAuthTarget::Server,
scheme: crate::RequestAuthScheme::Digest,
username: "user".to_owned(),
password: "password".to_owned(),
}))
.await?
.into_body(),
"html" => client.fetch_html_stream(request).await?.into_body(),
"raw" => client
.fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
.await?
.into_body(),
_ => unreachable!(),
};
body.into_materialized_bytes().await
}
#[tokio::test]
async fn compression_dictionary_registration_and_decoding_cover_every_transport() -> Result<()> {
for mode in ["raw", "html", "buffered"] {
for chunked in [false, true] {
let coding = [
("dcb", DCB, PAYLOAD),
("dcz", DCZ, PAYLOAD),
(
"gzip",
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.gzip")
.as_slice(),
DICTIONARY,
),
(
"deflate",
include_bytes!(
"../../tests/fixtures/compression-dictionary/dictionary.deflate"
)
.as_slice(),
DICTIONARY,
),
(
"deflate",
include_bytes!(
"../../tests/fixtures/compression-dictionary/dictionary.raw-deflate"
)
.as_slice(),
DICTIONARY,
),
(
"br",
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.br")
.as_slice(),
DICTIONARY,
),
(
"zstd",
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.zstd")
.as_slice(),
DICTIONARY,
),
];
let mut responses = vec![reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\", id=\"first\""),
],
DICTIONARY,
chunked,
)];
responses.extend(
coding
.iter()
.map(|(name, body, _)| reply(&[("Content-Encoding", name)], body, chunked)),
);
let (url, task) = server(responses).await?;
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
assert_eq!(
fetch(&client, &url, mode).await?,
DICTIONARY,
"{mode}/{chunked}"
);
for (name, _, expected) in coding {
assert_eq!(
fetch(&client, &url, mode).await?,
expected,
"{mode}/{chunked}/{name}"
);
}
let requests = task.await??;
assert_eq!(header(&requests[0], "available-dictionary"), None);
assert!(
!header(&requests[0], "accept-encoding")
.unwrap()
.contains("dcb")
);
for request in &requests[1..] {
assert_eq!(header(request, "available-dictionary"), Some(HASH));
assert_eq!(header(request, "dictionary-id"), Some("\"first\""));
let encoding = header(request, "accept-encoding").unwrap();
assert!(encoding.contains("dcb") && encoding.contains("dcz"));
}
}
}
Ok(())
}
#[tokio::test]
async fn compression_dictionary_clear_site_data_observes_cache_and_cookie_directives() -> Result<()>
{
let registered = reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\""),
],
DICTIONARY,
false,
);
for directive in ["storage", "cache", "cookies", "*"] {
let field = format!("\"{directive}\"");
let (url, task) = server(vec![
registered.clone(),
reply(&[("Clear-Site-Data", &field)], b"OK", false),
reply(&[], b"OK", false),
])
.await?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for _ in 0..3 {
fetch(&client, &url, "raw").await?;
}
let requests = task.await??;
assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH));
assert_eq!(
header(&requests[2], "available-dictionary"),
if directive == "storage" {
Some(HASH)
} else {
None
}
);
}
Ok(())
}
#[tokio::test]
async fn compression_dictionary_bad_hash_and_truncated_streams_do_not_register() -> Result<()> {
let registered = reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\", id=\"good\""),
],
DICTIONARY,
false,
);
for (coding, original) in [("dcb", DCB), ("dcz", DCZ)] {
let mut bad_hash = original.to_vec();
bad_hash[12] ^= 1;
for body in [bad_hash.as_slice(), &original[..original.len() - 1]] {
let invalid = reply(
&[
("Content-Encoding", coding),
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\", id=\"bad\""),
],
body,
true,
);
let (url, task) =
server(vec![registered.clone(), invalid, reply(&[], b"OK", false)]).await?;
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
fetch(&client, &url, "raw").await?;
assert!(fetch(&client, &url, "raw").await.is_err(), "{coding}");
fetch(&client, &url, "raw").await?;
assert_eq!(header(&task.await??[2], "dictionary-id"), Some("\"good\""));
}
}
Ok(())
}
#[tokio::test]
async fn compression_dictionary_request_snapshot_survives_later_registration() -> Result<()> {
let (url, task) = server(vec![
reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\""),
],
DICTIONARY,
false,
),
reply(&[], b"OK", false),
reply(&[], b"OK", false),
reply(&[], b"OK", false),
])
.await?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let mut early = Request::get(&url)?;
client.prepare_request(&mut early);
fetch(&client, &url, "raw").await?;
assert!(
!client
.cache_request_headers(&early)
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("available-dictionary"))
);
client.fetch_raw(early.clone()).await?;
fetch(&client, &url, "raw").await?;
client
.fetch_raw(early.with_network_partition_key(Some("https://other.test".to_owned())))
.await?;
let requests = task.await??;
assert_eq!(header(&requests[1], "available-dictionary"), None);
assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH));
assert_eq!(header(&requests[3], "available-dictionary"), None);
Ok(())
}
#[tokio::test]
async fn compression_dictionary_cached_body_registration_keeps_original_freshness() -> Result<()> {
let (url, task) = server(vec![
reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\""),
],
DICTIONARY,
false,
),
reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false),
reply(&[], b"OK", false),
reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false),
reply(&[], b"OK", false),
])
.await?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let request = Request::get(&url)?;
let cached = client.fetch_raw(request.clone()).await?;
fetch(&client, &url, "raw").await?;
client.observe_cached_response(&request, &cached, moli_http_cache::unix_now_ms() + 600_000);
fetch(&client, &url, "raw").await?;
fetch(&client, &url, "raw").await?;
client.observe_cached_response(
&request,
&cached,
moli_http_cache::unix_now_ms().saturating_sub(1),
);
fetch(&client, &url, "raw").await?;
let requests = task.await??;
assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH));
assert_eq!(header(&requests[4], "available-dictionary"), None);
Ok(())
}
#[tokio::test]
async fn compression_dictionary_disk_cache_hit_registers_decoded_dictionary() -> Result<()> {
let (url, task) = server(vec![
reply(
&[
("Cache-Control", "max-age=600"),
("Use-As-Dictionary", "match=\"/*\""),
],
DICTIONARY,
false,
),
reply(&[], b"probe", false),
])
.await?;
let cache_dir = unique_test_cache_dir();
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let request = Request::get(&url)?;
{
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
assert_eq!(
client.fetch_raw(request.clone()).await?.body_bytes(),
DICTIONARY
);
}
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
let hit = client.fetch_raw(request).await?;
assert_eq!(hit.cache_state, crate::ResponseCacheState::Local);
let probe = format!("{url}?probe");
client.fetch_raw(Request::get(&probe)?).await?;
let requests = task.await??;
assert_eq!(requests.len(), 2);
assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH));
fs::remove_dir_all(cache_dir)?;
Ok(())
}
#[tokio::test]
async fn compression_dictionary_registration_requires_readable_and_fresh_responses() -> Result<()> {
let cases = [
("match=\"/*\"", "max-age=600", "0", true),
("match=42", "max-age=600", "0", false),
("match=\"/:id(d.*)\"", "max-age=600", "0", false),
("match=\"/*\", type=other", "max-age=600", "0", false),
(
"match=\"/*\", match-dest=\"script\"",
"max-age=600",
"0",
false,
),
("match=\"/*\"", "max-age=600", "601", false),
("match=\"/*\"", "no-store, max-age=600", "0", false),
];
for (options, cache, age, expected) in cases {
let (url, task) = server(vec![
reply(
&[
("Use-As-Dictionary", options),
("Cache-Control", cache),
("Age", age),
],
DICTIONARY,
false,
),
reply(&[], b"OK", false),
])
.await?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
fetch(&client, &url, "raw").await?;
fetch(&client, &url, "raw").await?;
assert_eq!(
header(&task.await??[1], "available-dictionary"),
expected.then_some(HASH),
"{options}/{cache}/{age}"
);
}
for (mode, cors, expected) in [
(crate::RequestMode::NoCors, true, false),
(crate::RequestMode::Cors, false, false),
(crate::RequestMode::Cors, true, true),
] {
let mut headers = vec![
("Use-As-Dictionary", "match=\"/*\""),
("Cache-Control", "max-age=600"),
];
if cors {
headers.push(("Access-Control-Allow-Origin", "*"));
}
let (url, task) = server(vec![
reply(&headers, DICTIONARY, false),
reply(&[("Access-Control-Allow-Origin", "*")], b"OK", false),
])
.await?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let mut request = Request::get(&url)?
.with_request_origin(moli_url::WebOrigin::from_serialized("https://frame.test"));
request.request_mode = mode;
request.credentials_mode = crate::RequestCredentialsMode::Omit;
let _ = client.fetch_raw(request.clone()).await;
request.request_mode = crate::RequestMode::Cors;
client.fetch_raw(request).await?;
assert_eq!(
header(&task.await??[1], "available-dictionary"),
expected.then_some(HASH),
"{mode:?}/{cors}"
);
}
Ok(())
}
+1
View File
@@ -3,6 +3,7 @@ mod cache_request_controls_modes;
mod cache_request_directives;
mod cache_request_modes;
mod cache_vary_encoding;
mod compression_dictionary;
mod cookie_context;
mod cookie_store;
mod corp;
@@ -0,0 +1,7 @@
*.br binary
*.dcb binary
*.dcz binary
*.deflate binary
*.gzip binary
*.raw-deflate binary
*.zstd binary
@@ -0,0 +1,5 @@
# Compression dictionary fixtures
The `dictionary.*` and `payload.*` Brotli, gzip, Zstandard, dcb and dcz bytes are copied unchanged from the upstream WPT `fetch/compression-dictionary/resources/register-dictionary.py` and `compressed-data.py`, snapshot `db95fafd1fcef8428805e41eb5705d444e8c67ce`. The two deflate fixtures encode the same dictionary with Python zlib, with and without the zlib wrapper.
Dictionary bytes are `This is a test dictionary.\n`; payload bytes are `This is compressed test data using a test dictionary`. The dcb/dcz headers and dictionary SHA-256 remain part of each fixture.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
�DCBS–›ֿ^–�נ₪½�k >“�בV�[‘־ƒ‘bBpפ¡˜€b₪L�„�®ֲֺ`"n�ֹ·ֳD�¼אלֱמ43>
Binary file not shown.
@@ -13,8 +13,8 @@ use moli_fetch::{
BrowserRequestMetadata, RawResponse, Request, RequestHeaders, RequestResourceType, Response,
};
use moli_http_cache::{
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date,
response_vary_header_names, unix_now_ms, validation_headers_from_headers,
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date, unix_now_ms,
validation_headers_from_headers,
};
use parking_lot::Mutex;
use url::Url;
@@ -81,7 +81,10 @@ pub(in crate::network) struct RawSubresourceCacheKey {
}
pub(in crate::network) enum RawSubresourceCacheLookup {
Local(Box<RawResponse>),
Local {
response: Box<RawResponse>,
expires_at_unix_ms: u64,
},
Revalidate(Box<RawResponse>),
}
@@ -724,9 +727,13 @@ impl SharedMemoryResourceCache {
return Some(RawSubresourceCacheLookup::Revalidate(variant.response));
}
let mut hit = variants[index].response.clone();
let expires_at_unix_ms = variants[index].expires_at_unix_ms;
hit.final_url.set_fragment(request.url.fragment());
self.insert_raw_subresource_variants(key.clone(), variants);
Some(RawSubresourceCacheLookup::Local(hit))
Some(RawSubresourceCacheLookup::Local {
response: hit,
expires_at_unix_ms,
})
}
pub(in crate::network) fn remove_raw_subresource(
@@ -750,7 +757,9 @@ impl SharedMemoryResourceCache {
if response.body_bytes().len() > self.resource_body_bytes_limit {
return;
}
let Some(vary_names) = response_vary_header_names(&response.headers) else {
let Some(vary_names) =
moli_fetch::cache_vary_header_names(&response.final_url, &response.headers)
else {
return;
};
let variant = RawSubresourceVariant {
@@ -1089,7 +1098,7 @@ pub(in crate::network) fn raw_subresource_memory_cache_expiry(
// Match ordinary fields against the same encoded builder as the native
// request. Mutable transport-only fields need exact sent-value tracking
// before a renderer representation can safely vary on them.
if !response_vary_header_names(&response.headers)?
if !moli_fetch::cache_vary_header_names(&response.final_url, &response.headers)?
.iter()
.all(|name| {
name.bytes()
@@ -1019,7 +1019,15 @@ impl ResourceRequestClient {
});
let mut network_request = request.clone();
let cached = match lookup {
Some(RawSubresourceCacheLookup::Local(cached)) => {
Some(RawSubresourceCacheLookup::Local {
response: cached,
expires_at_unix_ms,
}) => {
self.resource_runtime.client().observe_cached_response(
&request,
&cached,
expires_at_unix_ms,
);
if let Some(sizes) = cached.body_sizes() {
cancel_handle.set_response_body_sizes(sizes);
}
@@ -2490,10 +2490,8 @@ fn preload_like_link_readiness_fetch_options(
};
let credentials_mode = if resource_type == SubresourceResourceType::Dictionary {
// Blink sets compression dictionary requests to CORS mode with
// credentials omitted and no referrer. Request::new already uses CORS
// for subresources; Moli does not yet model per-request referrer
// policy for this loader path, so credentials and priority are the
// observable pieces preserved here.
// credentials omitted and no referrer. Browser metadata and referrer
// suppression are applied by connected_link_readiness_request.
RequestCredentialsMode::Omit
} else if cross_origin.is_some() {
link_crossorigin_credentials_mode(cross_origin)
@@ -2654,6 +2652,11 @@ fn connected_link_readiness_request(
if let Some(resource_type) = options.request_resource_type {
request = request.with_resource_type(resource_type);
}
if options.resource_type == SubresourceResourceType::Dictionary {
request = request
.with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Dictionary)
.without_inferred_referrer();
}
if let Some(metadata) = options.script_fetch_metadata.as_ref() {
request = request.with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Script);
request = request.with_script_fetch_metadata(moli_fetch::ScriptFetchRequestMetadata {
@@ -5297,6 +5300,30 @@ mod tests {
Some(RequestResourceType::Dictionary)
);
assert_eq!(options.credentials_mode, RequestCredentialsMode::Omit);
let document_url = document.final_url().expect("HTML document URL");
let url = Url::parse("https://example.test/dict.bin").unwrap();
let request = connected_link_readiness_request(
document_url,
&moli_url::WebOrigin::from_url(document_url),
&url,
&options,
);
let headers = moli_fetch::outgoing_request_headers(&Default::default(), &request, None);
assert_eq!(
request.browser_request_metadata(),
Some(moli_fetch::BrowserRequestMetadata::Dictionary)
);
assert!(headers.iter().any(|(name, value)| name.eq_ignore_ascii_case("sec-fetch-mode") && value == "cors"));
assert!(
headers.iter().any(
|(name, value)| name.eq_ignore_ascii_case("sec-fetch-dest") && value == "empty"
)
);
assert!(
!headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("referer"))
);
assert!(!options.link_preload);
assert!(options.script_fetch_metadata.is_none());
}
@@ -8,6 +8,8 @@ pub fn is_forbidden_request_header_name(name: &str) -> bool {
name.as_str(),
"accept-charset"
| "accept-encoding"
| "available-dictionary"
| "dictionary-id"
| "access-control-request-headers"
| "access-control-request-method"
| "connection"