mirror of
https://github.com/lexmount/moli.git
synced 2026-11-21 00:03:15 +00:00
feat(fetch): support partitioned compression dictionaries
This commit is contained in:
Generated
+51
@@ -2964,10 +2964,13 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
"base64 0.22.1",
|
||||
"brotlic-sys",
|
||||
"cidr",
|
||||
"crossbeam-channel",
|
||||
"curl",
|
||||
"curl-sys",
|
||||
"flate2",
|
||||
"h2",
|
||||
"http",
|
||||
"moli-browser-profile",
|
||||
@@ -2986,8 +2989,11 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rcgen",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sfv",
|
||||
"sha2",
|
||||
"strum",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -2996,7 +3002,9 @@ dependencies = [
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlpattern",
|
||||
"wildcard",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5028,6 +5036,26 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.3"
|
||||
@@ -5423,6 +5451,17 @@ dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sfv"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "890ef0e5532daf21fdaa3dbaf991417187f3a87ab69231fe69be942495746480"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"indexmap",
|
||||
"ref-cast",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
@@ -6569,6 +6608,18 @@ version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "urlpattern"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df16f50ef4cc145211879a3867ba757076b25dfee812040dcb0658bd9ae7904b"
|
||||
dependencies = [
|
||||
"icu_properties",
|
||||
"regex",
|
||||
"serde",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "usvg"
|
||||
version = "0.48.1"
|
||||
|
||||
@@ -17,6 +17,9 @@ cidr = "0.3.2"
|
||||
crossbeam-channel = "0.5.15"
|
||||
curl = { version = "0.4.51", default-features = false, features = ["brotli", "browser-response-headers", "http2", "poll_7_68_0", "ssl", "static-curl"] }
|
||||
curl-sys = { version = "0.4.91", default-features = false }
|
||||
base64 = "0.22.1"
|
||||
brotlic-sys = "0.2.2"
|
||||
flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] }
|
||||
http = "1"
|
||||
moli-browser-profile = { path = "../moli-browser-profile" }
|
||||
moli-cookie-jar = { path = "../moli-cookie-jar" }
|
||||
@@ -31,14 +34,19 @@ moli-web-bot-auth = { path = "../moli-web-bot-auth" }
|
||||
moli-web-mime = { path = "../moli-web-mime" }
|
||||
parking_lot = "0.12"
|
||||
percent-encoding = "2"
|
||||
regex = "1"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
sfv = "0.15.0"
|
||||
sha2 = "0.11.0"
|
||||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
thiserror = "2.0.17"
|
||||
tokio = { version = "1.51.0", features = ["rt", "sync"] }
|
||||
tokio-util = { version = "0.7.18", features = ["rt"] }
|
||||
tracing = "0.1.41"
|
||||
url = "2.5.7"
|
||||
urlpattern = "0.6.0"
|
||||
wildcard = "0.3.0"
|
||||
zstd-sys = { version = "2.1.2", default-features = false, features = ["vendored"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = "1"
|
||||
|
||||
+4
-2
@@ -1,6 +1,6 @@
|
||||
use std::str;
|
||||
|
||||
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader, response_vary_header_names};
|
||||
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader};
|
||||
use url::Url;
|
||||
|
||||
use crate::{FetchConfig, Request};
|
||||
@@ -11,6 +11,8 @@ const SUPPORTED_CACHE_REQUEST_HEADERS: &[&str] = &[
|
||||
"accept",
|
||||
"accept-encoding",
|
||||
"accept-language",
|
||||
"available-dictionary",
|
||||
"dictionary-id",
|
||||
"referer",
|
||||
"sec-ch-ua",
|
||||
"sec-ch-ua-arch",
|
||||
@@ -38,7 +40,7 @@ pub(super) fn vary_headers_for_response(
|
||||
response_headers: &[(String, Vec<u8>)],
|
||||
) -> Option<Vec<HttpCacheVaryHeader>> {
|
||||
let mut out = Vec::new();
|
||||
for normalized_name in response_vary_header_names(response_headers)? {
|
||||
for normalized_name in crate::cache_vary_header_names(request_url, response_headers)? {
|
||||
if !supported_vary_header(&normalized_name) {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -282,10 +282,17 @@ pub(crate) fn outgoing_request_header_bytes_for_url(
|
||||
let value = if header_present(&outgoing, "range") {
|
||||
"identity".to_owned()
|
||||
} else {
|
||||
crate::content_coding::default_accept_encoding()
|
||||
if request.compression_dictionary.is_some() {
|
||||
crate::content_coding::dictionary_accept_encoding()
|
||||
} else {
|
||||
crate::content_coding::default_accept_encoding()
|
||||
}
|
||||
};
|
||||
outgoing.push(("Accept-Encoding".to_owned(), value.into_bytes()));
|
||||
}
|
||||
if let Some(dictionary) = &request.compression_dictionary {
|
||||
crate::compression_dictionary::append_request_headers(&mut outgoing, dictionary);
|
||||
}
|
||||
|
||||
if request.has_browser_identity_override() {
|
||||
append_encoded_header_if_missing(
|
||||
@@ -649,6 +656,10 @@ pub(crate) fn configure_easy(
|
||||
.context("failed to disable curl redirect following")?;
|
||||
easy.accept_encoding("")
|
||||
.context("failed to enable curl response decompression")?;
|
||||
easy.http_content_decoding(false)
|
||||
.context("failed to configure response content decoding")?;
|
||||
easy.get_mut()
|
||||
.begin_dictionary_response(request, request_url);
|
||||
|
||||
// TLS client identities are Fetch credentials. Check each redirect target;
|
||||
// curl also uses the client certificate configuration to match reusable
|
||||
|
||||
@@ -20,6 +20,7 @@ pub struct FetchClientHandle {
|
||||
config: FetchConfig,
|
||||
cookie_store: SharedBrowserCookieStore,
|
||||
runtime: FetchRuntimeHandle,
|
||||
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
|
||||
}
|
||||
|
||||
/// Unique structured-concurrency owner for one fetch client.
|
||||
@@ -39,6 +40,17 @@ impl FetchClientHandle {
|
||||
/// network admission, after the caller has selected the resource kind.
|
||||
pub fn prepare_request(&self, request: &mut Request) {
|
||||
request.prepare_accept_header(&self.config);
|
||||
let url = request.url.clone();
|
||||
if let std::borrow::Cow::Owned(prepared) =
|
||||
crate::compression_dictionary::DictionaryStore::prepare(
|
||||
&self.dictionaries,
|
||||
&self.config,
|
||||
request,
|
||||
&url,
|
||||
)
|
||||
{
|
||||
*request = prepared;
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates native WebSockets on this client's HTTP owner and Multi. The
|
||||
@@ -207,14 +219,42 @@ impl FetchClientHandle {
|
||||
/// Cookie-store and other mutable transport-only fields are not included;
|
||||
/// renderer caches must not admit Vary on those fields from this view.
|
||||
pub fn cache_request_headers(&self, request: &Request) -> crate::RequestHeaders {
|
||||
crate::blocking::outgoing_request_header_bytes_for_url(
|
||||
let prepared = crate::compression_dictionary::DictionaryStore::prepare(
|
||||
&self.dictionaries,
|
||||
&self.config,
|
||||
request,
|
||||
&request.url,
|
||||
);
|
||||
crate::blocking::outgoing_request_header_bytes_for_url(
|
||||
&self.config,
|
||||
&prepared,
|
||||
&request.url,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
/// A complete cached representation can register its dictionary again.
|
||||
/// Its retained freshness deadline must not be extended by this read.
|
||||
pub fn observe_cached_response(
|
||||
&self,
|
||||
request: &Request,
|
||||
response: &RawResponse,
|
||||
fresh_until_unix_ms: u64,
|
||||
) {
|
||||
if response
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
|
||||
{
|
||||
crate::compression_dictionary::DictionaryResponse::from_cached(
|
||||
Arc::clone(&self.dictionaries),
|
||||
request,
|
||||
response,
|
||||
fresh_until_unix_ms,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn matches_config(&self, config: &FetchConfig) -> bool {
|
||||
&self.config == config
|
||||
}
|
||||
@@ -274,15 +314,18 @@ impl FetchClient {
|
||||
}
|
||||
let client_hint_preferences =
|
||||
Arc::new(parking_lot::Mutex::new(ClientHintPreferences::default()));
|
||||
let runtime_owner = FetchRuntimeOwner::new_with_client_hint_preferences(
|
||||
let dictionaries = Arc::default();
|
||||
let runtime_owner = FetchRuntimeOwner::new_with_network_state(
|
||||
config,
|
||||
Arc::clone(&cookie_store),
|
||||
Arc::clone(&client_hint_preferences),
|
||||
Arc::clone(&dictionaries),
|
||||
);
|
||||
let handle = FetchClientHandle {
|
||||
config: config.clone(),
|
||||
runtime: runtime_owner.handle(),
|
||||
cookie_store,
|
||||
dictionaries,
|
||||
};
|
||||
Self {
|
||||
handle,
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
//! Decode response content with the linked Brotli/zlib/Zstandard libraries.
|
||||
//! The same stack validates completion and counts wire bytes for ordinary and
|
||||
//! dictionary-compressed responses, independently of libcurl's decoder build.
|
||||
|
||||
use std::{
|
||||
ffi::c_void,
|
||||
ptr::{self, NonNull},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use anyhow::{Result, anyhow, bail, ensure};
|
||||
use flate2::{Decompress, FlushDecompress, Status};
|
||||
|
||||
use super::Dictionary;
|
||||
|
||||
const OUTPUT_SIZE: usize = 16 * 1024;
|
||||
type Output<'a> = dyn FnMut(&[u8]) -> Result<()> + 'a;
|
||||
|
||||
pub(crate) struct DictionaryDecoding {
|
||||
dictionary: Option<Arc<Dictionary>>,
|
||||
stages: Option<Vec<Stage>>,
|
||||
pub(crate) encoded_bytes: u64,
|
||||
}
|
||||
|
||||
impl DictionaryDecoding {
|
||||
pub(crate) fn new(dictionary: Option<Arc<Dictionary>>) -> Self {
|
||||
Self {
|
||||
dictionary,
|
||||
stages: None,
|
||||
encoded_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&mut self) {
|
||||
self.stages = None;
|
||||
self.encoded_bytes = 0;
|
||||
}
|
||||
|
||||
pub(crate) fn uses_dictionary(&self) -> bool {
|
||||
self.dictionary.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn initialize(&mut self, headers: &[(String, Vec<u8>)]) -> Result<()> {
|
||||
if self.stages.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut names = Vec::new();
|
||||
for (_, value) in headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("content-encoding"))
|
||||
{
|
||||
let value = crate::decode_header_value(value);
|
||||
names.extend(
|
||||
crate::split_http_header_list(&value)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_ascii_lowercase),
|
||||
);
|
||||
}
|
||||
ensure!(names.len() <= 5, "too many content encodings");
|
||||
let mut stages = Vec::new();
|
||||
for name in names.into_iter().rev() {
|
||||
let stage = match name.as_str() {
|
||||
"identity" => continue,
|
||||
"gzip" | "x-gzip" => Stage::Flate(Flate::gzip()),
|
||||
"deflate" => Stage::Flate(Flate::deflate()),
|
||||
"br" => Stage::Brotli(Brotli::new(None)?),
|
||||
"zstd" => Stage::Zstd(Zstd::new(None)?),
|
||||
"dcb" | "dcz" => Stage::Dictionary {
|
||||
brotli: name == "dcb",
|
||||
dictionary: Arc::clone(self.dictionary.as_ref().ok_or_else(|| {
|
||||
anyhow!("dictionary compression without a negotiated dictionary")
|
||||
})?),
|
||||
prefix: Vec::new(),
|
||||
decoder: None,
|
||||
},
|
||||
_ => bail!("unsupported content encoding {name}"),
|
||||
};
|
||||
stages.push(stage);
|
||||
}
|
||||
self.stages = Some(stages);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
ensure!(
|
||||
self.stages.is_some(),
|
||||
"compressed response body arrived before headers"
|
||||
);
|
||||
feed(
|
||||
self.stages.as_mut().expect("initialized decoding stack"),
|
||||
input,
|
||||
output,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn finish(&mut self, headers: &[(String, Vec<u8>)], bodyless: bool) -> Result<()> {
|
||||
if bodyless {
|
||||
return Ok(());
|
||||
}
|
||||
// An empty payload never enters a content decoder. HEAD and null-body
|
||||
// responses likewise do not validate an unused coding declaration.
|
||||
if self.encoded_bytes == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
self.initialize(headers)?;
|
||||
for stage in self.stages.as_ref().expect("initialized decoding stack") {
|
||||
ensure!(stage.finished(), "truncated compressed response");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn feed(stages: &mut [Stage], input: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
let Some((stage, remaining)) = stages.split_first_mut() else {
|
||||
return output(input);
|
||||
};
|
||||
stage.push(input, &mut |bytes| feed(remaining, bytes, output))
|
||||
}
|
||||
|
||||
enum Stage {
|
||||
Flate(Flate),
|
||||
Brotli(Brotli),
|
||||
Zstd(Zstd),
|
||||
Dictionary {
|
||||
brotli: bool,
|
||||
dictionary: Arc<Dictionary>,
|
||||
prefix: Vec<u8>,
|
||||
decoder: Option<Box<Stage>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Stage {
|
||||
fn push(&mut self, mut input: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
match self {
|
||||
Self::Flate(decoder) => decoder.push(input, output),
|
||||
Self::Brotli(decoder) => decoder.push(input, output),
|
||||
Self::Zstd(decoder) => decoder.push(input, output),
|
||||
Self::Dictionary {
|
||||
brotli,
|
||||
dictionary,
|
||||
prefix,
|
||||
decoder,
|
||||
} => {
|
||||
if decoder.is_none() {
|
||||
let magic: &[u8] = if *brotli {
|
||||
b"\xffDCB"
|
||||
} else {
|
||||
b"\x5e\x2a\x4d\x18\x20\0\0\0"
|
||||
};
|
||||
let length = magic.len() + 32;
|
||||
let count = input.len().min(length - prefix.len());
|
||||
prefix.extend_from_slice(&input[..count]);
|
||||
input = &input[count..];
|
||||
if prefix.len() < length {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
&prefix[..magic.len()] == magic,
|
||||
"invalid dictionary compression header"
|
||||
);
|
||||
ensure!(
|
||||
prefix[magic.len()..] == dictionary.hash,
|
||||
"compression dictionary hash mismatch"
|
||||
);
|
||||
*decoder = Some(Box::new(if *brotli {
|
||||
Self::Brotli(Brotli::new(Some(Arc::clone(dictionary)))?)
|
||||
} else {
|
||||
Self::Zstd(Zstd::new(Some(Arc::clone(dictionary)))?)
|
||||
}));
|
||||
}
|
||||
decoder
|
||||
.as_mut()
|
||||
.expect("validated dictionary header")
|
||||
.push(input, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finished(&self) -> bool {
|
||||
match self {
|
||||
Self::Flate(decoder) => decoder.finished,
|
||||
Self::Brotli(decoder) => decoder.finished,
|
||||
Self::Zstd(decoder) => decoder.finished,
|
||||
Self::Dictionary { decoder, .. } => {
|
||||
decoder.as_ref().is_some_and(|decoder| decoder.finished())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Flate {
|
||||
state: Option<Decompress>,
|
||||
gzip: bool,
|
||||
prefix: Vec<u8>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl Flate {
|
||||
fn gzip() -> Self {
|
||||
Self {
|
||||
state: Some(Decompress::new_gzip(15)),
|
||||
gzip: true,
|
||||
prefix: Vec::new(),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
fn deflate() -> Self {
|
||||
Self {
|
||||
state: None,
|
||||
gzip: false,
|
||||
prefix: Vec::new(),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
if self.state.is_none() {
|
||||
self.prefix.extend_from_slice(input);
|
||||
if self.prefix.len() < 2 {
|
||||
return Ok(());
|
||||
}
|
||||
let wrapped = self.prefix[0] & 15 == 8
|
||||
&& u16::from_be_bytes([self.prefix[0], self.prefix[1]]).is_multiple_of(31);
|
||||
self.state = Some(Decompress::new(wrapped));
|
||||
let input = std::mem::take(&mut self.prefix);
|
||||
return self.push(&input, output);
|
||||
}
|
||||
let mut remaining = input;
|
||||
loop {
|
||||
if self.finished {
|
||||
if remaining.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(self.gzip, "bytes after a deflate stream");
|
||||
self.state = Some(Decompress::new_gzip(15));
|
||||
self.finished = false;
|
||||
}
|
||||
let decoder = self.state.as_mut().expect("initialized zlib decoder");
|
||||
let mut buffer = [0; OUTPUT_SIZE];
|
||||
let before_in = decoder.total_in();
|
||||
let before_out = decoder.total_out();
|
||||
let status = decoder.decompress(remaining, &mut buffer, FlushDecompress::None)?;
|
||||
let read = (decoder.total_in() - before_in) as usize;
|
||||
let written = (decoder.total_out() - before_out) as usize;
|
||||
remaining = &remaining[read..];
|
||||
if written != 0 {
|
||||
output(&buffer[..written])?;
|
||||
}
|
||||
self.finished = status == Status::StreamEnd;
|
||||
if self.finished && !remaining.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if (read == 0 && written == 0) || (remaining.is_empty() && written < buffer.len()) {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Brotli {
|
||||
state: NonNull<brotlic_sys::BrotliDecoderState>,
|
||||
_dictionary: Option<Arc<Dictionary>>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
// The decoder has one owner and is accessed only through &mut self. Moving it
|
||||
// to the curl owner thread does not create concurrent access to native state.
|
||||
unsafe impl Send for Brotli {}
|
||||
|
||||
impl Brotli {
|
||||
fn new(dictionary: Option<Arc<Dictionary>>) -> Result<Self> {
|
||||
let state = NonNull::new(unsafe {
|
||||
brotlic_sys::BrotliDecoderCreateInstance(None, None, ptr::null_mut())
|
||||
})
|
||||
.ok_or_else(|| anyhow!("failed to allocate Brotli decoder"))?;
|
||||
let decoder = Self {
|
||||
state,
|
||||
_dictionary: dictionary,
|
||||
finished: false,
|
||||
};
|
||||
if let Some(dictionary) = &decoder._dictionary {
|
||||
let attached = unsafe {
|
||||
brotlic_sys::BrotliDecoderSetParameter(
|
||||
state.as_ptr(),
|
||||
brotlic_sys::BrotliDecoderParameter_BROTLI_DECODER_PARAM_LARGE_WINDOW,
|
||||
1,
|
||||
);
|
||||
brotlic_sys::BrotliDecoderAttachDictionary(
|
||||
state.as_ptr(),
|
||||
brotlic_sys::BrotliSharedDictionaryType_BROTLI_SHARED_DICTIONARY_RAW,
|
||||
dictionary.bytes.len(),
|
||||
dictionary.bytes.as_ptr(),
|
||||
)
|
||||
};
|
||||
ensure!(attached != 0, "failed to attach Brotli dictionary");
|
||||
}
|
||||
Ok(decoder)
|
||||
}
|
||||
|
||||
fn push(&mut self, input: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
if input.is_empty() && self.finished {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(!self.finished, "bytes after a Brotli stream");
|
||||
let mut available_in = input.len();
|
||||
let mut next_in = input.as_ptr();
|
||||
loop {
|
||||
let mut buffer = [0; OUTPUT_SIZE];
|
||||
let mut available_out = buffer.len();
|
||||
let mut next_out = buffer.as_mut_ptr();
|
||||
let result = unsafe {
|
||||
brotlic_sys::BrotliDecoderDecompressStream(
|
||||
self.state.as_ptr(),
|
||||
&mut available_in,
|
||||
&mut next_in,
|
||||
&mut available_out,
|
||||
&mut next_out,
|
||||
ptr::null_mut(),
|
||||
)
|
||||
};
|
||||
let written = buffer.len() - available_out;
|
||||
if written != 0 {
|
||||
output(&buffer[..written])?;
|
||||
}
|
||||
match result {
|
||||
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_ERROR => {
|
||||
bail!("invalid Brotli response")
|
||||
}
|
||||
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_SUCCESS => {
|
||||
ensure!(available_in == 0, "bytes after a Brotli stream");
|
||||
self.finished = true;
|
||||
return Ok(());
|
||||
}
|
||||
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT => {
|
||||
return Ok(());
|
||||
}
|
||||
brotlic_sys::BrotliDecoderResult_BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT => {}
|
||||
_ => bail!("invalid Brotli decoder status"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Brotli {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
brotlic_sys::BrotliDecoderDestroyInstance(self.state.as_ptr());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Zstd {
|
||||
state: NonNull<zstd_sys::ZSTD_DCtx>,
|
||||
dictionary: Option<Arc<Dictionary>>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
// Like Brotli, this context has a unique owner and no concurrent native calls.
|
||||
unsafe impl Send for Zstd {}
|
||||
|
||||
impl Zstd {
|
||||
fn new(dictionary: Option<Arc<Dictionary>>) -> Result<Self> {
|
||||
let state = NonNull::new(unsafe { zstd_sys::ZSTD_createDStream() })
|
||||
.ok_or_else(|| anyhow!("failed to allocate Zstandard decoder"))?;
|
||||
let decoder = Self {
|
||||
state,
|
||||
dictionary,
|
||||
finished: false,
|
||||
};
|
||||
if let Some(dictionary) = &decoder.dictionary {
|
||||
let result = unsafe {
|
||||
zstd_sys::ZSTD_DCtx_refPrefix(
|
||||
state.as_ptr(),
|
||||
dictionary.bytes.as_ptr().cast(),
|
||||
dictionary.bytes.len(),
|
||||
)
|
||||
};
|
||||
ensure!(
|
||||
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
|
||||
"failed to attach Zstandard dictionary"
|
||||
);
|
||||
}
|
||||
Ok(decoder)
|
||||
}
|
||||
|
||||
fn push(&mut self, bytes: &[u8], output: &mut Output<'_>) -> Result<()> {
|
||||
if bytes.is_empty() && self.finished {
|
||||
return Ok(());
|
||||
}
|
||||
let mut input = zstd_sys::ZSTD_inBuffer {
|
||||
src: bytes.as_ptr().cast::<c_void>(),
|
||||
size: bytes.len(),
|
||||
pos: 0,
|
||||
};
|
||||
loop {
|
||||
if self.finished {
|
||||
if let Some(dictionary) = &self.dictionary {
|
||||
let result = unsafe {
|
||||
zstd_sys::ZSTD_DCtx_refPrefix(
|
||||
self.state.as_ptr(),
|
||||
dictionary.bytes.as_ptr().cast(),
|
||||
dictionary.bytes.len(),
|
||||
)
|
||||
};
|
||||
ensure!(
|
||||
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
|
||||
"failed to reattach Zstandard dictionary"
|
||||
);
|
||||
}
|
||||
self.finished = false;
|
||||
}
|
||||
let mut buffer = [0; OUTPUT_SIZE];
|
||||
let mut decoded = zstd_sys::ZSTD_outBuffer {
|
||||
dst: buffer.as_mut_ptr().cast::<c_void>(),
|
||||
size: buffer.len(),
|
||||
pos: 0,
|
||||
};
|
||||
let before = input.pos;
|
||||
let result = unsafe {
|
||||
zstd_sys::ZSTD_decompressStream(self.state.as_ptr(), &mut decoded, &mut input)
|
||||
};
|
||||
ensure!(
|
||||
unsafe { zstd_sys::ZSTD_isError(result) } == 0,
|
||||
"invalid Zstandard response"
|
||||
);
|
||||
if decoded.pos != 0 {
|
||||
output(&buffer[..decoded.pos])?;
|
||||
}
|
||||
self.finished = result == 0;
|
||||
if self.finished && input.pos < input.size {
|
||||
continue;
|
||||
}
|
||||
if (input.pos == input.size && decoded.pos < buffer.len())
|
||||
|| (input.pos == before && decoded.pos == 0)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Zstd {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
zstd_sys::ZSTD_freeDStream(self.state.as_ptr());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
use sfv::{BareItem, Dictionary, List, ListEntry, Parser};
|
||||
use url::Url;
|
||||
use urlpattern::{UrlPattern, UrlPatternInit, UrlPatternMatchInput};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RegistrationOptions {
|
||||
pattern: UrlPattern,
|
||||
match_text: String,
|
||||
destinations: Vec<String>,
|
||||
id: String,
|
||||
}
|
||||
|
||||
impl RegistrationOptions {
|
||||
pub(crate) fn parse(url: &Url, headers: &[(String, Vec<u8>)]) -> Option<Self> {
|
||||
let values = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
|
||||
.map(|(_, value)| std::str::from_utf8(value).ok())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let combined = values.join(", ");
|
||||
if combined.len() > 16 * 1024 {
|
||||
return None;
|
||||
}
|
||||
let dictionary: Dictionary = Parser::new(&combined).parse().ok()?;
|
||||
let ListEntry::Item(item) = dictionary.get("match")? else {
|
||||
return None;
|
||||
};
|
||||
let BareItem::String(value) = &item.bare_item else {
|
||||
return None;
|
||||
};
|
||||
let match_text = value.as_str().to_owned();
|
||||
if match_text.len() > 4096 {
|
||||
return None;
|
||||
}
|
||||
let init = UrlPatternInit::parse_constructor_string::<regex::Regex>(
|
||||
&match_text,
|
||||
Some(url.clone()),
|
||||
)
|
||||
.ok()?;
|
||||
let pattern = UrlPattern::parse(init, Default::default()).ok()?;
|
||||
if pattern.has_regexp_groups() {
|
||||
return None;
|
||||
}
|
||||
let id = match dictionary.get("id") {
|
||||
None => String::new(),
|
||||
Some(ListEntry::Item(item)) => item.bare_item.as_string()?.as_str().to_owned(),
|
||||
Some(ListEntry::InnerList(_)) => return None,
|
||||
};
|
||||
if id.len() > 1024 {
|
||||
return None;
|
||||
}
|
||||
if let Some(kind) = dictionary.get("type") {
|
||||
let ListEntry::Item(item) = kind else {
|
||||
return None;
|
||||
};
|
||||
if item.bare_item.as_token()?.as_str() != "raw" {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let destinations = match dictionary.get("match-dest") {
|
||||
None => Vec::new(),
|
||||
Some(ListEntry::InnerList(list)) => list
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| Some(item.bare_item.as_string()?.as_str().to_owned()))
|
||||
.collect::<Option<Vec<_>>>()?,
|
||||
Some(ListEntry::Item(_)) => return None,
|
||||
};
|
||||
Some(Self {
|
||||
pattern,
|
||||
match_text,
|
||||
destinations,
|
||||
id,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, url: &Url, destination: &str) -> bool {
|
||||
(self.destinations.is_empty() || self.destinations.iter().any(|value| value == destination))
|
||||
&& self
|
||||
.pattern
|
||||
.test(UrlPatternMatchInput::Url(url.clone()))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub(crate) fn match_length(&self) -> usize {
|
||||
self.match_text.len()
|
||||
}
|
||||
|
||||
pub(crate) fn same_scope(&self, other: &Self) -> bool {
|
||||
self.match_text == other.match_text && self.destinations == other.destinations
|
||||
}
|
||||
|
||||
pub(crate) fn id_header(&self) -> Option<Vec<u8>> {
|
||||
(!self.id.is_empty()).then(|| {
|
||||
format!("\"{}\"", self.id.replace('\\', "\\\\").replace('"', "\\\"")).into_bytes()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn clears_dictionaries(headers: &[(String, Vec<u8>)]) -> bool {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("clear-site-data"))
|
||||
.filter_map(|(_, value)| std::str::from_utf8(value).ok())
|
||||
.filter_map(|value| Parser::new(value).parse::<List>().ok())
|
||||
.flatten()
|
||||
.any(|entry| match entry {
|
||||
ListEntry::Item(item) => item
|
||||
.bare_item
|
||||
.as_string()
|
||||
.is_some_and(|value| matches!(value.as_str(), "cache" | "cookies" | "*")),
|
||||
ListEntry::InnerList(_) => false,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
//! Compression dictionaries are network-context state, partitioned like the
|
||||
//! requests which may use them. Bytes enter the store only after a readable
|
||||
//! response completes; a header alone cannot register a dictionary.
|
||||
|
||||
mod decoding;
|
||||
mod fields;
|
||||
|
||||
use std::{borrow::Cow, collections::VecDeque, fmt, io::Read, sync::Arc};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use moli_http_cache::{response_cache_policy, unix_now_ms};
|
||||
use moli_url::{is_potentially_trustworthy_url, origin_ascii_serialization};
|
||||
use parking_lot::Mutex;
|
||||
use sha2::{Digest, Sha256};
|
||||
use url::Url;
|
||||
|
||||
use crate::{FetchConfig, Request, RequestMode};
|
||||
|
||||
pub(crate) use decoding::DictionaryDecoding;
|
||||
pub(crate) use fields::RegistrationOptions;
|
||||
|
||||
const MAX_DICTIONARY_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_STORE_BYTES: usize = 128 * 1024 * 1024;
|
||||
const MAX_STORE_ENTRIES: usize = 256;
|
||||
|
||||
pub(crate) type SharedDictionaryStore = Arc<Mutex<DictionaryStore>>;
|
||||
|
||||
pub(crate) struct Dictionary {
|
||||
pub(crate) bytes: Arc<[u8]>,
|
||||
pub(crate) hash: [u8; 32],
|
||||
available: Vec<u8>,
|
||||
id: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Dictionary {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("Dictionary")
|
||||
.field("byte_length", &self.bytes.len())
|
||||
.field("available", &self.available)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct DictionaryRequestSnapshot {
|
||||
url: Url,
|
||||
partition: Option<Partition>,
|
||||
destination: String,
|
||||
permitted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct Partition {
|
||||
top_frame: String,
|
||||
frame: String,
|
||||
network: Option<String>,
|
||||
}
|
||||
|
||||
impl Partition {
|
||||
fn for_request(request: &Request, url: &Url) -> Option<Self> {
|
||||
let frame = match request.request_origin() {
|
||||
Some(origin) if origin.is_opaque() => return None,
|
||||
Some(origin) => origin.ascii_serialization().to_owned(),
|
||||
None => origin_ascii_serialization(url),
|
||||
};
|
||||
Some(Self {
|
||||
top_frame: request
|
||||
.cookie_context
|
||||
.browser_context
|
||||
.top_frame_origin_url
|
||||
.as_ref()
|
||||
.or(request
|
||||
.cookie_context
|
||||
.browser_context
|
||||
.site_for_cookies_url
|
||||
.as_ref())
|
||||
.map_or_else(|| frame.clone(), origin_ascii_serialization),
|
||||
frame,
|
||||
network: request.network_partition_key().map(str::to_owned),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Entry {
|
||||
partition: Partition,
|
||||
origin: String,
|
||||
options: RegistrationOptions,
|
||||
dictionary: Arc<Dictionary>,
|
||||
expires: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DictionaryStore {
|
||||
entries: VecDeque<Entry>,
|
||||
bytes: usize,
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
impl DictionaryStore {
|
||||
pub(crate) fn prepare<'a>(
|
||||
store: &SharedDictionaryStore,
|
||||
config: &FetchConfig,
|
||||
request: &'a Request,
|
||||
url: &Url,
|
||||
) -> Cow<'a, Request> {
|
||||
Self::prepare_inner(store, config, Cow::Borrowed(request), url)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_owned(
|
||||
store: &SharedDictionaryStore,
|
||||
config: &FetchConfig,
|
||||
request: Request,
|
||||
url: &Url,
|
||||
) -> Request {
|
||||
Self::prepare_inner(store, config, Cow::Owned(request), url).into_owned()
|
||||
}
|
||||
|
||||
fn prepare_inner<'a>(
|
||||
store: &SharedDictionaryStore,
|
||||
config: &FetchConfig,
|
||||
mut prepared: Cow<'a, Request>,
|
||||
url: &Url,
|
||||
) -> Cow<'a, Request> {
|
||||
let headers =
|
||||
crate::blocking::outgoing_request_header_bytes_for_url(config, &prepared, url, None);
|
||||
let destination = headers
|
||||
.iter()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("sec-fetch-dest"))
|
||||
.and_then(|(_, value)| std::str::from_utf8(value).ok())
|
||||
.unwrap_or("");
|
||||
let key = DictionaryRequestSnapshot {
|
||||
url: url.clone(),
|
||||
partition: Partition::for_request(&prepared, url),
|
||||
destination: if destination == "empty" {
|
||||
""
|
||||
} else {
|
||||
destination
|
||||
}
|
||||
.to_owned(),
|
||||
permitted: is_potentially_trustworthy_url(url)
|
||||
&& matches!(url.scheme(), "http" | "https")
|
||||
&& !(prepared.request_mode == RequestMode::NoCors
|
||||
&& prepared.has_cross_origin_url(url))
|
||||
&& !prepared
|
||||
.request_headers
|
||||
.iter()
|
||||
.any(|(name, _)| overrides_dictionary_negotiation(name))
|
||||
&& !config
|
||||
.default_request_headers()
|
||||
.iter()
|
||||
.any(|(name, _)| overrides_dictionary_negotiation(name)),
|
||||
};
|
||||
if prepared.compression_dictionary_snapshot.as_ref() == Some(&key)
|
||||
&& prepared
|
||||
.compression_dictionary_store
|
||||
.as_ref()
|
||||
.is_some_and(|previous| Arc::ptr_eq(previous, store))
|
||||
{
|
||||
return prepared;
|
||||
}
|
||||
let request = prepared.to_mut();
|
||||
request.compression_dictionary = None;
|
||||
request.compression_dictionary_store = Some(Arc::clone(store));
|
||||
request.compression_dictionary_snapshot = Some(key.clone());
|
||||
if !key.permitted {
|
||||
return prepared;
|
||||
}
|
||||
let Some(partition) = key.partition else {
|
||||
return prepared;
|
||||
};
|
||||
let origin = origin_ascii_serialization(url);
|
||||
let now = unix_now_ms();
|
||||
let mut locked = store.lock();
|
||||
locked.retain(|entry| entry.expires > now);
|
||||
let selected = locked
|
||||
.entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, entry)| {
|
||||
entry.partition == partition
|
||||
&& entry.origin == origin
|
||||
&& entry.options.matches(url, &key.destination)
|
||||
})
|
||||
.max_by_key(|(index, entry)| (entry.options.match_length(), *index))
|
||||
.map(|(index, _)| index);
|
||||
if let Some(index) = selected {
|
||||
let entry = locked
|
||||
.entries
|
||||
.remove(index)
|
||||
.expect("selected dictionary exists");
|
||||
request.compression_dictionary = Some(Arc::clone(&entry.dictionary));
|
||||
locked.entries.push_back(entry);
|
||||
}
|
||||
prepared
|
||||
}
|
||||
|
||||
fn retain(&mut self, mut predicate: impl FnMut(&Entry) -> bool) {
|
||||
self.entries.retain(|entry| predicate(entry));
|
||||
self.bytes = self
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| entry.dictionary.bytes.len())
|
||||
.sum();
|
||||
}
|
||||
|
||||
fn clear_origin(&mut self, origin: &str) {
|
||||
self.retain(|entry| entry.origin != origin);
|
||||
// Cancel captures already in progress as well as retained entries.
|
||||
self.generation = self.generation.wrapping_add(1);
|
||||
}
|
||||
|
||||
fn insert(&mut self, entry: Entry) {
|
||||
self.retain(|old| {
|
||||
!(old.partition == entry.partition
|
||||
&& old.origin == entry.origin
|
||||
&& old.options.same_scope(&entry.options))
|
||||
});
|
||||
self.bytes += entry.dictionary.bytes.len();
|
||||
self.entries.push_back(entry);
|
||||
while self.bytes > MAX_STORE_BYTES || self.entries.len() > MAX_STORE_ENTRIES {
|
||||
if let Some(old) = self.entries.pop_front() {
|
||||
self.bytes -= old.dictionary.bytes.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns one response capture, including the clear-data generation at request
|
||||
/// admission. Interrupted, opaque, stale and partial responses never commit.
|
||||
pub(crate) struct DictionaryResponse {
|
||||
store: SharedDictionaryStore,
|
||||
request: ResponsePolicy,
|
||||
url: Url,
|
||||
generation: u64,
|
||||
status: u16,
|
||||
headers: Vec<(String, Vec<u8>)>,
|
||||
capture: Option<Capture>,
|
||||
headers_finalized: bool,
|
||||
}
|
||||
|
||||
struct ResponsePolicy {
|
||||
partition: Option<Partition>,
|
||||
head: bool,
|
||||
credentials: bool,
|
||||
opaque: bool,
|
||||
cors: Option<(String, crate::RequestCredentialsMode)>,
|
||||
}
|
||||
|
||||
impl ResponsePolicy {
|
||||
fn new(request: &Request, url: &Url) -> Self {
|
||||
Self {
|
||||
partition: Partition::for_request(request, url),
|
||||
head: request.method == "HEAD",
|
||||
credentials: request.allows_credentials_for_url(url),
|
||||
opaque: request.request_mode == RequestMode::NoCors
|
||||
&& request.has_cross_origin_url(url),
|
||||
cors: request
|
||||
.request_origin()
|
||||
.filter(|_| {
|
||||
request.request_mode == RequestMode::Cors && request.has_cross_origin_url(url)
|
||||
})
|
||||
.map(|origin| {
|
||||
(
|
||||
request.url_list(url).serialized_origin(origin),
|
||||
request.credentials_mode,
|
||||
)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn readable(&self, headers: &[(String, Vec<u8>)]) -> bool {
|
||||
!self.opaque
|
||||
&& self.cors.as_ref().is_none_or(|(origin, mode)| {
|
||||
crate::validate_cors_response_for_origin(origin, headers, *mode).is_ok()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Capture {
|
||||
options: RegistrationOptions,
|
||||
partition: Partition,
|
||||
expires: u64,
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl DictionaryResponse {
|
||||
pub(crate) fn new(store: SharedDictionaryStore, request: &Request, url: Url) -> Self {
|
||||
let generation = store.lock().generation;
|
||||
Self {
|
||||
store,
|
||||
request: ResponsePolicy::new(request, &url),
|
||||
url,
|
||||
generation,
|
||||
status: 0,
|
||||
headers: Vec::new(),
|
||||
capture: None,
|
||||
headers_finalized: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn header(&mut self, data: &[u8]) {
|
||||
if let Some(status) = crate::response::parse_http_response_status_line(data) {
|
||||
self.status = status.status;
|
||||
self.headers.clear();
|
||||
self.capture = None;
|
||||
self.headers_finalized = false;
|
||||
return;
|
||||
}
|
||||
if self.headers_finalized {
|
||||
return;
|
||||
}
|
||||
if let Some(header) = crate::headers::parse_http_response_header_line(data) {
|
||||
self.headers.push(header);
|
||||
return;
|
||||
}
|
||||
if data.iter().any(|byte| !matches!(byte, b'\r' | b'\n')) || self.status < 200 {
|
||||
return;
|
||||
}
|
||||
self.headers_finalized = true;
|
||||
if !is_potentially_trustworthy_url(&self.url) {
|
||||
return;
|
||||
}
|
||||
if fields::clears_dictionaries(&self.headers) && self.request.credentials {
|
||||
self.store
|
||||
.lock()
|
||||
.clear_origin(&origin_ascii_serialization(&self.url));
|
||||
}
|
||||
self.begin_capture(None);
|
||||
}
|
||||
|
||||
fn begin_capture(&mut self, fresh_until: Option<u64>) {
|
||||
if self.status != 200 || self.request.head || !self.request.readable(&self.headers) {
|
||||
return;
|
||||
}
|
||||
let Some(partition) = self.request.partition.clone() else {
|
||||
return;
|
||||
};
|
||||
let Some(options) = RegistrationOptions::parse(&self.url, &self.headers) else {
|
||||
return;
|
||||
};
|
||||
let policy = response_cache_policy(&self.headers);
|
||||
let Some(mut expires) = policy
|
||||
.expires_at_unix_ms
|
||||
.filter(|expires| *expires > unix_now_ms())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if let Some(deadline) = fresh_until {
|
||||
expires = expires.min(deadline);
|
||||
}
|
||||
if expires <= unix_now_ms() {
|
||||
return;
|
||||
}
|
||||
if !policy.store {
|
||||
return;
|
||||
}
|
||||
self.capture = Some(Capture {
|
||||
options,
|
||||
partition,
|
||||
expires,
|
||||
bytes: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn decoded(&mut self, bytes: &[u8]) {
|
||||
let Some(capture) = self.capture.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if bytes.len() > MAX_DICTIONARY_BYTES.saturating_sub(capture.bytes.len()) {
|
||||
self.capture = None;
|
||||
} else {
|
||||
capture.bytes.extend_from_slice(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn headers(&self) -> &[(String, Vec<u8>)] {
|
||||
&self.headers
|
||||
}
|
||||
|
||||
pub(crate) fn headers_finalized(&self) -> bool {
|
||||
self.headers_finalized
|
||||
}
|
||||
|
||||
pub(crate) fn bodyless(&self) -> bool {
|
||||
self.request.head || matches!(self.status, 204 | 205 | 304)
|
||||
}
|
||||
|
||||
pub(crate) fn from_cached(
|
||||
store: SharedDictionaryStore,
|
||||
request: &Request,
|
||||
response: &crate::RawResponse,
|
||||
fresh_until: u64,
|
||||
) {
|
||||
let Some(mut capture) = Self::cached(
|
||||
store,
|
||||
request,
|
||||
&response.final_url,
|
||||
response.status,
|
||||
&response.headers,
|
||||
Some(fresh_until),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
capture.decoded(response.body_bytes());
|
||||
capture.finish();
|
||||
}
|
||||
|
||||
pub(crate) fn cached(
|
||||
store: SharedDictionaryStore,
|
||||
request: &Request,
|
||||
url: &Url,
|
||||
status: u16,
|
||||
headers: &[(String, Vec<u8>)],
|
||||
fresh_until: Option<u64>,
|
||||
) -> Option<Self> {
|
||||
if !is_potentially_trustworthy_url(url)
|
||||
|| !headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("use-as-dictionary"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let deadline = fresh_until.filter(|deadline| *deadline > unix_now_ms())?;
|
||||
let mut capture = Self::new(store, request, url.clone());
|
||||
capture.status = status;
|
||||
capture.headers = headers.to_vec();
|
||||
capture.begin_capture(Some(deadline));
|
||||
capture.capture.as_ref()?;
|
||||
Some(capture)
|
||||
}
|
||||
|
||||
pub(crate) fn readable(&self) -> bool {
|
||||
self.request.readable(&self.headers)
|
||||
}
|
||||
|
||||
pub(crate) fn finish(&mut self) {
|
||||
let Some(capture) = self.capture.take() else {
|
||||
return;
|
||||
};
|
||||
if capture.expires <= unix_now_ms() {
|
||||
return;
|
||||
}
|
||||
let hash: [u8; 32] = Sha256::digest(&capture.bytes).into();
|
||||
let available = format!(":{}:", STANDARD.encode(hash)).into_bytes();
|
||||
let dictionary = Arc::new(Dictionary {
|
||||
bytes: capture.bytes.into(),
|
||||
hash,
|
||||
available,
|
||||
id: capture.options.id_header(),
|
||||
});
|
||||
let mut store = self.store.lock();
|
||||
if store.generation != self.generation {
|
||||
return;
|
||||
}
|
||||
store.insert(Entry {
|
||||
partition: capture.partition,
|
||||
origin: origin_ascii_serialization(&self.url),
|
||||
options: capture.options,
|
||||
dictionary,
|
||||
expires: capture.expires,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DictionaryCachedReader<R> {
|
||||
inner: R,
|
||||
capture: Option<DictionaryResponse>,
|
||||
cancel: crate::FetchCancelHandle,
|
||||
}
|
||||
|
||||
impl<R> DictionaryCachedReader<R> {
|
||||
pub(crate) fn new(
|
||||
inner: R,
|
||||
capture: Option<DictionaryResponse>,
|
||||
cancel: crate::FetchCancelHandle,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
capture,
|
||||
cancel,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for DictionaryCachedReader<R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
let count = self.inner.read(buffer)?;
|
||||
if buffer.is_empty() {
|
||||
return Ok(count);
|
||||
}
|
||||
if let Some(capture) = self.capture.as_mut() {
|
||||
if count == 0 {
|
||||
if !self.cancel.is_cancelled() {
|
||||
capture.finish();
|
||||
}
|
||||
self.capture = None;
|
||||
} else {
|
||||
capture.decoded(&buffer[..count]);
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
fn overrides_dictionary_negotiation(name: &str) -> bool {
|
||||
["range", "available-dictionary", "dictionary-id"]
|
||||
.iter()
|
||||
.any(|value| name.eq_ignore_ascii_case(value))
|
||||
}
|
||||
|
||||
pub(crate) fn append_request_headers(headers: &mut crate::RequestHeaders, dictionary: &Dictionary) {
|
||||
headers.push((
|
||||
"Available-Dictionary".to_owned(),
|
||||
dictionary.available.clone(),
|
||||
));
|
||||
if let Some(id) = &dictionary.id {
|
||||
headers.push(("Dictionary-ID".to_owned(), id.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Dictionary representations remain reusable after registration changes the
|
||||
/// browser-generated negotiation fields. Other Vary fields remain in force.
|
||||
pub fn cache_vary_header_names(url: &Url, headers: &[(String, Vec<u8>)]) -> Option<Vec<String>> {
|
||||
let mut names = moli_http_cache::response_vary_header_names(headers)?;
|
||||
if is_potentially_trustworthy_url(url) && RegistrationOptions::parse(url, headers).is_some() {
|
||||
names.retain(|name| name != "available-dictionary" && name != "accept-encoding");
|
||||
}
|
||||
Some(names)
|
||||
}
|
||||
@@ -1,21 +1,16 @@
|
||||
use moli_header_field::{HeaderFieldTokenMode, HeaderFieldTokenizer};
|
||||
|
||||
fn supported_content_codings() -> impl Iterator<Item = &'static str> {
|
||||
let version = curl::Version::get();
|
||||
[
|
||||
("deflate", version.feature_libz()),
|
||||
("gzip", version.feature_libz()),
|
||||
("br", version.feature_brotli()),
|
||||
("zstd", version.feature_zstd()),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(coding, supported)| supported.then_some(coding))
|
||||
["deflate", "gzip", "br", "zstd", "dcb", "dcz"].into_iter()
|
||||
}
|
||||
|
||||
/// Choose the browser's header before cache lookup and transport setup so
|
||||
/// negotiation and Vary matching use the same actual value.
|
||||
pub(crate) fn default_accept_encoding() -> String {
|
||||
let value = supported_content_codings().collect::<Vec<_>>().join(", ");
|
||||
let value = supported_content_codings()
|
||||
.filter(|coding| !matches!(*coding, "dcb" | "dcz"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
if value.is_empty() {
|
||||
"identity".to_owned()
|
||||
} else {
|
||||
@@ -23,6 +18,10 @@ pub(crate) fn default_accept_encoding() -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn dictionary_accept_encoding() -> String {
|
||||
supported_content_codings().collect::<Vec<_>>().join(", ")
|
||||
}
|
||||
|
||||
/// Resource Timing exposes registered, supported content codings rather than
|
||||
/// reflecting an arbitrary response header into the timing entry.
|
||||
pub fn resource_timing_content_encoding(headers: &[(String, Vec<u8>)]) -> &'static str {
|
||||
|
||||
@@ -9,6 +9,7 @@ mod blocking;
|
||||
mod cancellation;
|
||||
mod client;
|
||||
mod client_hints;
|
||||
mod compression_dictionary;
|
||||
mod config;
|
||||
mod content_coding;
|
||||
mod cors;
|
||||
@@ -43,6 +44,7 @@ pub use blocking::{
|
||||
};
|
||||
pub use cancellation::{FetchCancelHandle, FetchResponseBodySizes, FetchResponseTiming};
|
||||
pub use client::{FetchClient, FetchClientHandle};
|
||||
pub use compression_dictionary::cache_vary_header_names;
|
||||
pub use config::FetchConfig;
|
||||
pub use content_coding::resource_timing_content_encoding;
|
||||
pub use cors::validate_cors_response_for_origin;
|
||||
|
||||
@@ -112,6 +112,11 @@ pub struct Request {
|
||||
redirect_policy: Option<RequestRedirectPolicy>,
|
||||
upload_observer: Option<crate::UploadObserver>,
|
||||
redirect_check: Option<RequestRedirectCheck>,
|
||||
pub(crate) compression_dictionary: Option<Arc<crate::compression_dictionary::Dictionary>>,
|
||||
pub(crate) compression_dictionary_store:
|
||||
Option<crate::compression_dictionary::SharedDictionaryStore>,
|
||||
pub(crate) compression_dictionary_snapshot:
|
||||
Option<crate::compression_dictionary::DictionaryRequestSnapshot>,
|
||||
}
|
||||
|
||||
/// Request-owned admission check, run before following each redirect target.
|
||||
@@ -272,6 +277,7 @@ pub enum BrowserRequestMetadata {
|
||||
Audio,
|
||||
AudioWorklet,
|
||||
Beacon,
|
||||
Dictionary,
|
||||
EventSource,
|
||||
Fetch,
|
||||
Font,
|
||||
@@ -309,7 +315,12 @@ impl BrowserRequestMetadata {
|
||||
Self::Style | Self::StyleModule => "style",
|
||||
Self::TextTrack => "track",
|
||||
Self::Video => "video",
|
||||
Self::Beacon | Self::EventSource | Self::Fetch | Self::Ping | Self::Xhr => "empty",
|
||||
Self::Beacon
|
||||
| Self::Dictionary
|
||||
| Self::EventSource
|
||||
| Self::Fetch
|
||||
| Self::Ping
|
||||
| Self::Xhr => "empty",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -665,6 +676,9 @@ impl Request {
|
||||
redirect_policy: None,
|
||||
upload_observer: None,
|
||||
redirect_check: None,
|
||||
compression_dictionary: None,
|
||||
compression_dictionary_store: None,
|
||||
compression_dictionary_snapshot: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -707,6 +721,9 @@ impl Request {
|
||||
redirect_policy: None,
|
||||
upload_observer: None,
|
||||
redirect_check: None,
|
||||
compression_dictionary: None,
|
||||
compression_dictionary_store: None,
|
||||
compression_dictionary_snapshot: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -782,6 +799,9 @@ impl Request {
|
||||
redirect_policy: None,
|
||||
upload_observer: None,
|
||||
redirect_check: None,
|
||||
compression_dictionary: None,
|
||||
compression_dictionary_store: None,
|
||||
compression_dictionary_snapshot: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -995,6 +995,8 @@ mod resource_timing_tests {
|
||||
(vec!["deflate"], "deflate"),
|
||||
(vec!["br"], "br"),
|
||||
(vec!["zStd "], "zstd"),
|
||||
(vec!["dcb"], "dcb"),
|
||||
(vec!["dcz"], "dcz"),
|
||||
(vec!["gzip, deflate, Apple"], "multiple"),
|
||||
(vec!["gzip, "], "multiple"),
|
||||
(vec!["gzip", "deflate"], "multiple"),
|
||||
|
||||
+219
-21
@@ -370,19 +370,21 @@ impl PendingStreamingRawResponse {
|
||||
impl FetchRuntimeOwner {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new(config: &FetchConfig, cookie_store: SharedBrowserCookieStore) -> Self {
|
||||
Self::new_with_client_hint_preferences(
|
||||
Self::new_with_network_state(
|
||||
config,
|
||||
cookie_store,
|
||||
Arc::new(Mutex::new(
|
||||
crate::client_hints::ClientHintPreferences::default(),
|
||||
)),
|
||||
Arc::default(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_client_hint_preferences(
|
||||
pub(crate) fn new_with_network_state(
|
||||
config: &FetchConfig,
|
||||
cookie_store: SharedBrowserCookieStore,
|
||||
client_hint_preferences: SharedClientHintPreferences,
|
||||
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
|
||||
) -> Self {
|
||||
let runtime_id = NEXT_FETCH_RUNTIME_ID
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
@@ -401,6 +403,7 @@ impl FetchRuntimeOwner {
|
||||
config: config.clone(),
|
||||
cookie_store,
|
||||
client_hint_preferences,
|
||||
dictionaries,
|
||||
request_rx,
|
||||
curl_runtime,
|
||||
curl_http,
|
||||
@@ -711,6 +714,7 @@ struct RuntimeOwner {
|
||||
config: FetchConfig,
|
||||
cookie_store: SharedBrowserCookieStore,
|
||||
client_hint_preferences: SharedClientHintPreferences,
|
||||
dictionaries: crate::compression_dictionary::SharedDictionaryStore,
|
||||
request_rx: Receiver<RuntimeCommand>,
|
||||
curl_runtime: CurlMultiRuntime<FetchTransferHandler, ActiveTransferContext>,
|
||||
curl_http: CurlHttpSender<FetchTransferHandler, ActiveTransferContext>,
|
||||
@@ -721,6 +725,27 @@ struct RuntimeOwner {
|
||||
}
|
||||
|
||||
impl RuntimeOwner {
|
||||
fn prepare_request(
|
||||
&self,
|
||||
restarts: &SharedNavigationClientHintRestarts,
|
||||
request: &Request,
|
||||
url: &Url,
|
||||
) -> crate::client_hints::PreparedClientHintRequest {
|
||||
let mut prepared = prepare_client_hint_request(
|
||||
&self.client_hint_preferences,
|
||||
restarts,
|
||||
&self.config,
|
||||
request,
|
||||
url,
|
||||
);
|
||||
prepared.request = crate::compression_dictionary::DictionaryStore::prepare_owned(
|
||||
&self.dictionaries,
|
||||
&self.config,
|
||||
prepared.request,
|
||||
url,
|
||||
);
|
||||
prepared
|
||||
}
|
||||
fn run(self) {
|
||||
#[cfg(test)]
|
||||
self.owner_started.store(true, Ordering::SeqCst);
|
||||
@@ -900,10 +925,8 @@ impl RuntimeOwner {
|
||||
None
|
||||
};
|
||||
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
|
||||
let prepared_request = prepare_client_hint_request(
|
||||
&self.client_hint_preferences,
|
||||
let prepared_request = self.prepare_request(
|
||||
&job.client_hint_navigation_restarts,
|
||||
&self.config,
|
||||
&job.request,
|
||||
&job.current_url,
|
||||
);
|
||||
@@ -1105,10 +1128,8 @@ impl RuntimeOwner {
|
||||
None
|
||||
};
|
||||
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
|
||||
let prepared_request = prepare_client_hint_request(
|
||||
&self.client_hint_preferences,
|
||||
let prepared_request = self.prepare_request(
|
||||
&job.client_hint_navigation_restarts,
|
||||
&self.config,
|
||||
&job.request,
|
||||
&job.current_url,
|
||||
);
|
||||
@@ -1349,10 +1370,8 @@ impl RuntimeOwner {
|
||||
None
|
||||
};
|
||||
let cookie_header = cookie_header_from_report(request_cookie_report.as_ref());
|
||||
let prepared_request = prepare_client_hint_request(
|
||||
&self.client_hint_preferences,
|
||||
let prepared_request = self.prepare_request(
|
||||
&job.client_hint_navigation_restarts,
|
||||
&self.config,
|
||||
&job.request,
|
||||
&job.current_url,
|
||||
);
|
||||
@@ -1535,7 +1554,28 @@ impl RuntimeOwner {
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_active_transfer(&self, state: &mut OwnerState, completion: RuntimeCurlCompletion) {
|
||||
fn finish_active_transfer(
|
||||
&self,
|
||||
state: &mut OwnerState,
|
||||
mut completion: RuntimeCurlCompletion,
|
||||
) {
|
||||
let cancelled = match &completion.context {
|
||||
ActiveTransferContext::Buffered(context) => context.job.cancel_handle.is_cancelled(),
|
||||
ActiveTransferContext::Streaming(context) => context.job.cancel_handle.is_cancelled(),
|
||||
ActiveTransferContext::StreamingRaw(context) => {
|
||||
context.job.cancel_handle.is_cancelled()
|
||||
}
|
||||
};
|
||||
if let Some(easy) = completion.easy.as_mut() {
|
||||
if let Some(error) = easy.get_mut().dictionary_error.take() {
|
||||
completion.result = Err(anyhow!(error));
|
||||
} else if completion.result.is_ok()
|
||||
&& !self.shutdown_requested.load(Ordering::SeqCst)
|
||||
&& !cancelled
|
||||
{
|
||||
completion.result = easy.get_mut().finish_dictionary_response();
|
||||
}
|
||||
}
|
||||
state.active_transfers = state.active_transfers.saturating_sub(1);
|
||||
match completion.context {
|
||||
ActiveTransferContext::Buffered(context) => {
|
||||
@@ -2612,6 +2652,7 @@ impl RuntimeOwner {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries));
|
||||
complete_cached_streaming_html_job(job, cached, request_cookie_report);
|
||||
Ok(())
|
||||
}
|
||||
@@ -2674,6 +2715,7 @@ impl RuntimeOwner {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
job.request.compression_dictionary_store = Some(Arc::clone(&self.dictionaries));
|
||||
complete_cached_streaming_raw_job(job, cached, request_cookie_report);
|
||||
Ok(())
|
||||
}
|
||||
@@ -2898,6 +2940,9 @@ pub(crate) struct FetchTransferHandler {
|
||||
network_observation_recorder: Option<NetworkObservationRecorder>,
|
||||
upload_observer: Option<crate::UploadObserver>,
|
||||
proxy_connect_response_recorder: ProxyConnectResponseRecorder,
|
||||
dictionary_response: Option<crate::compression_dictionary::DictionaryResponse>,
|
||||
dictionary_decoding: Option<crate::compression_dictionary::DictionaryDecoding>,
|
||||
dictionary_error: Option<String>,
|
||||
}
|
||||
|
||||
enum FetchResponseCollector {
|
||||
@@ -2943,6 +2988,9 @@ impl FetchTransferHandler {
|
||||
network_observation_recorder: None,
|
||||
upload_observer: None,
|
||||
proxy_connect_response_recorder: ProxyConnectResponseRecorder::default(),
|
||||
dictionary_response: None,
|
||||
dictionary_decoding: None,
|
||||
dictionary_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3001,6 +3049,31 @@ impl FetchTransferHandler {
|
||||
.begin_transfer(capture_proxy_connect_response);
|
||||
}
|
||||
|
||||
pub(crate) fn begin_dictionary_response(&mut self, request: &Request, url: &Url) {
|
||||
self.dictionary_response = request.compression_dictionary_store.as_ref().map(|store| {
|
||||
crate::compression_dictionary::DictionaryResponse::new(
|
||||
Arc::clone(store),
|
||||
request,
|
||||
url.clone(),
|
||||
)
|
||||
});
|
||||
self.dictionary_decoding = Some(crate::compression_dictionary::DictionaryDecoding::new(
|
||||
request.compression_dictionary.clone(),
|
||||
));
|
||||
self.dictionary_error = None;
|
||||
}
|
||||
|
||||
fn finish_dictionary_response(&mut self) -> Result<()> {
|
||||
let Some(response) = self.dictionary_response.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(decoding) = self.dictionary_decoding.as_mut() {
|
||||
decoding.finish(response.headers(), response.bodyless())?;
|
||||
}
|
||||
response.finish();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn take_failed_proxy_connect_response(
|
||||
&mut self,
|
||||
connect_status: u32,
|
||||
@@ -3035,17 +3108,101 @@ impl Handler for FetchTransferHandler {
|
||||
if !data.is_empty() {
|
||||
self.mark_response_body_received();
|
||||
}
|
||||
if let Some(timing) = self.response_progress() {
|
||||
timing.record_decoded_response_body_size(data.len());
|
||||
if self.dictionary_decoding.is_none() {
|
||||
if let Some(timing) = self.response_progress() {
|
||||
timing.record_decoded_response_body_size(data.len());
|
||||
}
|
||||
let result = match &mut self.response {
|
||||
FetchResponseCollector::Buffered(collector) => collector.write(data),
|
||||
FetchResponseCollector::Streaming(collector) => collector.write(data),
|
||||
FetchResponseCollector::StreamingRaw(collector) => collector.write(data),
|
||||
};
|
||||
if result
|
||||
.as_ref()
|
||||
.is_ok_and(|accepted| *accepted == data.len())
|
||||
&& let Some(policy) = self.dictionary_response.as_mut()
|
||||
{
|
||||
policy.decoded(data);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
match &mut self.response {
|
||||
FetchResponseCollector::Buffered(collector) => collector.write(data),
|
||||
FetchResponseCollector::Streaming(collector) => collector.write(data),
|
||||
FetchResponseCollector::StreamingRaw(collector) => collector.write(data),
|
||||
if self.dictionary_error.is_some() {
|
||||
return Ok(0);
|
||||
}
|
||||
let timing = self.response_progress().cloned();
|
||||
let decoding = self
|
||||
.dictionary_decoding
|
||||
.as_mut()
|
||||
.expect("content decoding selected");
|
||||
decoding.encoded_bytes = decoding.encoded_bytes.saturating_add(data.len() as u64);
|
||||
if let Some(timing) = &timing {
|
||||
timing.record_encoded_response_body_size(decoding.encoded_bytes);
|
||||
}
|
||||
if let Err(error) = decoding.initialize(
|
||||
self.dictionary_response
|
||||
.as_ref()
|
||||
.map_or(&[], |response| response.headers()),
|
||||
) {
|
||||
self.dictionary_error = Some(error.to_string());
|
||||
return Ok(0);
|
||||
}
|
||||
let policy = &mut self.dictionary_response;
|
||||
let response = &mut self.response;
|
||||
let mut output = |bytes: &[u8]| -> Result<()> {
|
||||
if let Some(timing) = &timing {
|
||||
timing.record_decoded_response_body_size(bytes.len());
|
||||
}
|
||||
let accepted = match response {
|
||||
FetchResponseCollector::Buffered(collector) => collector.write(bytes),
|
||||
FetchResponseCollector::Streaming(collector) => collector.write(bytes),
|
||||
FetchResponseCollector::StreamingRaw(collector) => collector.write(bytes),
|
||||
}
|
||||
.map_err(|_| anyhow!("response collector rejected decoded bytes"))?;
|
||||
anyhow::ensure!(
|
||||
accepted == bytes.len(),
|
||||
"response collector rejected decoded bytes"
|
||||
);
|
||||
if let Some(policy) = policy.as_mut() {
|
||||
policy.decoded(bytes);
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = self
|
||||
.dictionary_decoding
|
||||
.as_mut()
|
||||
.expect("dictionary decoding selected")
|
||||
.push(data, &mut output);
|
||||
match result {
|
||||
Ok(()) => Ok(data.len()),
|
||||
Err(error) => {
|
||||
self.dictionary_error = Some(error.to_string());
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn header(&mut self, data: &[u8]) -> bool {
|
||||
if crate::response::parse_http_response_status_line(data)
|
||||
.is_some_and(|line| line.status >= 200)
|
||||
&& let Some(decoding) = self.dictionary_decoding.as_mut()
|
||||
{
|
||||
decoding.reset();
|
||||
self.dictionary_error = None;
|
||||
}
|
||||
if let Some(response) = self.dictionary_response.as_mut() {
|
||||
response.header(data);
|
||||
}
|
||||
if data.iter().all(|byte| matches!(byte, b'\r' | b'\n'))
|
||||
&& let Some(response) = self.dictionary_response.as_ref()
|
||||
&& response.headers_finalized()
|
||||
&& !response.bodyless()
|
||||
&& let Some(decoding) = self.dictionary_decoding.as_mut()
|
||||
&& decoding.uses_dictionary()
|
||||
&& !response.readable()
|
||||
{
|
||||
self.dictionary_error =
|
||||
Some("dictionary response failed its CORS readability check".to_owned());
|
||||
}
|
||||
if crate::response::parse_http_response_status_line(data)
|
||||
.is_some_and(|line| line.status >= 200)
|
||||
&& let Some(timing) = self.response_progress()
|
||||
@@ -3251,7 +3408,7 @@ fn complete_cached_streaming_html_job(
|
||||
final_url,
|
||||
status,
|
||||
headers,
|
||||
mut body,
|
||||
body,
|
||||
..
|
||||
} = cached;
|
||||
let final_url = match Url::parse(&final_url) {
|
||||
@@ -3270,9 +3427,29 @@ fn complete_cached_streaming_html_job(
|
||||
}
|
||||
};
|
||||
|
||||
let capture = job
|
||||
.request
|
||||
.compression_dictionary_store
|
||||
.as_ref()
|
||||
.and_then(|store| {
|
||||
crate::compression_dictionary::DictionaryResponse::cached(
|
||||
Arc::clone(store),
|
||||
&job.request,
|
||||
&final_url,
|
||||
status,
|
||||
&headers,
|
||||
metadata.expires_at_unix_ms,
|
||||
)
|
||||
});
|
||||
let mut body = crate::compression_dictionary::DictionaryCachedReader::new(
|
||||
body,
|
||||
capture,
|
||||
job.cancel_handle.clone(),
|
||||
);
|
||||
let response_timing = job
|
||||
.response_progress
|
||||
.mark_cached_response_available(body_sizes);
|
||||
|
||||
if let Some(started_tx) = job.started_tx {
|
||||
let _ = started_tx.send(Ok(StreamingHtmlResponseStart {
|
||||
status_text: metadata.status_text,
|
||||
@@ -3381,7 +3558,7 @@ fn complete_cached_streaming_raw_job(
|
||||
final_url,
|
||||
status,
|
||||
headers,
|
||||
mut body,
|
||||
body,
|
||||
..
|
||||
} = cached;
|
||||
let final_url = match Url::parse(&final_url) {
|
||||
@@ -3400,6 +3577,25 @@ fn complete_cached_streaming_raw_job(
|
||||
}
|
||||
};
|
||||
|
||||
let capture = job
|
||||
.request
|
||||
.compression_dictionary_store
|
||||
.as_ref()
|
||||
.and_then(|store| {
|
||||
crate::compression_dictionary::DictionaryResponse::cached(
|
||||
Arc::clone(store),
|
||||
&job.request,
|
||||
&final_url,
|
||||
status,
|
||||
&headers,
|
||||
metadata.expires_at_unix_ms,
|
||||
)
|
||||
});
|
||||
let mut body = crate::compression_dictionary::DictionaryCachedReader::new(
|
||||
body,
|
||||
capture,
|
||||
job.cancel_handle.clone(),
|
||||
);
|
||||
let response_timing = job
|
||||
.response_progress
|
||||
.mark_cached_response_available(body_sizes);
|
||||
@@ -3474,6 +3670,7 @@ fn request_fetch_load_priority(request: &Request) -> crate::ResourceLoadPriority
|
||||
Some(
|
||||
crate::BrowserRequestMetadata::Audio
|
||||
| crate::BrowserRequestMetadata::Beacon
|
||||
| crate::BrowserRequestMetadata::Dictionary
|
||||
| crate::BrowserRequestMetadata::Font
|
||||
| crate::BrowserRequestMetadata::Image
|
||||
| crate::BrowserRequestMetadata::Ping
|
||||
@@ -4430,7 +4627,8 @@ mod tests {
|
||||
.with_resource_type(crate::RequestResourceType::LinkPrefetch);
|
||||
let dictionary = Request::new("GET", "https://example.test/dict.bin", None, Vec::new())
|
||||
.expect("test request should parse")
|
||||
.with_resource_type(crate::RequestResourceType::Dictionary);
|
||||
.with_resource_type(crate::RequestResourceType::Dictionary)
|
||||
.with_browser_request_metadata(crate::BrowserRequestMetadata::Dictionary);
|
||||
|
||||
assert_eq!(
|
||||
request_fetch_load_priority(&stylesheet),
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
use super::*;
|
||||
|
||||
const DICTIONARY: &[u8] = b"This is a test dictionary.\n";
|
||||
const PAYLOAD: &[u8] = b"This is compressed test data using a test dictionary";
|
||||
const HASH: &str = ":U5abz16WDg7b8KS93msLPpOB4Vbef1uRzoORYkJw9BY=:";
|
||||
const DCB: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcb");
|
||||
const DCZ: &[u8] = include_bytes!("../../tests/fixtures/compression-dictionary/payload.dcz");
|
||||
|
||||
fn reply(headers: &[(&str, &str)], body: &[u8], chunked: bool) -> Vec<u8> {
|
||||
let mut head =
|
||||
"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain\r\n".to_owned();
|
||||
for (name, value) in headers {
|
||||
head.push_str(&format!("{name}: {value}\r\n"));
|
||||
}
|
||||
if chunked {
|
||||
head.push_str("Transfer-Encoding: chunked\r\n\r\n");
|
||||
} else {
|
||||
head.push_str(&format!("Content-Length: {}\r\n\r\n", body.len()));
|
||||
}
|
||||
let mut response = head.into_bytes();
|
||||
if chunked {
|
||||
for byte in body {
|
||||
response.extend_from_slice(&[b'1', b'\r', b'\n', *byte, b'\r', b'\n']);
|
||||
}
|
||||
response.extend_from_slice(b"0\r\nX-Trailer: retained\r\n\r\n");
|
||||
} else {
|
||||
response.extend_from_slice(body);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
async fn server(
|
||||
responses: Vec<Vec<u8>>,
|
||||
) -> Result<(String, tokio::task::JoinHandle<Result<Vec<Vec<u8>>>>)> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let url = format!("http://{}/data", listener.local_addr()?);
|
||||
let task = tokio::spawn(async move {
|
||||
let mut requests = Vec::new();
|
||||
for response in responses {
|
||||
let (mut stream, _) =
|
||||
tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
|
||||
let mut request = Vec::new();
|
||||
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
|
||||
let mut buffer = [0; 4096];
|
||||
let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer))
|
||||
.await??;
|
||||
anyhow::ensure!(count != 0 && request.len() < 65536, "incomplete request");
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
requests.push(request);
|
||||
// Error responses may make the client close the socket early.
|
||||
let _ = stream.write_all(&response).await;
|
||||
let _ = stream.shutdown().await;
|
||||
}
|
||||
Ok(requests)
|
||||
});
|
||||
Ok((url, task))
|
||||
}
|
||||
|
||||
fn header<'a>(request: &'a [u8], name: &str) -> Option<&'a str> {
|
||||
std::str::from_utf8(request)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(field, _)| field.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.trim())
|
||||
}
|
||||
|
||||
async fn fetch(client: &FetchClientHandle, url: &str, mode: &str) -> Result<Vec<u8>> {
|
||||
let request = Request::new("GET", url, None, crate::RequestHeaders::default())?;
|
||||
let (_, body) = match mode {
|
||||
"buffered" => client
|
||||
.fetch(request.with_auth(crate::RequestAuth {
|
||||
target: crate::RequestAuthTarget::Server,
|
||||
scheme: crate::RequestAuthScheme::Digest,
|
||||
username: "user".to_owned(),
|
||||
password: "password".to_owned(),
|
||||
}))
|
||||
.await?
|
||||
.into_body(),
|
||||
"html" => client.fetch_html_stream(request).await?.into_body(),
|
||||
"raw" => client
|
||||
.fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
|
||||
.await?
|
||||
.into_body(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
body.into_materialized_bytes().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_registration_and_decoding_cover_every_transport() -> Result<()> {
|
||||
for mode in ["raw", "html", "buffered"] {
|
||||
for chunked in [false, true] {
|
||||
let coding = [
|
||||
("dcb", DCB, PAYLOAD),
|
||||
("dcz", DCZ, PAYLOAD),
|
||||
(
|
||||
"gzip",
|
||||
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.gzip")
|
||||
.as_slice(),
|
||||
DICTIONARY,
|
||||
),
|
||||
(
|
||||
"deflate",
|
||||
include_bytes!(
|
||||
"../../tests/fixtures/compression-dictionary/dictionary.deflate"
|
||||
)
|
||||
.as_slice(),
|
||||
DICTIONARY,
|
||||
),
|
||||
(
|
||||
"deflate",
|
||||
include_bytes!(
|
||||
"../../tests/fixtures/compression-dictionary/dictionary.raw-deflate"
|
||||
)
|
||||
.as_slice(),
|
||||
DICTIONARY,
|
||||
),
|
||||
(
|
||||
"br",
|
||||
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.br")
|
||||
.as_slice(),
|
||||
DICTIONARY,
|
||||
),
|
||||
(
|
||||
"zstd",
|
||||
include_bytes!("../../tests/fixtures/compression-dictionary/dictionary.zstd")
|
||||
.as_slice(),
|
||||
DICTIONARY,
|
||||
),
|
||||
];
|
||||
let mut responses = vec![reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\", id=\"first\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
chunked,
|
||||
)];
|
||||
responses.extend(
|
||||
coding
|
||||
.iter()
|
||||
.map(|(name, body, _)| reply(&[("Content-Encoding", name)], body, chunked)),
|
||||
);
|
||||
let (url, task) = server(responses).await?;
|
||||
let client =
|
||||
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
assert_eq!(
|
||||
fetch(&client, &url, mode).await?,
|
||||
DICTIONARY,
|
||||
"{mode}/{chunked}"
|
||||
);
|
||||
for (name, _, expected) in coding {
|
||||
assert_eq!(
|
||||
fetch(&client, &url, mode).await?,
|
||||
expected,
|
||||
"{mode}/{chunked}/{name}"
|
||||
);
|
||||
}
|
||||
let requests = task.await??;
|
||||
assert_eq!(header(&requests[0], "available-dictionary"), None);
|
||||
assert!(
|
||||
!header(&requests[0], "accept-encoding")
|
||||
.unwrap()
|
||||
.contains("dcb")
|
||||
);
|
||||
for request in &requests[1..] {
|
||||
assert_eq!(header(request, "available-dictionary"), Some(HASH));
|
||||
assert_eq!(header(request, "dictionary-id"), Some("\"first\""));
|
||||
let encoding = header(request, "accept-encoding").unwrap();
|
||||
assert!(encoding.contains("dcb") && encoding.contains("dcz"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_clear_site_data_observes_cache_and_cookie_directives() -> Result<()>
|
||||
{
|
||||
let registered = reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
);
|
||||
for directive in ["storage", "cache", "cookies", "*"] {
|
||||
let field = format!("\"{directive}\"");
|
||||
let (url, task) = server(vec![
|
||||
registered.clone(),
|
||||
reply(&[("Clear-Site-Data", &field)], b"OK", false),
|
||||
reply(&[], b"OK", false),
|
||||
])
|
||||
.await?;
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
for _ in 0..3 {
|
||||
fetch(&client, &url, "raw").await?;
|
||||
}
|
||||
let requests = task.await??;
|
||||
assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH));
|
||||
assert_eq!(
|
||||
header(&requests[2], "available-dictionary"),
|
||||
if directive == "storage" {
|
||||
Some(HASH)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_bad_hash_and_truncated_streams_do_not_register() -> Result<()> {
|
||||
let registered = reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\", id=\"good\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
);
|
||||
for (coding, original) in [("dcb", DCB), ("dcz", DCZ)] {
|
||||
let mut bad_hash = original.to_vec();
|
||||
bad_hash[12] ^= 1;
|
||||
for body in [bad_hash.as_slice(), &original[..original.len() - 1]] {
|
||||
let invalid = reply(
|
||||
&[
|
||||
("Content-Encoding", coding),
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\", id=\"bad\""),
|
||||
],
|
||||
body,
|
||||
true,
|
||||
);
|
||||
let (url, task) =
|
||||
server(vec![registered.clone(), invalid, reply(&[], b"OK", false)]).await?;
|
||||
let client =
|
||||
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
fetch(&client, &url, "raw").await?;
|
||||
assert!(fetch(&client, &url, "raw").await.is_err(), "{coding}");
|
||||
fetch(&client, &url, "raw").await?;
|
||||
assert_eq!(header(&task.await??[2], "dictionary-id"), Some("\"good\""));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_request_snapshot_survives_later_registration() -> Result<()> {
|
||||
let (url, task) = server(vec![
|
||||
reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
),
|
||||
reply(&[], b"OK", false),
|
||||
reply(&[], b"OK", false),
|
||||
reply(&[], b"OK", false),
|
||||
])
|
||||
.await?;
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
let mut early = Request::get(&url)?;
|
||||
client.prepare_request(&mut early);
|
||||
fetch(&client, &url, "raw").await?;
|
||||
assert!(
|
||||
!client
|
||||
.cache_request_headers(&early)
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("available-dictionary"))
|
||||
);
|
||||
client.fetch_raw(early.clone()).await?;
|
||||
fetch(&client, &url, "raw").await?;
|
||||
client
|
||||
.fetch_raw(early.with_network_partition_key(Some("https://other.test".to_owned())))
|
||||
.await?;
|
||||
let requests = task.await??;
|
||||
assert_eq!(header(&requests[1], "available-dictionary"), None);
|
||||
assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH));
|
||||
assert_eq!(header(&requests[3], "available-dictionary"), None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_cached_body_registration_keeps_original_freshness() -> Result<()> {
|
||||
let (url, task) = server(vec![
|
||||
reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
),
|
||||
reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false),
|
||||
reply(&[], b"OK", false),
|
||||
reply(&[("Clear-Site-Data", "\"cache\"")], b"OK", false),
|
||||
reply(&[], b"OK", false),
|
||||
])
|
||||
.await?;
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
let request = Request::get(&url)?;
|
||||
let cached = client.fetch_raw(request.clone()).await?;
|
||||
fetch(&client, &url, "raw").await?;
|
||||
client.observe_cached_response(&request, &cached, moli_http_cache::unix_now_ms() + 600_000);
|
||||
fetch(&client, &url, "raw").await?;
|
||||
fetch(&client, &url, "raw").await?;
|
||||
client.observe_cached_response(
|
||||
&request,
|
||||
&cached,
|
||||
moli_http_cache::unix_now_ms().saturating_sub(1),
|
||||
);
|
||||
fetch(&client, &url, "raw").await?;
|
||||
let requests = task.await??;
|
||||
assert_eq!(header(&requests[2], "available-dictionary"), Some(HASH));
|
||||
assert_eq!(header(&requests[4], "available-dictionary"), None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_disk_cache_hit_registers_decoded_dictionary() -> Result<()> {
|
||||
let (url, task) = server(vec![
|
||||
reply(
|
||||
&[
|
||||
("Cache-Control", "max-age=600"),
|
||||
("Use-As-Dictionary", "match=\"/*\""),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
),
|
||||
reply(&[], b"probe", false),
|
||||
])
|
||||
.await?;
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let request = Request::get(&url)?;
|
||||
{
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
assert_eq!(
|
||||
client.fetch_raw(request.clone()).await?.body_bytes(),
|
||||
DICTIONARY
|
||||
);
|
||||
}
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
let hit = client.fetch_raw(request).await?;
|
||||
assert_eq!(hit.cache_state, crate::ResponseCacheState::Local);
|
||||
let probe = format!("{url}?probe");
|
||||
client.fetch_raw(Request::get(&probe)?).await?;
|
||||
let requests = task.await??;
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(header(&requests[1], "available-dictionary"), Some(HASH));
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compression_dictionary_registration_requires_readable_and_fresh_responses() -> Result<()> {
|
||||
let cases = [
|
||||
("match=\"/*\"", "max-age=600", "0", true),
|
||||
("match=42", "max-age=600", "0", false),
|
||||
("match=\"/:id(d.*)\"", "max-age=600", "0", false),
|
||||
("match=\"/*\", type=other", "max-age=600", "0", false),
|
||||
(
|
||||
"match=\"/*\", match-dest=\"script\"",
|
||||
"max-age=600",
|
||||
"0",
|
||||
false,
|
||||
),
|
||||
("match=\"/*\"", "max-age=600", "601", false),
|
||||
("match=\"/*\"", "no-store, max-age=600", "0", false),
|
||||
];
|
||||
for (options, cache, age, expected) in cases {
|
||||
let (url, task) = server(vec![
|
||||
reply(
|
||||
&[
|
||||
("Use-As-Dictionary", options),
|
||||
("Cache-Control", cache),
|
||||
("Age", age),
|
||||
],
|
||||
DICTIONARY,
|
||||
false,
|
||||
),
|
||||
reply(&[], b"OK", false),
|
||||
])
|
||||
.await?;
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
fetch(&client, &url, "raw").await?;
|
||||
fetch(&client, &url, "raw").await?;
|
||||
assert_eq!(
|
||||
header(&task.await??[1], "available-dictionary"),
|
||||
expected.then_some(HASH),
|
||||
"{options}/{cache}/{age}"
|
||||
);
|
||||
}
|
||||
for (mode, cors, expected) in [
|
||||
(crate::RequestMode::NoCors, true, false),
|
||||
(crate::RequestMode::Cors, false, false),
|
||||
(crate::RequestMode::Cors, true, true),
|
||||
] {
|
||||
let mut headers = vec![
|
||||
("Use-As-Dictionary", "match=\"/*\""),
|
||||
("Cache-Control", "max-age=600"),
|
||||
];
|
||||
if cors {
|
||||
headers.push(("Access-Control-Allow-Origin", "*"));
|
||||
}
|
||||
let (url, task) = server(vec![
|
||||
reply(&headers, DICTIONARY, false),
|
||||
reply(&[("Access-Control-Allow-Origin", "*")], b"OK", false),
|
||||
])
|
||||
.await?;
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
let mut request = Request::get(&url)?
|
||||
.with_request_origin(moli_url::WebOrigin::from_serialized("https://frame.test"));
|
||||
request.request_mode = mode;
|
||||
request.credentials_mode = crate::RequestCredentialsMode::Omit;
|
||||
let _ = client.fetch_raw(request.clone()).await;
|
||||
request.request_mode = crate::RequestMode::Cors;
|
||||
client.fetch_raw(request).await?;
|
||||
assert_eq!(
|
||||
header(&task.await??[1], "available-dictionary"),
|
||||
expected.then_some(HASH),
|
||||
"{mode:?}/{cors}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -3,6 +3,7 @@ mod cache_request_controls_modes;
|
||||
mod cache_request_directives;
|
||||
mod cache_request_modes;
|
||||
mod cache_vary_encoding;
|
||||
mod compression_dictionary;
|
||||
mod cookie_context;
|
||||
mod cookie_store;
|
||||
mod corp;
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
*.br binary
|
||||
*.dcb binary
|
||||
*.dcz binary
|
||||
*.deflate binary
|
||||
*.gzip binary
|
||||
*.raw-deflate binary
|
||||
*.zstd binary
|
||||
@@ -0,0 +1,5 @@
|
||||
# Compression dictionary fixtures
|
||||
|
||||
The `dictionary.*` and `payload.*` Brotli, gzip, Zstandard, dcb and dcz bytes are copied unchanged from the upstream WPT `fetch/compression-dictionary/resources/register-dictionary.py` and `compressed-data.py`, snapshot `db95fafd1fcef8428805e41eb5705d444e8c67ce`. The two deflate fixtures encode the same dictionary with Python zlib, with and without the zlib wrapper.
|
||||
|
||||
Dictionary bytes are `This is a test dictionary.\n`; payload bytes are `This is compressed test data using a test dictionary`. The dcb/dcz headers and dictionary SHA-256 remain part of each fixture.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
�DCBS–›ֿ^–�נ₪½�k>“�בV�[‘־ƒ‘bBpפ¡˜€b₪L�„�®ֲֺ`"n�ֹ·ֳD�¼אלֱמ43>
|
||||
Binary file not shown.
@@ -13,8 +13,8 @@ use moli_fetch::{
|
||||
BrowserRequestMetadata, RawResponse, Request, RequestHeaders, RequestResourceType, Response,
|
||||
};
|
||||
use moli_http_cache::{
|
||||
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date,
|
||||
response_vary_header_names, unix_now_ms, validation_headers_from_headers,
|
||||
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date, unix_now_ms,
|
||||
validation_headers_from_headers,
|
||||
};
|
||||
use parking_lot::Mutex;
|
||||
use url::Url;
|
||||
@@ -81,7 +81,10 @@ pub(in crate::network) struct RawSubresourceCacheKey {
|
||||
}
|
||||
|
||||
pub(in crate::network) enum RawSubresourceCacheLookup {
|
||||
Local(Box<RawResponse>),
|
||||
Local {
|
||||
response: Box<RawResponse>,
|
||||
expires_at_unix_ms: u64,
|
||||
},
|
||||
Revalidate(Box<RawResponse>),
|
||||
}
|
||||
|
||||
@@ -724,9 +727,13 @@ impl SharedMemoryResourceCache {
|
||||
return Some(RawSubresourceCacheLookup::Revalidate(variant.response));
|
||||
}
|
||||
let mut hit = variants[index].response.clone();
|
||||
let expires_at_unix_ms = variants[index].expires_at_unix_ms;
|
||||
hit.final_url.set_fragment(request.url.fragment());
|
||||
self.insert_raw_subresource_variants(key.clone(), variants);
|
||||
Some(RawSubresourceCacheLookup::Local(hit))
|
||||
Some(RawSubresourceCacheLookup::Local {
|
||||
response: hit,
|
||||
expires_at_unix_ms,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::network) fn remove_raw_subresource(
|
||||
@@ -750,7 +757,9 @@ impl SharedMemoryResourceCache {
|
||||
if response.body_bytes().len() > self.resource_body_bytes_limit {
|
||||
return;
|
||||
}
|
||||
let Some(vary_names) = response_vary_header_names(&response.headers) else {
|
||||
let Some(vary_names) =
|
||||
moli_fetch::cache_vary_header_names(&response.final_url, &response.headers)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let variant = RawSubresourceVariant {
|
||||
@@ -1089,7 +1098,7 @@ pub(in crate::network) fn raw_subresource_memory_cache_expiry(
|
||||
// Match ordinary fields against the same encoded builder as the native
|
||||
// request. Mutable transport-only fields need exact sent-value tracking
|
||||
// before a renderer representation can safely vary on them.
|
||||
if !response_vary_header_names(&response.headers)?
|
||||
if !moli_fetch::cache_vary_header_names(&response.final_url, &response.headers)?
|
||||
.iter()
|
||||
.all(|name| {
|
||||
name.bytes()
|
||||
|
||||
@@ -1019,7 +1019,15 @@ impl ResourceRequestClient {
|
||||
});
|
||||
let mut network_request = request.clone();
|
||||
let cached = match lookup {
|
||||
Some(RawSubresourceCacheLookup::Local(cached)) => {
|
||||
Some(RawSubresourceCacheLookup::Local {
|
||||
response: cached,
|
||||
expires_at_unix_ms,
|
||||
}) => {
|
||||
self.resource_runtime.client().observe_cached_response(
|
||||
&request,
|
||||
&cached,
|
||||
expires_at_unix_ms,
|
||||
);
|
||||
if let Some(sizes) = cached.body_sizes() {
|
||||
cancel_handle.set_response_body_sizes(sizes);
|
||||
}
|
||||
|
||||
@@ -2490,10 +2490,8 @@ fn preload_like_link_readiness_fetch_options(
|
||||
};
|
||||
let credentials_mode = if resource_type == SubresourceResourceType::Dictionary {
|
||||
// Blink sets compression dictionary requests to CORS mode with
|
||||
// credentials omitted and no referrer. Request::new already uses CORS
|
||||
// for subresources; Moli does not yet model per-request referrer
|
||||
// policy for this loader path, so credentials and priority are the
|
||||
// observable pieces preserved here.
|
||||
// credentials omitted and no referrer. Browser metadata and referrer
|
||||
// suppression are applied by connected_link_readiness_request.
|
||||
RequestCredentialsMode::Omit
|
||||
} else if cross_origin.is_some() {
|
||||
link_crossorigin_credentials_mode(cross_origin)
|
||||
@@ -2654,6 +2652,11 @@ fn connected_link_readiness_request(
|
||||
if let Some(resource_type) = options.request_resource_type {
|
||||
request = request.with_resource_type(resource_type);
|
||||
}
|
||||
if options.resource_type == SubresourceResourceType::Dictionary {
|
||||
request = request
|
||||
.with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Dictionary)
|
||||
.without_inferred_referrer();
|
||||
}
|
||||
if let Some(metadata) = options.script_fetch_metadata.as_ref() {
|
||||
request = request.with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Script);
|
||||
request = request.with_script_fetch_metadata(moli_fetch::ScriptFetchRequestMetadata {
|
||||
@@ -5297,6 +5300,30 @@ mod tests {
|
||||
Some(RequestResourceType::Dictionary)
|
||||
);
|
||||
assert_eq!(options.credentials_mode, RequestCredentialsMode::Omit);
|
||||
let document_url = document.final_url().expect("HTML document URL");
|
||||
let url = Url::parse("https://example.test/dict.bin").unwrap();
|
||||
let request = connected_link_readiness_request(
|
||||
document_url,
|
||||
&moli_url::WebOrigin::from_url(document_url),
|
||||
&url,
|
||||
&options,
|
||||
);
|
||||
let headers = moli_fetch::outgoing_request_headers(&Default::default(), &request, None);
|
||||
assert_eq!(
|
||||
request.browser_request_metadata(),
|
||||
Some(moli_fetch::BrowserRequestMetadata::Dictionary)
|
||||
);
|
||||
assert!(headers.iter().any(|(name, value)| name.eq_ignore_ascii_case("sec-fetch-mode") && value == "cors"));
|
||||
assert!(
|
||||
headers.iter().any(
|
||||
|(name, value)| name.eq_ignore_ascii_case("sec-fetch-dest") && value == "empty"
|
||||
)
|
||||
);
|
||||
assert!(
|
||||
!headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("referer"))
|
||||
);
|
||||
assert!(!options.link_preload);
|
||||
assert!(options.script_fetch_metadata.is_none());
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ pub fn is_forbidden_request_header_name(name: &str) -> bool {
|
||||
name.as_str(),
|
||||
"accept-charset"
|
||||
| "accept-encoding"
|
||||
| "available-dictionary"
|
||||
| "dictionary-id"
|
||||
| "access-control-request-headers"
|
||||
| "access-control-request-method"
|
||||
| "connection"
|
||||
|
||||
Reference in New Issue
Block a user