fix(fetch): honor cache-only requests and stale constraints

This commit is contained in:
ldm0
2026-11-21 20:03:34 +08:00
parent 6d2764f076
commit a45f2eca34
17 changed files with 1263 additions and 119 deletions
+15 -14
View File
@@ -6,8 +6,7 @@ mod vary;
use anyhow::{Context, Result, anyhow, bail};
use moli_http_cache::{
HttpCacheEntryMetadata, HttpCacheStats, HttpCacheStore, cached_response_is_stale,
request_header_requires_validation,
HttpCacheEntryMetadata, HttpCacheRequestPolicy, HttpCacheStats, HttpCacheStore,
};
use moli_url_policy::ensure_http_network_transport_url;
use url::Url;
@@ -16,7 +15,7 @@ use std::path::Path;
use crate::{FetchConfig, Request};
use super::outgoing_request_headers_for_url;
use super::outgoing_request_header_bytes_for_url;
pub fn clear_http_cache(config: &FetchConfig) -> Result<()> {
let Some(cache_dir) = config.http_cache_dir() else {
@@ -88,25 +87,27 @@ pub(crate) struct CachedStreamingResponseLookup {
pub(crate) status: u16,
pub(crate) headers: Vec<(String, Vec<u8>)>,
pub(crate) body: std::io::BufReader<std::fs::File>,
pub(crate) expires_at_unix_ms: Option<u64>,
pub(crate) force_validate: bool,
pub(crate) allow_stale: bool,
pub(crate) needs_validation: bool,
pub(crate) cache_state: crate::ResponseCacheState,
}
pub(crate) fn cached_streaming_response_is_stale(record: &CachedStreamingResponseLookup) -> bool {
!record.allow_stale
&& cached_response_is_stale(record.expires_at_unix_ms, record.force_validate)
pub(crate) fn cached_streaming_response_needs_validation(
record: &CachedStreamingResponseLookup,
) -> bool {
record.needs_validation
}
pub(super) fn request_cache_control_requires_validation(
pub(crate) fn request_cache_policy_for_url(
config: &FetchConfig,
request: &Request,
request_url: &Url,
) -> bool {
outgoing_request_headers_for_url(config, request, request_url, None)
.iter()
.any(|(name, value)| request_header_requires_validation(name, value))
) -> HttpCacheRequestPolicy {
HttpCacheRequestPolicy::from_headers(&outgoing_request_header_bytes_for_url(
config,
request,
request_url,
None,
))
}
pub(crate) fn next_redirect_url_from_parts(
+1 -1
View File
@@ -25,7 +25,7 @@ pub(super) fn cache_store_and_key_for_request(
|| !subresource_validation_allows_http_cache(request)
|| cookie_header.is_some()
|| literal_cookie_header_for_request(config, request, request_url).is_some()
|| !request_headers_allow_http_cache(&request.request_headers.to_byte_strings())
|| !request_headers_allow_http_cache(&request.request_headers)
|| !matches!(request_url.scheme(), "http" | "https")
|| !request_url.username().is_empty()
|| request_url.password().is_some()
+17 -30
View File
@@ -1,7 +1,5 @@
use anyhow::Result;
use moli_http_cache::{
HttpCacheBodyWriter, HttpCachedEntryReader, cached_response_is_fresh_immutable,
};
use moli_http_cache::{HttpCacheBodyWriter, HttpCachedEntryReader, unix_now_ms};
use url::Url;
use crate::{FetchConfig, Request};
@@ -10,7 +8,7 @@ use super::{
CachedStreamingResponseLookup,
key::{cache_store_and_key_for_request, normalized_cache_url_string},
metadata::cache_metadata_for_response_parts,
request_cache_control_requires_validation,
request_cache_policy_for_url,
vary::vary_headers_match,
};
@@ -48,15 +46,23 @@ pub(crate) fn load_cached_streaming_response_lookup(
// Normalized cache keys should not leak the first request's fragment into
// later streaming hits.
cached.metadata.final_url = request_url.to_string();
let force_validate = cache_lookup_force_validate(config, request, request_url, &cached);
let needs_validation = !request
.effective_cache_mode()
.allows_stale_http_cache_response()
&& !request_cache_policy_for_url(config, request, request_url).allows_reuse_at(
&cached.metadata.headers,
cached.metadata.stored_at_unix_ms,
cached.metadata.expires_at_unix_ms,
request
.effective_cache_mode()
.requires_http_cache_validation(),
unix_now_ms(),
);
let _ = store.touch_loaded_entry(&key, &cached.metadata);
Ok(Some(cached_streaming_record_from_entry(
key,
cached,
force_validate,
request
.effective_cache_mode()
.allows_stale_http_cache_response(),
needs_validation,
)))
}
@@ -156,8 +162,7 @@ pub(crate) fn finish_streaming_cached_response(
fn cached_streaming_record_from_entry(
key: String,
entry: HttpCachedEntryReader,
force_validate: bool,
allow_stale: bool,
needs_validation: bool,
) -> CachedStreamingResponseLookup {
let metadata = entry.metadata.clone();
CachedStreamingResponseLookup {
@@ -167,9 +172,7 @@ fn cached_streaming_record_from_entry(
status: entry.metadata.status,
headers: entry.metadata.headers,
body: entry.body,
expires_at_unix_ms: entry.metadata.expires_at_unix_ms,
force_validate,
allow_stale,
needs_validation,
cache_state: crate::ResponseCacheState::Local,
}
}
@@ -183,19 +186,3 @@ fn entry_reader_body_exceeds_response_limit(
.zip(cached.body_len().ok())
.is_some_and(|(limit, body_len)| body_len > limit as u64)
}
fn cache_lookup_force_validate(
config: &FetchConfig,
request: &Request,
request_url: &Url,
cached: &HttpCachedEntryReader,
) -> bool {
request
.effective_cache_mode()
.requires_http_cache_validation()
|| (request_cache_control_requires_validation(config, request, request_url)
&& !cached_response_is_fresh_immutable(
&cached.metadata.headers,
cached.metadata.expires_at_unix_ms,
))
}
+1 -1
View File
@@ -26,6 +26,7 @@ pub(crate) fn merge_cached_not_modified_streaming_response_lookup(
let headers = merge_not_modified_headers(&cached.headers, not_modified_headers);
cached.headers = headers.clone();
cached.cache_state = crate::ResponseCacheState::Validated;
cached.needs_validation = false;
cached.metadata.headers = headers.clone();
let final_url = Url::parse(&cached.final_url).with_context(|| {
@@ -52,7 +53,6 @@ pub(crate) fn merge_cached_not_modified_streaming_response_lookup(
cached.metadata.last_used_at_unix_ms = metadata.last_used_at_unix_ms;
cached.metadata.expires_at_unix_ms = metadata.expires_at_unix_ms;
cached.metadata.vary_headers = metadata.vary_headers;
cached.expires_at_unix_ms = cached.metadata.expires_at_unix_ms;
if let Some((store, key)) =
cache_store_and_key_for_request(config, request, request_url, cookie_header)
&& key == cached.key
+12 -15
View File
@@ -1,9 +1,6 @@
use std::str;
use moli_http_cache::{
HttpCacheVaryHeader, request_cache_control_forbids_storage, request_header_requires_validation,
response_vary_header_names,
};
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader, response_vary_header_names};
use url::Url;
use crate::{FetchConfig, Request};
@@ -64,17 +61,17 @@ pub(super) fn vary_headers_match(
})
}
pub(super) fn request_headers_allow_http_cache(headers: &[(String, String)]) -> bool {
headers.iter().enumerate().all(|(index, (name, value))| {
// Revalidation controls select the existing conditional-cache path;
// they must not turn a fresh entry into an unconditional cache miss.
// Other unsupported controls keep the conservative bypass policy.
(supported_cache_request_header(name) || request_header_requires_validation(name, value))
&& !(name.eq_ignore_ascii_case("cache-control")
&& request_cache_control_forbids_storage(value))
&& !headers[..index]
.iter()
.any(|(previous, _)| previous.eq_ignore_ascii_case(name))
pub(super) fn request_headers_allow_http_cache(headers: &crate::RequestHeaders) -> bool {
if HttpCacheRequestPolicy::from_headers(headers).no_store {
return false;
}
headers.iter().enumerate().all(|(index, (name, _))| {
name.eq_ignore_ascii_case("cache-control")
|| name.eq_ignore_ascii_case("pragma")
|| (supported_cache_request_header(name)
&& !headers[..index]
.iter()
.any(|(previous, _)| previous.eq_ignore_ascii_case(name)))
})
}
+5 -4
View File
@@ -22,10 +22,11 @@ use url::{Host, Url};
pub(crate) use self::cache::{
CachedStreamingResponseLookup, cached_streaming_response_body_exceeds_response_limit,
cached_streaming_response_is_stale, create_streaming_cache_body_writer_for_response_parts,
finish_streaming_cached_response, load_cached_streaming_response_lookup,
merge_cached_not_modified_streaming_response_lookup, next_followed_redirect_url_from_parts,
next_redirect_url_from_parts, remove_cached_response, response_headers_forbid_cache_storage,
cached_streaming_response_needs_validation,
create_streaming_cache_body_writer_for_response_parts, finish_streaming_cached_response,
load_cached_streaming_response_lookup, merge_cached_not_modified_streaming_response_lookup,
next_followed_redirect_url_from_parts, next_redirect_url_from_parts, remove_cached_response,
request_cache_policy_for_url, response_headers_forbid_cache_storage,
validation_headers_for_cached_streaming_response_lookup,
};
pub use self::cache::{
+99 -10
View File
@@ -42,12 +42,12 @@ use crate::{
CachedStreamingResponseLookup, RawStreamingResponseCollector, RequestHttpVersion,
RequestTransferMetrics, ResponseCollector, StreamingCachePlan, StreamingHtmlResponseStart,
StreamingResponseCollector, cached_streaming_response_body_exceeds_response_limit,
cached_streaming_response_is_stale, configure_easy, cookie_access_report_for_request,
cookie_header_from_report, finish_streaming_cached_response,
load_cached_streaming_response_lookup, log_request_completion,
merge_cached_not_modified_streaming_response_lookup,
cached_streaming_response_needs_validation, configure_easy,
cookie_access_report_for_request, cookie_header_from_report,
finish_streaming_cached_response, load_cached_streaming_response_lookup,
log_request_completion, merge_cached_not_modified_streaming_response_lookup,
network_request_extra_info_from_headers, next_followed_redirect_url_from_parts,
outgoing_request_headers_for_url, remove_cached_response,
outgoing_request_headers_for_url, remove_cached_response, request_cache_policy_for_url,
response_headers_forbid_cache_storage, store_response_cookies, transfer_metrics_from_easy,
validation_headers_for_cached_streaming_response_lookup,
},
@@ -886,6 +886,7 @@ impl RuntimeOwner {
) {
return Err((job.response_tx, error));
}
let request_cookie_report = if job.request.allows_credentials_for_url(&job.current_url) {
match cookie_access_report_for_request(
&self.cookie_store,
@@ -912,7 +913,7 @@ impl RuntimeOwner {
&job.current_url,
cookie_header.as_deref(),
) {
Ok(Some(cached)) if !cached_streaming_response_is_stale(&cached) => {
Ok(Some(cached)) if !cached_streaming_response_needs_validation(&cached) => {
if prepared_request
.response_policy
.observe_response(&job.current_url, &cached.headers)
@@ -940,6 +941,17 @@ impl RuntimeOwner {
if job.request.effective_cache_mode() == crate::RequestCacheMode::OnlyIfCached {
return Err((job.response_tx, anyhow::Error::new(crate::FetchCacheMiss)));
}
if request_cache_policy_for_url(&self.config, &job.request, &job.current_url).only_if_cached
{
let head = cache_only_miss_start(&job.request, &job.current_url).into_head();
job.response_progress.commit();
return Ok(JobOutcome::Complete(
job.response_tx,
Box::new(CompletedBufferedResponse::Raw(
RawResponse::from_head_and_body(head, Vec::new()),
)),
));
}
let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new(
Some(job.cancel_handle.clone()),
)));
@@ -1065,9 +1077,10 @@ impl RuntimeOwner {
job.request.prepare_accept_header(&self.config);
job.response_progress = job.cancel_handle.begin_response_progress();
if job.cancel_handle.is_cancelled() {
let easy = job.easy.take();
return Err((
Box::new(job),
None,
easy,
anyhow::Error::new(crate::FetchCancelled)
.context("fetch runtime request cancelled"),
));
@@ -1105,7 +1118,9 @@ impl RuntimeOwner {
&job.current_url,
cookie_header.as_deref(),
) {
Ok(Some(cached_lookup)) if !cached_streaming_response_is_stale(&cached_lookup) => {
Ok(Some(cached_lookup))
if !cached_streaming_response_needs_validation(&cached_lookup) =>
{
if prepared_request
.response_policy
.observe_response(&job.current_url, &cached_lookup.headers)
@@ -1141,6 +1156,12 @@ impl RuntimeOwner {
anyhow::Error::new(crate::FetchCacheMiss),
));
}
if request_cache_policy_for_url(&self.config, &prepared_request.request, &job.current_url)
.only_if_cached
{
complete_cache_only_html_miss(job);
return Ok(StreamingJobOutcome::Complete);
}
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_streaming(
@@ -1300,9 +1321,10 @@ impl RuntimeOwner {
job.request.prepare_accept_header(&self.config);
job.response_progress = job.cancel_handle.begin_response_progress();
if job.cancel_handle.is_cancelled() {
let easy = job.easy.take();
return Err((
Box::new(job),
None,
easy,
anyhow::Error::new(crate::FetchCancelled)
.context("fetch runtime request cancelled"),
));
@@ -1341,7 +1363,9 @@ impl RuntimeOwner {
&job.current_url,
cookie_header.as_deref(),
) {
Ok(Some(cached_lookup)) if !cached_streaming_response_is_stale(&cached_lookup) => {
Ok(Some(cached_lookup))
if !cached_streaming_response_needs_validation(&cached_lookup) =>
{
if prepared_request
.response_policy
.observe_response(&job.current_url, &cached_lookup.headers)
@@ -1380,6 +1404,12 @@ impl RuntimeOwner {
anyhow::Error::new(crate::FetchCacheMiss),
));
}
if request_cache_policy_for_url(&self.config, &prepared_request.request, &job.current_url)
.only_if_cached
{
complete_cache_only_raw_miss(job);
return Ok(StreamingJobOutcome::Complete);
}
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_raw_streaming(
@@ -3819,6 +3849,65 @@ fn take_failed_proxy_connect_response(
.take_failed_proxy_connect_response(connect_status)
}
fn cache_only_miss_start(request: &Request, current_url: &Url) -> StreamingHtmlResponseStart {
StreamingHtmlResponseStart {
final_url: current_url.clone(),
status: 504,
status_text: Some("Gateway Timeout".to_owned()),
headers: vec![
("content-length".to_owned(), b"0".to_vec()),
("cache-control".to_owned(), b"no-store".to_vec()),
],
request_cookie_report: None,
cookie_set_reports: Vec::new(),
redirected: !request.redirect_chain.is_empty(),
redirect_chain: request.redirect_chain.clone(),
from_cache: false,
// This is a local result, not a stored representation or a wire
// response. Consumers must not invent transfer bytes or protocol data.
cache_state: crate::ResponseCacheState::Local,
preload_state: Default::default(),
negotiated_http_version: None,
network_request_extra_info: None,
}
}
fn complete_cache_only_html_miss(mut job: StreamingRuntimeJob) {
let start = cache_only_miss_start(&job.request, &job.current_url);
let (started_tx, body_tx) = if let Some(easy) = job.easy.as_mut() {
easy.get_mut()
.streaming_mut()
.expect("HTML job requires an HTML collector")
.take_response_channels()
} else {
(job.started_tx.take(), job.body_tx.take())
};
drop(body_tx);
job.response_progress.commit();
if let Some(started_tx) = started_tx {
let _ = started_tx.send(Ok(start));
}
let _ = job.completion_tx.send(Ok(()));
}
fn complete_cache_only_raw_miss(mut job: StreamingRawRuntimeJob) {
let start = cache_only_miss_start(&job.request, &job.current_url);
let (started_tx, body_tx) = if let Some(easy) = job.easy.as_mut() {
easy.get_mut()
.raw_streaming_mut()
.expect("raw job requires a raw collector")
.take_response_channels()
} else {
(job.started_tx.take(), job.body_tx.take())
};
drop(body_tx);
job.response_progress.commit();
if let Some(started_tx) = started_tx {
let _ = started_tx.send(Ok(start));
}
let _ = job.completion_tx.send(Ok(()));
}
fn proxy_connect_response_start(
current_url: &Url,
redirect_chain: &[RedirectInfo],
@@ -0,0 +1,374 @@
use super::*;
use crate::ResponseCacheState;
fn controlled(url: &str, value: &str) -> Result<Request> {
let mut request = Request::get(url)?;
request.request_headers = vec![("Cache-Control".to_owned(), value.to_owned())].into();
Ok(request)
}
#[tokio::test]
async fn cache_only_requests_preserve_cancellation_before_local_completion() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("must not be requested")]);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for follow_redirects in [true, false] {
let cancel = FetchCancelHandle::new();
cancel.cancel();
let request = controlled(&server.url(), "only-if-cached")?.with_redirect_handling(
if follow_redirects {
crate::RequestRedirectHandling::Follow
} else {
crate::RequestRedirectHandling::FollowByCaller
},
);
let error = client
.fetch_with_cancel(request, cancel)
.await
.expect_err("an already cancelled cache-only request must not complete with 504");
assert!(error.is::<crate::FetchCancelled>(), "{error:#}");
}
client.shutdown();
assert_eq!(server.hits(), 0);
server.shutdown();
Ok(())
}
#[tokio::test]
async fn cache_only_misses_never_contact_origin_on_raw_text_html_or_buffered_paths() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("must not be requested")]);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for path in 0..4 {
let cancel = FetchCancelHandle::new();
let request = controlled(&server.url(), "only-if-cached")?;
let head = match path {
0 => {
let observed = client
.fetch_raw_stream_with_cancel_and_network_metadata(request, cancel.clone())
.await?;
let (stream, journal) = observed.into_parts_with_observation_journal();
assert!(journal.is_empty());
assert!(stream.network_request_extra_info().is_none());
let response = stream.into_materialized_raw_response().await?;
assert!(response.body_bytes().is_empty());
assert!(cancel.response_timing().request_started_at().is_none());
assert!(cancel.response_timing().response_started_at().is_none());
assert!(cancel.response_timing().response_end_at().is_some());
response.head()
}
1 | 2 => {
let request = request.with_redirect_handling(if path == 1 {
crate::RequestRedirectHandling::Follow
} else {
crate::RequestRedirectHandling::FollowByCaller
});
let observed = client
.fetch_with_cancel_and_network_metadata(request, cancel.clone())
.await?;
let (response, journal) = observed.into_parts_with_observation_journal();
assert!(journal.is_empty());
assert!(response.body_bytes().is_empty());
assert!(cancel.response_timing().request_started_at().is_none());
response.head()
}
_ => {
let response = client
.fetch_html_stream(request)
.await?
.into_materialized_text_response()
.await?;
assert!(response.body_bytes().is_empty());
response.head()
}
};
assert_eq!(head.status, 504, "path {path}");
assert_eq!(head.status_text.as_deref(), Some("Gateway Timeout"));
assert!(!head.from_cache);
assert_eq!(head.cache_state, ResponseCacheState::Local);
assert!(head.negotiated_http_version.is_none());
}
client.shutdown();
assert_eq!(server.hits(), 0);
assert!(server.requests().is_empty());
server.shutdown();
Ok(())
}
#[tokio::test]
async fn disk_cache_staleness_bounds_and_cache_only_misses_preserve_stored_entries() -> Result<()> {
let cache_dir = unique_test_cache_dir();
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::ok("stale")
.with_header("Cache-Control", "max-age=1500")
.with_header("Age", "2000"),
ScriptedResponse::ok("fresh").with_header("Cache-Control", "max-age=600"),
]);
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
assert_eq!(
client
.fetch_raw(Request::get(&server.url())?)
.await?
.body_bytes(),
b"stale"
);
for control in [
"only-if-cached",
"only-if-cached, max-stale=499",
"only-if-cached, max-stale=1000, no-store",
] {
let response = client
.fetch_raw(controlled(&server.url(), control)?)
.await?;
assert_eq!(response.status, 504, "{control}");
assert!(response.body_bytes().is_empty());
}
for control in ["max-stale=1000, only-if-cached", "max-stale"] {
let response = client
.fetch_raw(controlled(&server.url(), control)?)
.await?;
assert_eq!(response.status, 200);
assert_eq!(response.body_bytes(), b"stale");
assert_eq!(response.cache_state, ResponseCacheState::Local);
}
assert_eq!(server.hits(), 1);
assert_eq!(
client
.fetch_raw(Request::get(&server.url())?)
.await?
.body_bytes(),
b"fresh"
);
let hit = client
.fetch_raw(controlled(&server.url(), "only-if-cached")?)
.await?;
assert_eq!(hit.body_bytes(), b"fresh");
assert_eq!(hit.cache_state, ResponseCacheState::Local);
client.shutdown();
assert_eq!(server.hits(), 2);
server.shutdown();
fs::remove_dir_all(cache_dir)?;
Ok(())
}
#[tokio::test]
async fn disk_cache_request_age_and_freshness_constraints_validate_with_original_body() -> Result<()>
{
let cache_dir = unique_test_cache_dir();
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::ok("original")
.with_header("Cache-Control", "max-age=600")
.with_header("Age", "200")
.with_header("ETag", "\"original\""),
ScriptedResponse::status(304, "Not Modified")
.with_header("Cache-Control", "max-age=600")
.with_header("Age", "0"),
ScriptedResponse::status(304, "Not Modified")
.with_header("Cache-Control", "max-age=600")
.with_header("Age", "0"),
]);
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
client.fetch_raw(Request::get(&server.url())?).await?;
for control in ["max-age=100", "min-fresh=1000"] {
let response = client
.fetch_raw(controlled(&server.url(), control)?)
.await?;
assert_eq!(response.body_bytes(), b"original");
assert_eq!(response.cache_state, ResponseCacheState::Validated);
}
let hit = client
.fetch_raw(controlled(
&server.url(),
"only-if-cached, max-age=600, min-fresh=500",
)?)
.await?;
assert_eq!(hit.body_bytes(), b"original");
assert_eq!(hit.cache_state, ResponseCacheState::Local);
client.shutdown();
assert_eq!(server.hits(), 3);
assert!(server.requests()[1..].iter().all(|request| {
request
.to_ascii_lowercase()
.contains("if-none-match: \"original\"")
}));
server.shutdown();
fs::remove_dir_all(cache_dir)?;
Ok(())
}
#[tokio::test]
async fn disk_cache_max_stale_cannot_override_response_validation_requirements() -> Result<()> {
for control in ["no-cache, max-age=600", "must-revalidate, max-age=0"] {
let cache_dir = unique_test_cache_dir();
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::ok("original")
.with_header("Cache-Control", control)
.with_header("ETag", "\"original\""),
ScriptedResponse::status(304, "Not Modified")
.with_header("Cache-Control", "max-age=600"),
]);
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
client.fetch_raw(Request::get(&server.url())?).await?;
let miss = client
.fetch_raw(controlled(&server.url(), "only-if-cached, max-stale")?)
.await?;
assert_eq!(miss.status, 504, "{control}");
assert_eq!(server.hits(), 1);
let validated = client
.fetch_raw(controlled(&server.url(), "max-stale")?)
.await?;
assert_eq!(validated.body_bytes(), b"original");
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
client.shutdown();
assert_eq!(server.hits(), 2);
assert!(
server.requests()[1]
.to_ascii_lowercase()
.contains("if-none-match: \"original\"")
);
server.shutdown();
fs::remove_dir_all(cache_dir)?;
}
Ok(())
}
#[tokio::test]
async fn disk_cache_only_redirects_stop_at_uncached_hops_without_network() -> Result<()> {
for html in [false, true] {
let cache_dir = unique_test_cache_dir();
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::status(302, "Found")
.with_header("Location", "/missing")
.with_header("Cache-Control", "max-age=600"),
ScriptedResponse::ok("must not be requested"),
]);
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
let prime = Request::get(&server.url())?.with_redirect_handling(if false {
crate::RequestRedirectHandling::Follow
} else {
crate::RequestRedirectHandling::FollowByCaller
});
assert_eq!(client.fetch_raw(prime).await?.status, 302);
let request = controlled(&server.url(), "only-if-cached")?;
let head = if html {
client
.fetch_html_stream(request)
.await?
.into_materialized_text_response()
.await?
.head()
} else {
let observed = client.fetch_raw_with_network_metadata(request).await?;
let (response, journal) = observed.into_parts_with_observation_journal();
assert!(journal.is_empty());
response.head()
};
assert_eq!(head.status, 504);
assert_eq!(head.final_url.path(), "/missing");
assert!(head.redirected);
assert_eq!(head.redirect_chain.len(), 1);
assert!(head.redirect_chain[0].from_cache);
client.shutdown();
assert_eq!(server.hits(), 1);
server.shutdown();
fs::remove_dir_all(cache_dir)?;
}
Ok(())
}
#[tokio::test]
async fn cache_only_hits_are_looked_up_before_misses_on_every_transport() -> Result<()> {
for (age, directive, expected_status) in [
(None, "only-if-cached", 200),
(Some("60"), "only-if-cached, max-stale=120", 200),
(Some("60"), "only-if-cached", 504),
] {
let cache_dir = unique_test_cache_dir();
let mut response = ScriptedResponse::ok("cached").with_header(
"Cache-Control",
if age.is_some() {
"max-age=0"
} else {
"max-age=600"
},
);
if let Some(age) = age {
response = response.with_header("Age", age);
}
let server = ScriptedHttpServer::spawn(vec![
response,
ScriptedResponse::ok("must not be requested"),
]);
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
assert_eq!(
client
.fetch_raw(Request::get(&server.url())?)
.await?
.body_bytes(),
b"cached",
);
for path in 0..4 {
let cancel = FetchCancelHandle::new();
let request = controlled(&server.url(), directive)?;
let response = match path {
0 => {
let observed = client
.fetch_raw_stream_with_cancel_and_network_metadata(request, cancel.clone())
.await?;
let (response, journal) = observed.into_parts_with_observation_journal();
assert!(journal.is_empty());
response.into_materialized_raw_response().await?
}
1 | 2 => {
let request = if path == 2 {
request.with_single_hop()
} else {
request
};
let observed = client
.fetch_with_cancel_and_network_metadata(request, cancel.clone())
.await?;
let (response, journal) = observed.into_parts_with_observation_journal();
assert!(journal.is_empty());
RawResponse::from_head_and_body(response.head(), response.body_bytes().to_vec())
}
_ => {
let response = client
.fetch_html_stream(request)
.await?
.into_materialized_text_response()
.await?;
RawResponse::from_head_and_body(response.head(), response.body_bytes().to_vec())
}
};
assert_eq!(
response.status, expected_status,
"age={age:?}, {directive}, path={path}"
);
assert_eq!(response.cache_state, ResponseCacheState::Local);
assert_eq!(response.from_cache, expected_status == 200);
assert_eq!(
response.body_bytes(),
if expected_status == 200 {
b"cached".as_slice()
} else {
b"".as_slice()
}
);
assert!(cancel.response_timing().request_started_at().is_none());
assert_eq!(server.hits(), 1, "cache-only must not contact the origin");
}
client.shutdown();
server.shutdown();
fs::remove_dir_all(cache_dir)?;
}
Ok(())
}
+1
View File
@@ -1,5 +1,6 @@
mod cache_request_controls;
mod cache_request_controls_modes;
mod cache_request_directives;
mod cache_request_modes;
mod cache_vary_encoding;
mod cookie_context;
+2
View File
@@ -15,6 +15,7 @@ mod eviction;
mod metadata;
mod path_safety;
mod policy;
mod request_policy;
mod revalidation;
mod store;
mod time;
@@ -30,6 +31,7 @@ pub use self::{
request_header_requires_validation, request_pragma_requires_validation,
response_cache_policy, response_date, unix_now_ms,
},
request_policy::HttpCacheRequestPolicy,
revalidation::{merge_not_modified_headers, validation_headers_from_headers},
store::{HttpCacheEntryInfo, HttpCacheStats, HttpCacheStore, HttpCachedEntryReader},
vary::response_vary_header_names,
+31 -13
View File
@@ -18,13 +18,23 @@ pub fn response_date(headers: &[(String, Vec<u8>)]) -> Option<std::time::SystemT
pub struct HttpCacheResponsePolicy {
pub store: bool,
pub expires_at_unix_ms: Option<u64>,
pub requires_validation: bool,
pub must_revalidate: bool,
}
/// Parses the subset of HTTP response cache policy currently enforced by
/// Moli's conservative disk cache.
pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponsePolicy {
response_cache_policy_at(headers, unix_now_ms())
}
pub(crate) fn response_cache_policy_at(
headers: &[(String, Vec<u8>)],
now_ms: u64,
) -> HttpCacheResponsePolicy {
let mut store = true;
let mut requires_validation = false;
let mut must_revalidate = false;
let mut max_age_seconds = None;
let mut expires_header_unix_ms = None;
let mut date_unix_ms = None;
@@ -51,6 +61,8 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
store = false;
} else if directive_name.eq_ignore_ascii_case("no-cache") {
requires_validation = true;
} else if directive_name.eq_ignore_ascii_case("must-revalidate") {
must_revalidate = true;
} else if directive_name.eq_ignore_ascii_case("max-age")
&& let Some(raw) = directive_value
&& let Ok(seconds) = raw.parse::<u64>()
@@ -89,7 +101,7 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
let expires_at_unix_ms = (!requires_validation)
.then(|| {
cache_expires_at_unix_ms(
unix_now_ms(),
now_ms,
max_age_seconds,
expires_header_unix_ms,
date_unix_ms,
@@ -101,6 +113,8 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
HttpCacheResponsePolicy {
store,
expires_at_unix_ms,
requires_validation,
must_revalidate,
}
}
@@ -150,7 +164,7 @@ pub fn cached_response_is_fresh_immutable(
cached_response_is_fresh_immutable_at(unix_now_ms(), headers, expires_at_unix_ms)
}
fn cached_response_is_fresh_immutable_at(
pub(crate) fn cached_response_is_fresh_immutable_at(
now_ms: u64,
headers: &[(String, Vec<u8>)],
expires_at_unix_ms: Option<u64>,
@@ -224,17 +238,21 @@ pub fn cache_expires_at_unix_ms(
date_unix_ms: Option<u64>,
age_seconds: u64,
) -> Option<u64> {
if let Some(max_age_seconds) = max_age_seconds {
let apparent_age_ms = date_unix_ms
.map(|date_ms| now_ms.saturating_sub(date_ms))
.unwrap_or_default();
let response_age_ms = apparent_age_ms.max(age_seconds.saturating_mul(1000));
let freshness_ms = max_age_seconds
.saturating_mul(1000)
.saturating_sub(response_age_ms);
return Some(now_ms.saturating_add(freshness_ms));
}
expires_header_unix_ms
let apparent_age_ms = date_unix_ms
.map(|date| now_ms.saturating_sub(date))
.unwrap_or_default();
let response_age_ms = apparent_age_ms.max(age_seconds.saturating_mul(1000));
let lifetime_ms = if let Some(max_age_seconds) = max_age_seconds {
i128::from(max_age_seconds.saturating_mul(1000))
} else {
i128::from(expires_header_unix_ms?) - i128::from(date_unix_ms.unwrap_or(now_ms))
};
// Keep time already spent stale. Clamping the remaining lifetime to zero
// would make every old response appear to have only just expired.
Some(
(i128::from(now_ms) + lifetime_ms - i128::from(response_age_ms))
.clamp(0, i128::from(u64::MAX)) as u64,
)
}
pub fn unix_now_ms() -> u64 {
+355
View File
@@ -0,0 +1,355 @@
use moli_header_field::{split_outside_quoted_strings, unquote_parameter_value};
use crate::policy::{cached_response_is_fresh_immutable_at, response_cache_policy_at};
/// Constraints on reuse of an HTTP representation. `Some(u64::MAX)` permits
/// unbounded staleness; `None` never permits stale reuse.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct HttpCacheRequestPolicy {
pub no_store: bool,
pub no_cache: bool,
pub only_if_cached: bool,
pub max_age_seconds: Option<u64>,
pub max_stale_seconds: Option<u64>,
pub min_fresh_seconds: Option<u64>,
}
impl HttpCacheRequestPolicy {
pub fn from_headers(headers: &[(String, Vec<u8>)]) -> Self {
let mut policy = Self::default();
let mut has_cache_control = false;
let mut pragma_no_cache = false;
for (name, bytes) in headers {
let value = moli_header_field::decode_header_value(bytes);
if name.eq_ignore_ascii_case("pragma") {
pragma_no_cache |= crate::request_pragma_requires_validation(&value);
}
if !name.eq_ignore_ascii_case("cache-control") {
continue;
}
has_cache_control = true;
for directive in split_outside_quoted_strings(&value, ',') {
let (name, value) = directive
.trim()
.split_once('=')
.map(|(name, value)| (name.trim(), Some(unquote_parameter_value(value.trim()))))
.unwrap_or((directive.trim(), None));
match name.to_ascii_lowercase().as_str() {
"no-store" => policy.no_store = true,
"no-cache" => policy.no_cache = true,
"only-if-cached" => policy.only_if_cached = true,
"max-age" | "min-fresh" | "max-stale" => {
let seconds = match value.as_deref() {
None if name.eq_ignore_ascii_case("max-stale") => Some(u64::MAX),
Some(value) => delta_seconds(value),
None => None,
};
let Some(seconds) = seconds else {
// A malformed known constraint must not widen reuse,
// even if a later field supplies a valid duplicate.
policy.no_cache = true;
continue;
};
let slot = if name.eq_ignore_ascii_case("max-age") {
&mut policy.max_age_seconds
} else if name.eq_ignore_ascii_case("max-stale") {
&mut policy.max_stale_seconds
} else {
&mut policy.min_fresh_seconds
};
*slot = Some(slot.map_or(seconds, |previous| {
if name.eq_ignore_ascii_case("min-fresh") {
previous.max(seconds)
} else {
previous.min(seconds)
}
}));
}
_ => {}
}
}
}
if !has_cache_control && pragma_no_cache {
policy.no_cache = true;
}
policy
}
pub fn allows_reuse_at(
&self,
response_headers: &[(String, Vec<u8>)],
stored_at_unix_ms: u64,
expires_at_unix_ms: Option<u64>,
force_validate: bool,
now_unix_ms: u64,
) -> bool {
if self.no_store || self.no_cache || force_validate {
return false;
}
let response = response_cache_policy_at(response_headers, stored_at_unix_ms);
if !response.store || response.requires_validation {
return false;
}
// Recalculate explicit freshness from the stored header facts. This
// also handles persisted entries written before stale age was retained.
let expiry = response
.expires_at_unix_ms
.or(expires_at_unix_ms.filter(|value| *value != 0));
let Some(expiry) = expiry else {
return false;
};
let fresh = now_unix_ms < expiry;
if let Some(max_age) = self.max_age_seconds {
let immutable_reload = max_age == 0
&& fresh
&& cached_response_is_fresh_immutable_at(
now_unix_ms,
response_headers,
Some(expiry),
);
if !immutable_reload
&& (max_age == 0
|| current_age_ms(response_headers, stored_at_unix_ms, now_unix_ms)
> max_age.saturating_mul(1000))
{
return false;
}
}
if self.min_fresh_seconds.is_some_and(|seconds| {
now_unix_ms > expiry
|| expiry.saturating_sub(now_unix_ms) < seconds.saturating_mul(1000)
}) {
return false;
}
fresh
|| (!response.must_revalidate
&& self.max_stale_seconds.is_some_and(|seconds| {
now_unix_ms.saturating_sub(expiry) <= seconds.saturating_mul(1000)
}))
}
}
fn delta_seconds(value: &str) -> Option<u64> {
(!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()))
.then(|| value.parse().unwrap_or(u64::MAX))
}
fn current_age_ms(headers: &[(String, Vec<u8>)], stored_at: u64, now: u64) -> u64 {
let apparent_age = crate::response_date(headers)
.and_then(|date| date.duration_since(std::time::UNIX_EPOCH).ok())
.map(|duration| {
stored_at.saturating_sub(duration.as_millis().min(u128::from(u64::MAX)) as u64)
})
.unwrap_or_default();
let age = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("age"))
.filter_map(|(_, value)| {
delta_seconds(moli_header_field::decode_header_value(value).trim())
})
.max()
.unwrap_or_default()
.saturating_mul(1000);
apparent_age
.max(age)
.saturating_add(now.saturating_sub(stored_at))
}
#[cfg(test)]
mod tests {
use super::*;
fn headers(fields: &[(&str, &str)]) -> Vec<(String, Vec<u8>)> {
fields
.iter()
.map(|(name, value)| ((*name).into(), value.as_bytes().to_vec()))
.collect()
}
fn request(value: &str) -> HttpCacheRequestPolicy {
HttpCacheRequestPolicy::from_headers(&headers(&[("Cache-Control", value)]))
}
#[test]
fn request_directives_merge_fields_without_reading_quoted_extensions() {
let policy = HttpCacheRequestPolicy::from_headers(&headers(&[
(
"Cache-Control",
"extension=\"only-if-cached, no-store\", MAX-AGE=90, max-stale",
),
(
"cache-control",
"max-age=30, max-stale=80, min-fresh=10, ONLY-IF-CACHED",
),
("Cache-Control", "max-age=60, max-stale=90, min-fresh=20"),
]));
assert!(!policy.no_store && !policy.no_cache);
assert!(policy.only_if_cached);
assert_eq!(policy.max_age_seconds, Some(30));
assert_eq!(policy.max_stale_seconds, Some(80));
assert_eq!(policy.min_fresh_seconds, Some(20));
assert!(!request("extension=\"only-if-cached, no-store\"").only_if_cached);
}
#[test]
fn malformed_request_bounds_cannot_be_overridden_by_valid_duplicates() {
for name in ["max-age", "min-fresh", "max-stale"] {
for invalid in ["", "+1", "-1", "1.5", "one"] {
let value = format!("{name}={invalid}, {name}=999, max-stale");
assert!(request(&value).no_cache, "{value}");
}
}
assert!(request("max-age").no_cache);
assert!(request("min-fresh").no_cache);
assert_eq!(request("max-stale").max_stale_seconds, Some(u64::MAX));
assert_eq!(
request("max-age=99999999999999999999999").max_age_seconds,
Some(u64::MAX)
);
}
#[test]
fn pragma_is_a_fallback_only_when_cache_control_is_absent() {
assert!(HttpCacheRequestPolicy::from_headers(&headers(&[("Pragma", "no-cache")])).no_cache);
assert!(
!HttpCacheRequestPolicy::from_headers(&headers(&[
("Pragma", "no-cache"),
("Cache-Control", "max-age=60")
]))
.no_cache
);
}
#[test]
fn max_stale_counts_age_before_storage_and_resident_time() {
let response = headers(&[("Cache-Control", "max-age=1500"), ("Age", "2000")]);
let stored = 3_000_000;
// An older persisted expiry clamped to the storage time must not
// hide the response's existing five hundred seconds of staleness.
assert!(!request("max-stale=499").allows_reuse_at(
&response,
stored,
Some(stored),
false,
stored
));
assert!(request("max-stale=500").allows_reuse_at(
&response,
stored,
Some(stored),
false,
stored
));
assert!(!request("max-stale=500").allows_reuse_at(
&response,
stored,
Some(stored),
false,
stored + 1
));
assert!(request("max-stale").allows_reuse_at(
&response,
stored,
Some(stored),
false,
stored + 999_000
));
assert!(!request("").allows_reuse_at(&response, stored, Some(stored), false, stored));
}
#[test]
fn stale_reuse_never_overrides_response_or_caller_validation_requirements() {
for control in [
"max-age=0, no-cache",
"max-age=0, must-revalidate",
"max-age=0, no-store",
"max-age=0, private",
] {
let response = headers(&[("Cache-Control", control)]);
assert!(
!request("max-stale").allows_reuse_at(
&response,
100_000,
Some(100_000),
false,
101_000
),
"{control}"
);
}
let response = headers(&[("Cache-Control", "max-age=0")]);
for control in [
"no-cache, max-stale",
"no-store, max-stale",
"min-fresh=0, max-stale",
] {
assert!(
!request(control).allows_reuse_at(
&response,
100_000,
Some(100_000),
false,
101_000
),
"{control}"
);
}
assert!(!request("max-stale").allows_reuse_at(
&response,
100_000,
Some(100_000),
true,
101_000
));
}
#[test]
fn max_age_and_min_fresh_use_current_age_not_time_since_storage() {
let response = headers(&[("Cache-Control", "max-age=600"), ("Age", "100")]);
for (control, expected) in [
("max-age=130", true),
("max-age=129", false),
("min-fresh=470", true),
("min-fresh=471", false),
] {
assert_eq!(
request(control).allows_reuse_at(&response, 100_000, None, false, 130_000),
expected,
"{control}"
);
}
}
#[test]
fn immutable_only_relaxes_soft_reload_while_fresh() {
let response = headers(&[("Cache-Control", "max-age=600, immutable")]);
assert!(request("max-age=0").allows_reuse_at(&response, 100_000, None, false, 130_000));
assert!(!request("no-cache").allows_reuse_at(&response, 100_000, None, false, 130_000));
assert!(!request("max-age=0").allows_reuse_at(&response, 100_000, None, true, 130_000));
assert!(!request("max-age=0").allows_reuse_at(&response, 100_000, None, false, 701_000));
assert!(request("max-stale=2").allows_reuse_at(&response, 100_000, None, false, 701_000));
}
#[test]
fn max_stale_does_not_invent_freshness_for_validator_only_entries() {
let response = headers(&[("ETag", "\"validator\"")]);
for expiry in [None, Some(0)] {
assert!(
!request("max-stale").allows_reuse_at(&response, 100_000, expiry, false, 130_000)
);
}
}
#[test]
fn expires_lifetime_accounts_for_clock_skew_and_age() {
let response = headers(&[
("Date", "Thu, 01 Jan 1970 00:01:30 GMT"),
("Expires", "Thu, 01 Jan 1970 00:02:30 GMT"),
("Age", "20"),
]);
assert!(request("").allows_reuse_at(&response, 100_000, None, false, 139_999));
assert!(!request("").allows_reuse_at(&response, 100_000, None, false, 140_000));
assert!(request("max-stale=1").allows_reuse_at(&response, 100_000, None, false, 141_000));
assert!(!request("max-stale=1").allows_reuse_at(&response, 100_000, None, false, 141_001));
}
}
+2 -2
View File
@@ -53,7 +53,7 @@ fn cache_freshness_max_age_accounts_for_date_and_age() {
);
assert_eq!(
cache_expires_at_unix_ms(100_000, Some(60), None, Some(99_000), 80),
Some(100_000)
Some(80_000)
);
}
@@ -61,7 +61,7 @@ fn cache_freshness_max_age_accounts_for_date_and_age() {
fn cache_freshness_uses_expires_when_max_age_is_absent() {
assert_eq!(
cache_expires_at_unix_ms(100_000, None, Some(150_000), Some(90_000), 20),
Some(150_000)
Some(140_000)
);
}
@@ -13,8 +13,7 @@ use moli_fetch::{
BrowserRequestMetadata, RawResponse, Request, RequestHeaders, RequestResourceType, Response,
};
use moli_http_cache::{
cacheable_response_parts_policy, cached_response_is_fresh_immutable,
request_cache_control_forbids_storage, request_header_requires_validation, response_date,
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date,
response_vary_header_names, unix_now_ms, validation_headers_from_headers,
};
use parking_lot::Mutex;
@@ -82,7 +81,7 @@ pub(in crate::network) struct RawSubresourceCacheKey {
}
pub(in crate::network) enum RawSubresourceCacheLookup {
Fresh(Box<RawResponse>),
Local(Box<RawResponse>),
Revalidate(Box<RawResponse>),
}
@@ -120,6 +119,7 @@ enum MemoryCacheEntry {
struct RawSubresourceVariant {
response: Box<RawResponse>,
stored_at_unix_ms: u64,
expires_at_unix_ms: u64,
response_time: std::time::SystemTime,
vary_headers: Vec<(String, Option<Vec<u8>>)>,
@@ -687,22 +687,30 @@ impl SharedMemoryResourceCache {
)
})
.map(|(index, _)| index)?;
let policy = HttpCacheRequestPolicy::from_headers(request_headers);
let variant = &variants[index];
let reusable = request
.effective_cache_mode()
.allows_stale_http_cache_response()
|| policy.allows_reuse_at(
&variant.response.headers,
variant.stored_at_unix_ms,
Some(variant.expires_at_unix_ms),
request
.effective_cache_mode()
.requires_http_cache_validation(),
unix_now_ms(),
);
if !reusable && policy.only_if_cached {
// A cache-only miss must not retire a representation that a
// subsequent request with different constraints can still use.
return None;
}
let entry = self.remove_entry(&cache_key)?;
let MemoryCacheEntry::RawSubresource { mut variants, .. } = entry else {
unreachable!("raw cache key must map to a raw cache entry");
};
let force_validate = request
.effective_cache_mode()
.requires_http_cache_validation()
|| (request_headers
.to_byte_strings()
.iter()
.any(|(name, value)| request_header_requires_validation(name, value))
&& !cached_response_is_fresh_immutable(
&variants[index].response.headers,
Some(variants[index].expires_at_unix_ms),
));
if force_validate || variants[index].expires_at_unix_ms <= unix_now_ms() {
if !reusable {
// Retire the old entry before fetching its replacement. Failure,
// cancellation or a non-storable response must not resurrect it
// as a fresh hit for a later consumer.
@@ -713,7 +721,7 @@ impl SharedMemoryResourceCache {
}
let hit = variants[index].response.clone();
self.insert_raw_subresource_variants(key.clone(), variants);
Some(RawSubresourceCacheLookup::Fresh(hit))
Some(RawSubresourceCacheLookup::Local(hit))
}
pub(in crate::network) fn remove_raw_subresource(
@@ -741,6 +749,7 @@ impl SharedMemoryResourceCache {
return;
};
let variant = RawSubresourceVariant {
stored_at_unix_ms: unix_now_ms(),
response_time: response_date(&response.headers)
.unwrap_or_else(std::time::SystemTime::now),
response: Box::new(response),
@@ -1028,14 +1037,10 @@ fn raw_subresource_memory_cacheable_resource_type(request: &Request) -> bool {
}
fn raw_subresource_memory_cacheable_headers(headers: &RequestHeaders) -> bool {
headers.to_byte_strings().iter().all(|(name, value)| {
!memory_cache_sensitive_request_header(name)
&& (!name.eq_ignore_ascii_case("cache-control")
|| (request_header_requires_validation(name, value)
&& !request_cache_control_forbids_storage(value)))
&& (!name.eq_ignore_ascii_case("pragma")
|| request_header_requires_validation(name, value))
})
!HttpCacheRequestPolicy::from_headers(headers).no_store
&& headers
.iter()
.all(|(name, _)| !memory_cache_sensitive_request_header(name))
}
fn memory_cache_sensitive_request_header(name: &str) -> bool {
@@ -1105,9 +1110,9 @@ pub(in crate::network) fn raw_subresource_memory_cache_expiry(
&response.headers,
response.redirected,
)?;
let expires_at = policy.expires_at_unix_ms.unwrap_or(0);
(expires_at > unix_now_ms() || !validation_headers_from_headers(&response.headers).is_empty())
.then_some(expires_at)
policy
.expires_at_unix_ms
.or_else(|| (!validation_headers_from_headers(&response.headers).is_empty()).then_some(0))
}
fn script_text_memory_cache_expiry(request: &Request, response: &Response) -> Option<u64> {
+14 -2
View File
@@ -14,7 +14,9 @@ use moli_fetch::{
FetchCancelHandle, FetchConfig, NetworkFetchResult, RawResponse, Request, Response,
ResponseHead, StreamingRawResponse,
};
use moli_http_cache::{merge_not_modified_headers, validation_headers_from_headers};
use moli_http_cache::{
HttpCacheRequestPolicy, merge_not_modified_headers, validation_headers_from_headers,
};
use moli_parkable_image::ParkableImageManager;
use crate::{protocol_types::OptionalResourceFetchMask, types::SubresourceResourceType};
@@ -1003,6 +1005,8 @@ impl ResourceRequestClient {
.resource_runtime
.client()
.cache_request_headers(&request);
let cache_only =
HttpCacheRequestPolicy::from_headers(&cache_request_headers).only_if_cached;
let cache_key =
raw_subresource_memory_cache_key(&request, &cache_request_headers).map(|key| {
key.for_page_cache_partition(self.page_network_policy.memory_cache_partition_id())
@@ -1015,7 +1019,7 @@ impl ResourceRequestClient {
});
let mut network_request = request.clone();
let cached = match lookup {
Some(RawSubresourceCacheLookup::Fresh(cached)) => {
Some(RawSubresourceCacheLookup::Local(cached)) => {
return Ok(NetworkFetchResult::without_request_observation(
streaming_raw_response_from_cached_subresource(*cached)?,
));
@@ -1051,6 +1055,14 @@ impl ResourceRequestClient {
};
let (response, observation_journal) = observed.into_parts_with_observation_journal();
let mut response = response.with_lifetime_lease(self.resource_runtime.clone());
if cache_only && response.status == 504 {
// The transport checked its disk cache before producing this
// local miss. It must not evict any retained memory variant.
return Ok(NetworkFetchResult::with_observation_journal(
response,
observation_journal,
));
}
if response.status == 304
&& (!response.redirected || !request.follow_redirects)
&& response.final_url == request.url
@@ -1,4 +1,5 @@
use super::*;
mod directives;
mod vary;
use moli_fetch::{NetworkObservationJournal, RequestCacheMode, ResponseCacheState};
@@ -0,0 +1,301 @@
use super::*;
fn controlled(url: &str, control: &str) -> Result<Request> {
let mut request = request(url)?;
request.request_headers = vec![("Cache-Control".to_owned(), control.to_owned())].into();
Ok(request)
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_max_stale_uses_age_and_cache_only_misses_keep_the_body() -> Result<()> {
let (url, server) = server(vec![
reply(
200,
&[("Cache-Control", "max-age=1500"), ("Age", "2000")],
b"stale body",
),
reply(200, &[("Cache-Control", "max-age=600")], b"fresh body"),
])
.await?;
let client = ResourceRequestClient::new(&FetchConfig::default())?;
fetch(&client, request(&url)?, true).await?;
for control in [
"only-if-cached",
"only-if-cached, max-stale=499",
"only-if-cached, max-stale, no-store",
] {
let (miss, journal, cancel) = fetch(&client, controlled(&url, control)?, true).await?;
assert_eq!(miss.status, 504, "{control}");
assert!(miss.body_bytes().is_empty());
assert_eq!(miss.cache_state, ResponseCacheState::Local);
assert!(journal.is_empty());
assert!(cancel.response_timing().request_started_at().is_none());
assert!(miss.negotiated_http_version.is_none());
}
for control in ["only-if-cached, max-stale=1000", "max-stale"] {
let (hit, journal, _) = fetch(&client, controlled(&url, control)?, true).await?;
assert_eq!(hit.status, 200);
assert_eq!(hit.body_bytes(), b"stale body");
assert_eq!(hit.cache_state, ResponseCacheState::Local);
assert!(journal.is_empty());
}
assert_eq!(
fetch(&client, request(&url)?, false).await?.0.body_bytes(),
b"fresh body"
);
assert_eq!(server.await??.len(), 2);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_only_misses_preserve_all_matching_vary_variants() -> Result<()> {
let (url, server) = server(vec![
reply(
200,
&[
("Cache-Control", "max-age=1500"),
("Age", "2000"),
("Vary", "Foo"),
],
b"one",
),
reply(
200,
&[
("Cache-Control", "max-age=1500"),
("Age", "2000"),
("Vary", "Foo"),
],
b"two",
),
])
.await?;
let client = ResourceRequestClient::new(&FetchConfig::default())?;
let variant = |value: &str, control: &str| -> Result<Request> {
let mut request = controlled(&url, control)?;
request
.request_headers
.push(("Foo".into(), value.as_bytes().to_vec()));
Ok(request)
};
for value in ["one", "two"] {
assert_eq!(
fetch(&client, variant(value, "")?, false)
.await?
.0
.body_bytes(),
value.as_bytes()
);
}
for value in ["one", "two"] {
assert_eq!(
fetch(&client, variant(value, "only-if-cached")?, true)
.await?
.0
.status,
504
);
}
for value in ["two", "one"] {
let (hit, journal, _) = fetch(
&client,
variant(value, "max-stale=1000, only-if-cached")?,
true,
)
.await?;
assert_eq!(hit.body_bytes(), value.as_bytes());
assert_eq!(hit.cache_state, ResponseCacheState::Local);
assert!(journal.is_empty());
}
assert_eq!(server.await??.len(), 2);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_max_stale_obeys_response_validation_requirements() -> Result<()> {
for policy in ["no-cache, max-age=600", "must-revalidate, max-age=0"] {
let (url, server) = server(vec![
reply(
200,
&[("Cache-Control", policy), ("ETag", "\"version\"")],
b"original",
),
reply(304, &[("Cache-Control", "max-age=600")], b""),
])
.await?;
let client = ResourceRequestClient::new(&FetchConfig::default())?;
fetch(&client, request(&url)?, false).await?;
let (miss, journal, _) = fetch(
&client,
controlled(&url, "only-if-cached, max-stale")?,
true,
)
.await?;
assert_eq!(miss.status, 504, "{policy}");
assert!(journal.is_empty());
let (validated, journal, _) = fetch(&client, controlled(&url, "max-stale")?, true).await?;
assert_eq!(validated.body_bytes(), b"original");
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
assert_eq!(journal.final_response_observation().unwrap().status(), 304);
let requests = server.await??;
assert_eq!(requests.len(), 2);
assert_eq!(
header(&requests[1], "if-none-match"),
Some(b"\"version\"".as_slice())
);
}
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_request_constraints_validate_before_reusing_fresh_immutable_entries()
-> Result<()> {
let (url, server) = server(vec![
reply(
200,
&[
("Cache-Control", "max-age=600, immutable"),
("Age", "200"),
("ETag", "\"original\""),
],
b"original",
),
reply(
304,
&[("Cache-Control", "max-age=600, immutable"), ("Age", "0")],
b"",
),
reply(
304,
&[("Cache-Control", "max-age=600, immutable"), ("Age", "0")],
b"",
),
])
.await?;
let client = ResourceRequestClient::new(&FetchConfig::default())?;
fetch(&client, request(&url)?, false).await?;
assert_eq!(
fetch(&client, controlled(&url, "max-age=0")?, false)
.await?
.0
.cache_state,
ResponseCacheState::Local
);
for request in [
controlled(&url, "only-if-cached, no-cache")?,
controlled(&url, "only-if-cached")?.with_cache_mode(RequestCacheMode::Validate),
controlled(&url, "only-if-cached")?.with_cache_mode(RequestCacheMode::Bypass),
] {
let (miss, journal, _) = fetch(&client, request, true).await?;
assert_eq!(miss.status, 504);
assert!(journal.is_empty());
}
for control in ["max-age=100", "min-fresh=1000"] {
let validated = fetch(&client, controlled(&url, control)?, true).await?.0;
assert_eq!(validated.body_bytes(), b"original");
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
}
assert_eq!(
fetch(&client, controlled(&url, "only-if-cached")?, false)
.await?
.0
.cache_state,
ResponseCacheState::Local
);
let requests = server.await??;
assert_eq!(requests.len(), 3);
assert!(
requests[1..]
.iter()
.all(|request| header(request, "if-none-match") == Some(b"\"original\"".as_slice()))
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_only_redirects_stop_at_uncached_targets() -> Result<()> {
let (url, server) = server(vec![reply(
302,
&[("Location", "/missing"), ("Cache-Control", "max-age=600")],
b"",
)])
.await?;
let client = ResourceRequestClient::new(&FetchConfig::default())?;
let origin = Url::parse(&url)?;
let request = || {
Request::new_browser("GET", origin.clone(), None, Vec::new(), (&origin).into())
.with_credentials_mode(RequestCredentialsMode::SameOrigin)
.with_browser_request_metadata(BrowserRequestMetadata::Fetch)
.with_page_network_policy()
};
// Prime the complete redirect representation independently. The browser
// redirect loop may drop an unread body before its cache tee completes.
assert_eq!(
fetch(&client, request().with_follow_redirects(false), false)
.await?
.0
.status,
302
);
let mut request = request();
request.request_headers =
vec![("Cache-Control".to_owned(), "only-if-cached".to_owned())].into();
let observed = crate::network_host::fetch_browser_subresource_raw_stream_for_owner(
&client,
request,
None,
Vec::new(),
None,
)
.await
.map_err(anyhow::Error::msg)?;
let (stream, journal) = observed.into_parts_with_observation_journal();
let response = stream.into_materialized_raw_response().await?;
assert_eq!(response.status, 504);
assert_eq!(response.final_url.path(), "/missing");
assert_eq!(response.redirect_chain.len(), 1);
assert!(response.redirected);
assert!(journal.is_empty());
assert_eq!(server.await??.len(), 1);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn raw_memory_cache_only_miss_can_still_use_the_native_disk_cache() -> Result<()> {
let (url, server) = server(vec![reply(
200,
&[("Cache-Control", "max-age=600")],
b"disk body",
)])
.await?;
let cache_dir = std::env::temp_dir().join(format!(
"moli-renderer-cache-only-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)?
.as_nanos()
));
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let native =
moli_fetch::FetchClient::new(&config, moli_cookie_jar::new_shared_browser_cookie_store());
assert_eq!(
native.fetch_raw(request(&url)?).await?.body_bytes(),
b"disk body"
);
native.shutdown();
let client = ResourceRequestClient::new(&config)?;
for _ in 0..2 {
let (hit, journal, cancel) =
fetch(&client, controlled(&url, "only-if-cached")?, true).await?;
assert_eq!(hit.status, 200);
assert_eq!(hit.body_bytes(), b"disk body");
assert_eq!(hit.cache_state, ResponseCacheState::Local);
assert!(journal.is_empty());
assert!(cancel.response_timing().request_started_at().is_none());
}
assert_eq!(server.await??.len(), 1);
drop(client);
std::fs::remove_dir_all(cache_dir)?;
Ok(())
}