mirror of
https://github.com/lexmount/moli.git
synced 2026-11-21 16:03:10 +00:00
fix(fetch): honor cache-only requests and stale constraints
This commit is contained in:
@@ -6,8 +6,7 @@ mod vary;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use moli_http_cache::{
|
||||
HttpCacheEntryMetadata, HttpCacheStats, HttpCacheStore, cached_response_is_stale,
|
||||
request_header_requires_validation,
|
||||
HttpCacheEntryMetadata, HttpCacheRequestPolicy, HttpCacheStats, HttpCacheStore,
|
||||
};
|
||||
use moli_url_policy::ensure_http_network_transport_url;
|
||||
use url::Url;
|
||||
@@ -16,7 +15,7 @@ use std::path::Path;
|
||||
|
||||
use crate::{FetchConfig, Request};
|
||||
|
||||
use super::outgoing_request_headers_for_url;
|
||||
use super::outgoing_request_header_bytes_for_url;
|
||||
|
||||
pub fn clear_http_cache(config: &FetchConfig) -> Result<()> {
|
||||
let Some(cache_dir) = config.http_cache_dir() else {
|
||||
@@ -88,25 +87,27 @@ pub(crate) struct CachedStreamingResponseLookup {
|
||||
pub(crate) status: u16,
|
||||
pub(crate) headers: Vec<(String, Vec<u8>)>,
|
||||
pub(crate) body: std::io::BufReader<std::fs::File>,
|
||||
pub(crate) expires_at_unix_ms: Option<u64>,
|
||||
pub(crate) force_validate: bool,
|
||||
pub(crate) allow_stale: bool,
|
||||
pub(crate) needs_validation: bool,
|
||||
pub(crate) cache_state: crate::ResponseCacheState,
|
||||
}
|
||||
|
||||
pub(crate) fn cached_streaming_response_is_stale(record: &CachedStreamingResponseLookup) -> bool {
|
||||
!record.allow_stale
|
||||
&& cached_response_is_stale(record.expires_at_unix_ms, record.force_validate)
|
||||
pub(crate) fn cached_streaming_response_needs_validation(
|
||||
record: &CachedStreamingResponseLookup,
|
||||
) -> bool {
|
||||
record.needs_validation
|
||||
}
|
||||
|
||||
pub(super) fn request_cache_control_requires_validation(
|
||||
pub(crate) fn request_cache_policy_for_url(
|
||||
config: &FetchConfig,
|
||||
request: &Request,
|
||||
request_url: &Url,
|
||||
) -> bool {
|
||||
outgoing_request_headers_for_url(config, request, request_url, None)
|
||||
.iter()
|
||||
.any(|(name, value)| request_header_requires_validation(name, value))
|
||||
) -> HttpCacheRequestPolicy {
|
||||
HttpCacheRequestPolicy::from_headers(&outgoing_request_header_bytes_for_url(
|
||||
config,
|
||||
request,
|
||||
request_url,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn next_redirect_url_from_parts(
|
||||
|
||||
Vendored
+1
-1
@@ -25,7 +25,7 @@ pub(super) fn cache_store_and_key_for_request(
|
||||
|| !subresource_validation_allows_http_cache(request)
|
||||
|| cookie_header.is_some()
|
||||
|| literal_cookie_header_for_request(config, request, request_url).is_some()
|
||||
|| !request_headers_allow_http_cache(&request.request_headers.to_byte_strings())
|
||||
|| !request_headers_allow_http_cache(&request.request_headers)
|
||||
|| !matches!(request_url.scheme(), "http" | "https")
|
||||
|| !request_url.username().is_empty()
|
||||
|| request_url.password().is_some()
|
||||
|
||||
+17
-30
@@ -1,7 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use moli_http_cache::{
|
||||
HttpCacheBodyWriter, HttpCachedEntryReader, cached_response_is_fresh_immutable,
|
||||
};
|
||||
use moli_http_cache::{HttpCacheBodyWriter, HttpCachedEntryReader, unix_now_ms};
|
||||
use url::Url;
|
||||
|
||||
use crate::{FetchConfig, Request};
|
||||
@@ -10,7 +8,7 @@ use super::{
|
||||
CachedStreamingResponseLookup,
|
||||
key::{cache_store_and_key_for_request, normalized_cache_url_string},
|
||||
metadata::cache_metadata_for_response_parts,
|
||||
request_cache_control_requires_validation,
|
||||
request_cache_policy_for_url,
|
||||
vary::vary_headers_match,
|
||||
};
|
||||
|
||||
@@ -48,15 +46,23 @@ pub(crate) fn load_cached_streaming_response_lookup(
|
||||
// Normalized cache keys should not leak the first request's fragment into
|
||||
// later streaming hits.
|
||||
cached.metadata.final_url = request_url.to_string();
|
||||
let force_validate = cache_lookup_force_validate(config, request, request_url, &cached);
|
||||
let needs_validation = !request
|
||||
.effective_cache_mode()
|
||||
.allows_stale_http_cache_response()
|
||||
&& !request_cache_policy_for_url(config, request, request_url).allows_reuse_at(
|
||||
&cached.metadata.headers,
|
||||
cached.metadata.stored_at_unix_ms,
|
||||
cached.metadata.expires_at_unix_ms,
|
||||
request
|
||||
.effective_cache_mode()
|
||||
.requires_http_cache_validation(),
|
||||
unix_now_ms(),
|
||||
);
|
||||
let _ = store.touch_loaded_entry(&key, &cached.metadata);
|
||||
Ok(Some(cached_streaming_record_from_entry(
|
||||
key,
|
||||
cached,
|
||||
force_validate,
|
||||
request
|
||||
.effective_cache_mode()
|
||||
.allows_stale_http_cache_response(),
|
||||
needs_validation,
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -156,8 +162,7 @@ pub(crate) fn finish_streaming_cached_response(
|
||||
fn cached_streaming_record_from_entry(
|
||||
key: String,
|
||||
entry: HttpCachedEntryReader,
|
||||
force_validate: bool,
|
||||
allow_stale: bool,
|
||||
needs_validation: bool,
|
||||
) -> CachedStreamingResponseLookup {
|
||||
let metadata = entry.metadata.clone();
|
||||
CachedStreamingResponseLookup {
|
||||
@@ -167,9 +172,7 @@ fn cached_streaming_record_from_entry(
|
||||
status: entry.metadata.status,
|
||||
headers: entry.metadata.headers,
|
||||
body: entry.body,
|
||||
expires_at_unix_ms: entry.metadata.expires_at_unix_ms,
|
||||
force_validate,
|
||||
allow_stale,
|
||||
needs_validation,
|
||||
cache_state: crate::ResponseCacheState::Local,
|
||||
}
|
||||
}
|
||||
@@ -183,19 +186,3 @@ fn entry_reader_body_exceeds_response_limit(
|
||||
.zip(cached.body_len().ok())
|
||||
.is_some_and(|(limit, body_len)| body_len > limit as u64)
|
||||
}
|
||||
|
||||
fn cache_lookup_force_validate(
|
||||
config: &FetchConfig,
|
||||
request: &Request,
|
||||
request_url: &Url,
|
||||
cached: &HttpCachedEntryReader,
|
||||
) -> bool {
|
||||
request
|
||||
.effective_cache_mode()
|
||||
.requires_http_cache_validation()
|
||||
|| (request_cache_control_requires_validation(config, request, request_url)
|
||||
&& !cached_response_is_fresh_immutable(
|
||||
&cached.metadata.headers,
|
||||
cached.metadata.expires_at_unix_ms,
|
||||
))
|
||||
}
|
||||
|
||||
+1
-1
@@ -26,6 +26,7 @@ pub(crate) fn merge_cached_not_modified_streaming_response_lookup(
|
||||
let headers = merge_not_modified_headers(&cached.headers, not_modified_headers);
|
||||
cached.headers = headers.clone();
|
||||
cached.cache_state = crate::ResponseCacheState::Validated;
|
||||
cached.needs_validation = false;
|
||||
cached.metadata.headers = headers.clone();
|
||||
|
||||
let final_url = Url::parse(&cached.final_url).with_context(|| {
|
||||
@@ -52,7 +53,6 @@ pub(crate) fn merge_cached_not_modified_streaming_response_lookup(
|
||||
cached.metadata.last_used_at_unix_ms = metadata.last_used_at_unix_ms;
|
||||
cached.metadata.expires_at_unix_ms = metadata.expires_at_unix_ms;
|
||||
cached.metadata.vary_headers = metadata.vary_headers;
|
||||
cached.expires_at_unix_ms = cached.metadata.expires_at_unix_ms;
|
||||
if let Some((store, key)) =
|
||||
cache_store_and_key_for_request(config, request, request_url, cookie_header)
|
||||
&& key == cached.key
|
||||
|
||||
+12
-15
@@ -1,9 +1,6 @@
|
||||
use std::str;
|
||||
|
||||
use moli_http_cache::{
|
||||
HttpCacheVaryHeader, request_cache_control_forbids_storage, request_header_requires_validation,
|
||||
response_vary_header_names,
|
||||
};
|
||||
use moli_http_cache::{HttpCacheRequestPolicy, HttpCacheVaryHeader, response_vary_header_names};
|
||||
use url::Url;
|
||||
|
||||
use crate::{FetchConfig, Request};
|
||||
@@ -64,17 +61,17 @@ pub(super) fn vary_headers_match(
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn request_headers_allow_http_cache(headers: &[(String, String)]) -> bool {
|
||||
headers.iter().enumerate().all(|(index, (name, value))| {
|
||||
// Revalidation controls select the existing conditional-cache path;
|
||||
// they must not turn a fresh entry into an unconditional cache miss.
|
||||
// Other unsupported controls keep the conservative bypass policy.
|
||||
(supported_cache_request_header(name) || request_header_requires_validation(name, value))
|
||||
&& !(name.eq_ignore_ascii_case("cache-control")
|
||||
&& request_cache_control_forbids_storage(value))
|
||||
&& !headers[..index]
|
||||
.iter()
|
||||
.any(|(previous, _)| previous.eq_ignore_ascii_case(name))
|
||||
pub(super) fn request_headers_allow_http_cache(headers: &crate::RequestHeaders) -> bool {
|
||||
if HttpCacheRequestPolicy::from_headers(headers).no_store {
|
||||
return false;
|
||||
}
|
||||
headers.iter().enumerate().all(|(index, (name, _))| {
|
||||
name.eq_ignore_ascii_case("cache-control")
|
||||
|| name.eq_ignore_ascii_case("pragma")
|
||||
|| (supported_cache_request_header(name)
|
||||
&& !headers[..index]
|
||||
.iter()
|
||||
.any(|(previous, _)| previous.eq_ignore_ascii_case(name)))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -22,10 +22,11 @@ use url::{Host, Url};
|
||||
|
||||
pub(crate) use self::cache::{
|
||||
CachedStreamingResponseLookup, cached_streaming_response_body_exceeds_response_limit,
|
||||
cached_streaming_response_is_stale, create_streaming_cache_body_writer_for_response_parts,
|
||||
finish_streaming_cached_response, load_cached_streaming_response_lookup,
|
||||
merge_cached_not_modified_streaming_response_lookup, next_followed_redirect_url_from_parts,
|
||||
next_redirect_url_from_parts, remove_cached_response, response_headers_forbid_cache_storage,
|
||||
cached_streaming_response_needs_validation,
|
||||
create_streaming_cache_body_writer_for_response_parts, finish_streaming_cached_response,
|
||||
load_cached_streaming_response_lookup, merge_cached_not_modified_streaming_response_lookup,
|
||||
next_followed_redirect_url_from_parts, next_redirect_url_from_parts, remove_cached_response,
|
||||
request_cache_policy_for_url, response_headers_forbid_cache_storage,
|
||||
validation_headers_for_cached_streaming_response_lookup,
|
||||
};
|
||||
pub use self::cache::{
|
||||
|
||||
+99
-10
@@ -42,12 +42,12 @@ use crate::{
|
||||
CachedStreamingResponseLookup, RawStreamingResponseCollector, RequestHttpVersion,
|
||||
RequestTransferMetrics, ResponseCollector, StreamingCachePlan, StreamingHtmlResponseStart,
|
||||
StreamingResponseCollector, cached_streaming_response_body_exceeds_response_limit,
|
||||
cached_streaming_response_is_stale, configure_easy, cookie_access_report_for_request,
|
||||
cookie_header_from_report, finish_streaming_cached_response,
|
||||
load_cached_streaming_response_lookup, log_request_completion,
|
||||
merge_cached_not_modified_streaming_response_lookup,
|
||||
cached_streaming_response_needs_validation, configure_easy,
|
||||
cookie_access_report_for_request, cookie_header_from_report,
|
||||
finish_streaming_cached_response, load_cached_streaming_response_lookup,
|
||||
log_request_completion, merge_cached_not_modified_streaming_response_lookup,
|
||||
network_request_extra_info_from_headers, next_followed_redirect_url_from_parts,
|
||||
outgoing_request_headers_for_url, remove_cached_response,
|
||||
outgoing_request_headers_for_url, remove_cached_response, request_cache_policy_for_url,
|
||||
response_headers_forbid_cache_storage, store_response_cookies, transfer_metrics_from_easy,
|
||||
validation_headers_for_cached_streaming_response_lookup,
|
||||
},
|
||||
@@ -886,6 +886,7 @@ impl RuntimeOwner {
|
||||
) {
|
||||
return Err((job.response_tx, error));
|
||||
}
|
||||
|
||||
let request_cookie_report = if job.request.allows_credentials_for_url(&job.current_url) {
|
||||
match cookie_access_report_for_request(
|
||||
&self.cookie_store,
|
||||
@@ -912,7 +913,7 @@ impl RuntimeOwner {
|
||||
&job.current_url,
|
||||
cookie_header.as_deref(),
|
||||
) {
|
||||
Ok(Some(cached)) if !cached_streaming_response_is_stale(&cached) => {
|
||||
Ok(Some(cached)) if !cached_streaming_response_needs_validation(&cached) => {
|
||||
if prepared_request
|
||||
.response_policy
|
||||
.observe_response(&job.current_url, &cached.headers)
|
||||
@@ -940,6 +941,17 @@ impl RuntimeOwner {
|
||||
if job.request.effective_cache_mode() == crate::RequestCacheMode::OnlyIfCached {
|
||||
return Err((job.response_tx, anyhow::Error::new(crate::FetchCacheMiss)));
|
||||
}
|
||||
if request_cache_policy_for_url(&self.config, &job.request, &job.current_url).only_if_cached
|
||||
{
|
||||
let head = cache_only_miss_start(&job.request, &job.current_url).into_head();
|
||||
job.response_progress.commit();
|
||||
return Ok(JobOutcome::Complete(
|
||||
job.response_tx,
|
||||
Box::new(CompletedBufferedResponse::Raw(
|
||||
RawResponse::from_head_and_body(head, Vec::new()),
|
||||
)),
|
||||
));
|
||||
}
|
||||
let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new(
|
||||
Some(job.cancel_handle.clone()),
|
||||
)));
|
||||
@@ -1065,9 +1077,10 @@ impl RuntimeOwner {
|
||||
job.request.prepare_accept_header(&self.config);
|
||||
job.response_progress = job.cancel_handle.begin_response_progress();
|
||||
if job.cancel_handle.is_cancelled() {
|
||||
let easy = job.easy.take();
|
||||
return Err((
|
||||
Box::new(job),
|
||||
None,
|
||||
easy,
|
||||
anyhow::Error::new(crate::FetchCancelled)
|
||||
.context("fetch runtime request cancelled"),
|
||||
));
|
||||
@@ -1105,7 +1118,9 @@ impl RuntimeOwner {
|
||||
&job.current_url,
|
||||
cookie_header.as_deref(),
|
||||
) {
|
||||
Ok(Some(cached_lookup)) if !cached_streaming_response_is_stale(&cached_lookup) => {
|
||||
Ok(Some(cached_lookup))
|
||||
if !cached_streaming_response_needs_validation(&cached_lookup) =>
|
||||
{
|
||||
if prepared_request
|
||||
.response_policy
|
||||
.observe_response(&job.current_url, &cached_lookup.headers)
|
||||
@@ -1141,6 +1156,12 @@ impl RuntimeOwner {
|
||||
anyhow::Error::new(crate::FetchCacheMiss),
|
||||
));
|
||||
}
|
||||
if request_cache_policy_for_url(&self.config, &prepared_request.request, &job.current_url)
|
||||
.only_if_cached
|
||||
{
|
||||
complete_cache_only_html_miss(job);
|
||||
return Ok(StreamingJobOutcome::Complete);
|
||||
}
|
||||
|
||||
let mut easy = job.easy.take().unwrap_or_else(|| {
|
||||
Easy2::new(FetchTransferHandler::new_streaming(
|
||||
@@ -1300,9 +1321,10 @@ impl RuntimeOwner {
|
||||
job.request.prepare_accept_header(&self.config);
|
||||
job.response_progress = job.cancel_handle.begin_response_progress();
|
||||
if job.cancel_handle.is_cancelled() {
|
||||
let easy = job.easy.take();
|
||||
return Err((
|
||||
Box::new(job),
|
||||
None,
|
||||
easy,
|
||||
anyhow::Error::new(crate::FetchCancelled)
|
||||
.context("fetch runtime request cancelled"),
|
||||
));
|
||||
@@ -1341,7 +1363,9 @@ impl RuntimeOwner {
|
||||
&job.current_url,
|
||||
cookie_header.as_deref(),
|
||||
) {
|
||||
Ok(Some(cached_lookup)) if !cached_streaming_response_is_stale(&cached_lookup) => {
|
||||
Ok(Some(cached_lookup))
|
||||
if !cached_streaming_response_needs_validation(&cached_lookup) =>
|
||||
{
|
||||
if prepared_request
|
||||
.response_policy
|
||||
.observe_response(&job.current_url, &cached_lookup.headers)
|
||||
@@ -1380,6 +1404,12 @@ impl RuntimeOwner {
|
||||
anyhow::Error::new(crate::FetchCacheMiss),
|
||||
));
|
||||
}
|
||||
if request_cache_policy_for_url(&self.config, &prepared_request.request, &job.current_url)
|
||||
.only_if_cached
|
||||
{
|
||||
complete_cache_only_raw_miss(job);
|
||||
return Ok(StreamingJobOutcome::Complete);
|
||||
}
|
||||
|
||||
let mut easy = job.easy.take().unwrap_or_else(|| {
|
||||
Easy2::new(FetchTransferHandler::new_raw_streaming(
|
||||
@@ -3819,6 +3849,65 @@ fn take_failed_proxy_connect_response(
|
||||
.take_failed_proxy_connect_response(connect_status)
|
||||
}
|
||||
|
||||
fn cache_only_miss_start(request: &Request, current_url: &Url) -> StreamingHtmlResponseStart {
|
||||
StreamingHtmlResponseStart {
|
||||
final_url: current_url.clone(),
|
||||
status: 504,
|
||||
status_text: Some("Gateway Timeout".to_owned()),
|
||||
headers: vec![
|
||||
("content-length".to_owned(), b"0".to_vec()),
|
||||
("cache-control".to_owned(), b"no-store".to_vec()),
|
||||
],
|
||||
request_cookie_report: None,
|
||||
cookie_set_reports: Vec::new(),
|
||||
redirected: !request.redirect_chain.is_empty(),
|
||||
redirect_chain: request.redirect_chain.clone(),
|
||||
from_cache: false,
|
||||
// This is a local result, not a stored representation or a wire
|
||||
// response. Consumers must not invent transfer bytes or protocol data.
|
||||
cache_state: crate::ResponseCacheState::Local,
|
||||
preload_state: Default::default(),
|
||||
negotiated_http_version: None,
|
||||
network_request_extra_info: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn complete_cache_only_html_miss(mut job: StreamingRuntimeJob) {
|
||||
let start = cache_only_miss_start(&job.request, &job.current_url);
|
||||
let (started_tx, body_tx) = if let Some(easy) = job.easy.as_mut() {
|
||||
easy.get_mut()
|
||||
.streaming_mut()
|
||||
.expect("HTML job requires an HTML collector")
|
||||
.take_response_channels()
|
||||
} else {
|
||||
(job.started_tx.take(), job.body_tx.take())
|
||||
};
|
||||
drop(body_tx);
|
||||
job.response_progress.commit();
|
||||
if let Some(started_tx) = started_tx {
|
||||
let _ = started_tx.send(Ok(start));
|
||||
}
|
||||
let _ = job.completion_tx.send(Ok(()));
|
||||
}
|
||||
|
||||
fn complete_cache_only_raw_miss(mut job: StreamingRawRuntimeJob) {
|
||||
let start = cache_only_miss_start(&job.request, &job.current_url);
|
||||
let (started_tx, body_tx) = if let Some(easy) = job.easy.as_mut() {
|
||||
easy.get_mut()
|
||||
.raw_streaming_mut()
|
||||
.expect("raw job requires a raw collector")
|
||||
.take_response_channels()
|
||||
} else {
|
||||
(job.started_tx.take(), job.body_tx.take())
|
||||
};
|
||||
drop(body_tx);
|
||||
job.response_progress.commit();
|
||||
if let Some(started_tx) = started_tx {
|
||||
let _ = started_tx.send(Ok(start));
|
||||
}
|
||||
let _ = job.completion_tx.send(Ok(()));
|
||||
}
|
||||
|
||||
fn proxy_connect_response_start(
|
||||
current_url: &Url,
|
||||
redirect_chain: &[RedirectInfo],
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
use super::*;
|
||||
use crate::ResponseCacheState;
|
||||
|
||||
fn controlled(url: &str, value: &str) -> Result<Request> {
|
||||
let mut request = Request::get(url)?;
|
||||
request.request_headers = vec![("Cache-Control".to_owned(), value.to_owned())].into();
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_only_requests_preserve_cancellation_before_local_completion() -> Result<()> {
|
||||
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("must not be requested")]);
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
for follow_redirects in [true, false] {
|
||||
let cancel = FetchCancelHandle::new();
|
||||
cancel.cancel();
|
||||
let request = controlled(&server.url(), "only-if-cached")?.with_redirect_handling(
|
||||
if follow_redirects {
|
||||
crate::RequestRedirectHandling::Follow
|
||||
} else {
|
||||
crate::RequestRedirectHandling::FollowByCaller
|
||||
},
|
||||
);
|
||||
let error = client
|
||||
.fetch_with_cancel(request, cancel)
|
||||
.await
|
||||
.expect_err("an already cancelled cache-only request must not complete with 504");
|
||||
assert!(error.is::<crate::FetchCancelled>(), "{error:#}");
|
||||
}
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 0);
|
||||
server.shutdown();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_only_misses_never_contact_origin_on_raw_text_html_or_buffered_paths() -> Result<()> {
|
||||
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("must not be requested")]);
|
||||
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
|
||||
for path in 0..4 {
|
||||
let cancel = FetchCancelHandle::new();
|
||||
let request = controlled(&server.url(), "only-if-cached")?;
|
||||
let head = match path {
|
||||
0 => {
|
||||
let observed = client
|
||||
.fetch_raw_stream_with_cancel_and_network_metadata(request, cancel.clone())
|
||||
.await?;
|
||||
let (stream, journal) = observed.into_parts_with_observation_journal();
|
||||
assert!(journal.is_empty());
|
||||
assert!(stream.network_request_extra_info().is_none());
|
||||
let response = stream.into_materialized_raw_response().await?;
|
||||
assert!(response.body_bytes().is_empty());
|
||||
assert!(cancel.response_timing().request_started_at().is_none());
|
||||
assert!(cancel.response_timing().response_started_at().is_none());
|
||||
assert!(cancel.response_timing().response_end_at().is_some());
|
||||
response.head()
|
||||
}
|
||||
1 | 2 => {
|
||||
let request = request.with_redirect_handling(if path == 1 {
|
||||
crate::RequestRedirectHandling::Follow
|
||||
} else {
|
||||
crate::RequestRedirectHandling::FollowByCaller
|
||||
});
|
||||
let observed = client
|
||||
.fetch_with_cancel_and_network_metadata(request, cancel.clone())
|
||||
.await?;
|
||||
let (response, journal) = observed.into_parts_with_observation_journal();
|
||||
assert!(journal.is_empty());
|
||||
assert!(response.body_bytes().is_empty());
|
||||
assert!(cancel.response_timing().request_started_at().is_none());
|
||||
response.head()
|
||||
}
|
||||
_ => {
|
||||
let response = client
|
||||
.fetch_html_stream(request)
|
||||
.await?
|
||||
.into_materialized_text_response()
|
||||
.await?;
|
||||
assert!(response.body_bytes().is_empty());
|
||||
response.head()
|
||||
}
|
||||
};
|
||||
assert_eq!(head.status, 504, "path {path}");
|
||||
assert_eq!(head.status_text.as_deref(), Some("Gateway Timeout"));
|
||||
assert!(!head.from_cache);
|
||||
assert_eq!(head.cache_state, ResponseCacheState::Local);
|
||||
assert!(head.negotiated_http_version.is_none());
|
||||
}
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 0);
|
||||
assert!(server.requests().is_empty());
|
||||
server.shutdown();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_cache_staleness_bounds_and_cache_only_misses_preserve_stored_entries() -> Result<()> {
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let server = ScriptedHttpServer::spawn(vec![
|
||||
ScriptedResponse::ok("stale")
|
||||
.with_header("Cache-Control", "max-age=1500")
|
||||
.with_header("Age", "2000"),
|
||||
ScriptedResponse::ok("fresh").with_header("Cache-Control", "max-age=600"),
|
||||
]);
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
assert_eq!(
|
||||
client
|
||||
.fetch_raw(Request::get(&server.url())?)
|
||||
.await?
|
||||
.body_bytes(),
|
||||
b"stale"
|
||||
);
|
||||
for control in [
|
||||
"only-if-cached",
|
||||
"only-if-cached, max-stale=499",
|
||||
"only-if-cached, max-stale=1000, no-store",
|
||||
] {
|
||||
let response = client
|
||||
.fetch_raw(controlled(&server.url(), control)?)
|
||||
.await?;
|
||||
assert_eq!(response.status, 504, "{control}");
|
||||
assert!(response.body_bytes().is_empty());
|
||||
}
|
||||
for control in ["max-stale=1000, only-if-cached", "max-stale"] {
|
||||
let response = client
|
||||
.fetch_raw(controlled(&server.url(), control)?)
|
||||
.await?;
|
||||
assert_eq!(response.status, 200);
|
||||
assert_eq!(response.body_bytes(), b"stale");
|
||||
assert_eq!(response.cache_state, ResponseCacheState::Local);
|
||||
}
|
||||
assert_eq!(server.hits(), 1);
|
||||
assert_eq!(
|
||||
client
|
||||
.fetch_raw(Request::get(&server.url())?)
|
||||
.await?
|
||||
.body_bytes(),
|
||||
b"fresh"
|
||||
);
|
||||
let hit = client
|
||||
.fetch_raw(controlled(&server.url(), "only-if-cached")?)
|
||||
.await?;
|
||||
assert_eq!(hit.body_bytes(), b"fresh");
|
||||
assert_eq!(hit.cache_state, ResponseCacheState::Local);
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 2);
|
||||
server.shutdown();
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_cache_request_age_and_freshness_constraints_validate_with_original_body() -> Result<()>
|
||||
{
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let server = ScriptedHttpServer::spawn(vec![
|
||||
ScriptedResponse::ok("original")
|
||||
.with_header("Cache-Control", "max-age=600")
|
||||
.with_header("Age", "200")
|
||||
.with_header("ETag", "\"original\""),
|
||||
ScriptedResponse::status(304, "Not Modified")
|
||||
.with_header("Cache-Control", "max-age=600")
|
||||
.with_header("Age", "0"),
|
||||
ScriptedResponse::status(304, "Not Modified")
|
||||
.with_header("Cache-Control", "max-age=600")
|
||||
.with_header("Age", "0"),
|
||||
]);
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
client.fetch_raw(Request::get(&server.url())?).await?;
|
||||
for control in ["max-age=100", "min-fresh=1000"] {
|
||||
let response = client
|
||||
.fetch_raw(controlled(&server.url(), control)?)
|
||||
.await?;
|
||||
assert_eq!(response.body_bytes(), b"original");
|
||||
assert_eq!(response.cache_state, ResponseCacheState::Validated);
|
||||
}
|
||||
let hit = client
|
||||
.fetch_raw(controlled(
|
||||
&server.url(),
|
||||
"only-if-cached, max-age=600, min-fresh=500",
|
||||
)?)
|
||||
.await?;
|
||||
assert_eq!(hit.body_bytes(), b"original");
|
||||
assert_eq!(hit.cache_state, ResponseCacheState::Local);
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 3);
|
||||
assert!(server.requests()[1..].iter().all(|request| {
|
||||
request
|
||||
.to_ascii_lowercase()
|
||||
.contains("if-none-match: \"original\"")
|
||||
}));
|
||||
server.shutdown();
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_cache_max_stale_cannot_override_response_validation_requirements() -> Result<()> {
|
||||
for control in ["no-cache, max-age=600", "must-revalidate, max-age=0"] {
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let server = ScriptedHttpServer::spawn(vec![
|
||||
ScriptedResponse::ok("original")
|
||||
.with_header("Cache-Control", control)
|
||||
.with_header("ETag", "\"original\""),
|
||||
ScriptedResponse::status(304, "Not Modified")
|
||||
.with_header("Cache-Control", "max-age=600"),
|
||||
]);
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
client.fetch_raw(Request::get(&server.url())?).await?;
|
||||
let miss = client
|
||||
.fetch_raw(controlled(&server.url(), "only-if-cached, max-stale")?)
|
||||
.await?;
|
||||
assert_eq!(miss.status, 504, "{control}");
|
||||
assert_eq!(server.hits(), 1);
|
||||
let validated = client
|
||||
.fetch_raw(controlled(&server.url(), "max-stale")?)
|
||||
.await?;
|
||||
assert_eq!(validated.body_bytes(), b"original");
|
||||
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 2);
|
||||
assert!(
|
||||
server.requests()[1]
|
||||
.to_ascii_lowercase()
|
||||
.contains("if-none-match: \"original\"")
|
||||
);
|
||||
server.shutdown();
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_cache_only_redirects_stop_at_uncached_hops_without_network() -> Result<()> {
|
||||
for html in [false, true] {
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let server = ScriptedHttpServer::spawn(vec![
|
||||
ScriptedResponse::status(302, "Found")
|
||||
.with_header("Location", "/missing")
|
||||
.with_header("Cache-Control", "max-age=600"),
|
||||
ScriptedResponse::ok("must not be requested"),
|
||||
]);
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
let prime = Request::get(&server.url())?.with_redirect_handling(if false {
|
||||
crate::RequestRedirectHandling::Follow
|
||||
} else {
|
||||
crate::RequestRedirectHandling::FollowByCaller
|
||||
});
|
||||
assert_eq!(client.fetch_raw(prime).await?.status, 302);
|
||||
let request = controlled(&server.url(), "only-if-cached")?;
|
||||
let head = if html {
|
||||
client
|
||||
.fetch_html_stream(request)
|
||||
.await?
|
||||
.into_materialized_text_response()
|
||||
.await?
|
||||
.head()
|
||||
} else {
|
||||
let observed = client.fetch_raw_with_network_metadata(request).await?;
|
||||
let (response, journal) = observed.into_parts_with_observation_journal();
|
||||
assert!(journal.is_empty());
|
||||
response.head()
|
||||
};
|
||||
assert_eq!(head.status, 504);
|
||||
assert_eq!(head.final_url.path(), "/missing");
|
||||
assert!(head.redirected);
|
||||
assert_eq!(head.redirect_chain.len(), 1);
|
||||
assert!(head.redirect_chain[0].from_cache);
|
||||
client.shutdown();
|
||||
assert_eq!(server.hits(), 1);
|
||||
server.shutdown();
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_only_hits_are_looked_up_before_misses_on_every_transport() -> Result<()> {
|
||||
for (age, directive, expected_status) in [
|
||||
(None, "only-if-cached", 200),
|
||||
(Some("60"), "only-if-cached, max-stale=120", 200),
|
||||
(Some("60"), "only-if-cached", 504),
|
||||
] {
|
||||
let cache_dir = unique_test_cache_dir();
|
||||
let mut response = ScriptedResponse::ok("cached").with_header(
|
||||
"Cache-Control",
|
||||
if age.is_some() {
|
||||
"max-age=0"
|
||||
} else {
|
||||
"max-age=600"
|
||||
},
|
||||
);
|
||||
if let Some(age) = age {
|
||||
response = response.with_header("Age", age);
|
||||
}
|
||||
let server = ScriptedHttpServer::spawn(vec![
|
||||
response,
|
||||
ScriptedResponse::ok("must not be requested"),
|
||||
]);
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
|
||||
assert_eq!(
|
||||
client
|
||||
.fetch_raw(Request::get(&server.url())?)
|
||||
.await?
|
||||
.body_bytes(),
|
||||
b"cached",
|
||||
);
|
||||
for path in 0..4 {
|
||||
let cancel = FetchCancelHandle::new();
|
||||
let request = controlled(&server.url(), directive)?;
|
||||
let response = match path {
|
||||
0 => {
|
||||
let observed = client
|
||||
.fetch_raw_stream_with_cancel_and_network_metadata(request, cancel.clone())
|
||||
.await?;
|
||||
let (response, journal) = observed.into_parts_with_observation_journal();
|
||||
assert!(journal.is_empty());
|
||||
response.into_materialized_raw_response().await?
|
||||
}
|
||||
1 | 2 => {
|
||||
let request = if path == 2 {
|
||||
request.with_single_hop()
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let observed = client
|
||||
.fetch_with_cancel_and_network_metadata(request, cancel.clone())
|
||||
.await?;
|
||||
let (response, journal) = observed.into_parts_with_observation_journal();
|
||||
assert!(journal.is_empty());
|
||||
RawResponse::from_head_and_body(response.head(), response.body_bytes().to_vec())
|
||||
}
|
||||
_ => {
|
||||
let response = client
|
||||
.fetch_html_stream(request)
|
||||
.await?
|
||||
.into_materialized_text_response()
|
||||
.await?;
|
||||
RawResponse::from_head_and_body(response.head(), response.body_bytes().to_vec())
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
response.status, expected_status,
|
||||
"age={age:?}, {directive}, path={path}"
|
||||
);
|
||||
assert_eq!(response.cache_state, ResponseCacheState::Local);
|
||||
assert_eq!(response.from_cache, expected_status == 200);
|
||||
assert_eq!(
|
||||
response.body_bytes(),
|
||||
if expected_status == 200 {
|
||||
b"cached".as_slice()
|
||||
} else {
|
||||
b"".as_slice()
|
||||
}
|
||||
);
|
||||
assert!(cancel.response_timing().request_started_at().is_none());
|
||||
assert_eq!(server.hits(), 1, "cache-only must not contact the origin");
|
||||
}
|
||||
client.shutdown();
|
||||
server.shutdown();
|
||||
fs::remove_dir_all(cache_dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
mod cache_request_controls;
|
||||
mod cache_request_controls_modes;
|
||||
mod cache_request_directives;
|
||||
mod cache_request_modes;
|
||||
mod cache_vary_encoding;
|
||||
mod cookie_context;
|
||||
|
||||
@@ -15,6 +15,7 @@ mod eviction;
|
||||
mod metadata;
|
||||
mod path_safety;
|
||||
mod policy;
|
||||
mod request_policy;
|
||||
mod revalidation;
|
||||
mod store;
|
||||
mod time;
|
||||
@@ -30,6 +31,7 @@ pub use self::{
|
||||
request_header_requires_validation, request_pragma_requires_validation,
|
||||
response_cache_policy, response_date, unix_now_ms,
|
||||
},
|
||||
request_policy::HttpCacheRequestPolicy,
|
||||
revalidation::{merge_not_modified_headers, validation_headers_from_headers},
|
||||
store::{HttpCacheEntryInfo, HttpCacheStats, HttpCacheStore, HttpCachedEntryReader},
|
||||
vary::response_vary_header_names,
|
||||
|
||||
@@ -18,13 +18,23 @@ pub fn response_date(headers: &[(String, Vec<u8>)]) -> Option<std::time::SystemT
|
||||
pub struct HttpCacheResponsePolicy {
|
||||
pub store: bool,
|
||||
pub expires_at_unix_ms: Option<u64>,
|
||||
pub requires_validation: bool,
|
||||
pub must_revalidate: bool,
|
||||
}
|
||||
|
||||
/// Parses the subset of HTTP response cache policy currently enforced by
|
||||
/// Moli's conservative disk cache.
|
||||
pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponsePolicy {
|
||||
response_cache_policy_at(headers, unix_now_ms())
|
||||
}
|
||||
|
||||
pub(crate) fn response_cache_policy_at(
|
||||
headers: &[(String, Vec<u8>)],
|
||||
now_ms: u64,
|
||||
) -> HttpCacheResponsePolicy {
|
||||
let mut store = true;
|
||||
let mut requires_validation = false;
|
||||
let mut must_revalidate = false;
|
||||
let mut max_age_seconds = None;
|
||||
let mut expires_header_unix_ms = None;
|
||||
let mut date_unix_ms = None;
|
||||
@@ -51,6 +61,8 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
|
||||
store = false;
|
||||
} else if directive_name.eq_ignore_ascii_case("no-cache") {
|
||||
requires_validation = true;
|
||||
} else if directive_name.eq_ignore_ascii_case("must-revalidate") {
|
||||
must_revalidate = true;
|
||||
} else if directive_name.eq_ignore_ascii_case("max-age")
|
||||
&& let Some(raw) = directive_value
|
||||
&& let Ok(seconds) = raw.parse::<u64>()
|
||||
@@ -89,7 +101,7 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
|
||||
let expires_at_unix_ms = (!requires_validation)
|
||||
.then(|| {
|
||||
cache_expires_at_unix_ms(
|
||||
unix_now_ms(),
|
||||
now_ms,
|
||||
max_age_seconds,
|
||||
expires_header_unix_ms,
|
||||
date_unix_ms,
|
||||
@@ -101,6 +113,8 @@ pub fn response_cache_policy(headers: &[(String, Vec<u8>)]) -> HttpCacheResponse
|
||||
HttpCacheResponsePolicy {
|
||||
store,
|
||||
expires_at_unix_ms,
|
||||
requires_validation,
|
||||
must_revalidate,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +164,7 @@ pub fn cached_response_is_fresh_immutable(
|
||||
cached_response_is_fresh_immutable_at(unix_now_ms(), headers, expires_at_unix_ms)
|
||||
}
|
||||
|
||||
fn cached_response_is_fresh_immutable_at(
|
||||
pub(crate) fn cached_response_is_fresh_immutable_at(
|
||||
now_ms: u64,
|
||||
headers: &[(String, Vec<u8>)],
|
||||
expires_at_unix_ms: Option<u64>,
|
||||
@@ -224,17 +238,21 @@ pub fn cache_expires_at_unix_ms(
|
||||
date_unix_ms: Option<u64>,
|
||||
age_seconds: u64,
|
||||
) -> Option<u64> {
|
||||
if let Some(max_age_seconds) = max_age_seconds {
|
||||
let apparent_age_ms = date_unix_ms
|
||||
.map(|date_ms| now_ms.saturating_sub(date_ms))
|
||||
.unwrap_or_default();
|
||||
let response_age_ms = apparent_age_ms.max(age_seconds.saturating_mul(1000));
|
||||
let freshness_ms = max_age_seconds
|
||||
.saturating_mul(1000)
|
||||
.saturating_sub(response_age_ms);
|
||||
return Some(now_ms.saturating_add(freshness_ms));
|
||||
}
|
||||
expires_header_unix_ms
|
||||
let apparent_age_ms = date_unix_ms
|
||||
.map(|date| now_ms.saturating_sub(date))
|
||||
.unwrap_or_default();
|
||||
let response_age_ms = apparent_age_ms.max(age_seconds.saturating_mul(1000));
|
||||
let lifetime_ms = if let Some(max_age_seconds) = max_age_seconds {
|
||||
i128::from(max_age_seconds.saturating_mul(1000))
|
||||
} else {
|
||||
i128::from(expires_header_unix_ms?) - i128::from(date_unix_ms.unwrap_or(now_ms))
|
||||
};
|
||||
// Keep time already spent stale. Clamping the remaining lifetime to zero
|
||||
// would make every old response appear to have only just expired.
|
||||
Some(
|
||||
(i128::from(now_ms) + lifetime_ms - i128::from(response_age_ms))
|
||||
.clamp(0, i128::from(u64::MAX)) as u64,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn unix_now_ms() -> u64 {
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
use moli_header_field::{split_outside_quoted_strings, unquote_parameter_value};
|
||||
|
||||
use crate::policy::{cached_response_is_fresh_immutable_at, response_cache_policy_at};
|
||||
|
||||
/// Constraints on reuse of an HTTP representation. `Some(u64::MAX)` permits
|
||||
/// unbounded staleness; `None` never permits stale reuse.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct HttpCacheRequestPolicy {
|
||||
pub no_store: bool,
|
||||
pub no_cache: bool,
|
||||
pub only_if_cached: bool,
|
||||
pub max_age_seconds: Option<u64>,
|
||||
pub max_stale_seconds: Option<u64>,
|
||||
pub min_fresh_seconds: Option<u64>,
|
||||
}
|
||||
|
||||
impl HttpCacheRequestPolicy {
|
||||
pub fn from_headers(headers: &[(String, Vec<u8>)]) -> Self {
|
||||
let mut policy = Self::default();
|
||||
let mut has_cache_control = false;
|
||||
let mut pragma_no_cache = false;
|
||||
for (name, bytes) in headers {
|
||||
let value = moli_header_field::decode_header_value(bytes);
|
||||
if name.eq_ignore_ascii_case("pragma") {
|
||||
pragma_no_cache |= crate::request_pragma_requires_validation(&value);
|
||||
}
|
||||
if !name.eq_ignore_ascii_case("cache-control") {
|
||||
continue;
|
||||
}
|
||||
has_cache_control = true;
|
||||
for directive in split_outside_quoted_strings(&value, ',') {
|
||||
let (name, value) = directive
|
||||
.trim()
|
||||
.split_once('=')
|
||||
.map(|(name, value)| (name.trim(), Some(unquote_parameter_value(value.trim()))))
|
||||
.unwrap_or((directive.trim(), None));
|
||||
match name.to_ascii_lowercase().as_str() {
|
||||
"no-store" => policy.no_store = true,
|
||||
"no-cache" => policy.no_cache = true,
|
||||
"only-if-cached" => policy.only_if_cached = true,
|
||||
"max-age" | "min-fresh" | "max-stale" => {
|
||||
let seconds = match value.as_deref() {
|
||||
None if name.eq_ignore_ascii_case("max-stale") => Some(u64::MAX),
|
||||
Some(value) => delta_seconds(value),
|
||||
None => None,
|
||||
};
|
||||
let Some(seconds) = seconds else {
|
||||
// A malformed known constraint must not widen reuse,
|
||||
// even if a later field supplies a valid duplicate.
|
||||
policy.no_cache = true;
|
||||
continue;
|
||||
};
|
||||
let slot = if name.eq_ignore_ascii_case("max-age") {
|
||||
&mut policy.max_age_seconds
|
||||
} else if name.eq_ignore_ascii_case("max-stale") {
|
||||
&mut policy.max_stale_seconds
|
||||
} else {
|
||||
&mut policy.min_fresh_seconds
|
||||
};
|
||||
*slot = Some(slot.map_or(seconds, |previous| {
|
||||
if name.eq_ignore_ascii_case("min-fresh") {
|
||||
previous.max(seconds)
|
||||
} else {
|
||||
previous.min(seconds)
|
||||
}
|
||||
}));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !has_cache_control && pragma_no_cache {
|
||||
policy.no_cache = true;
|
||||
}
|
||||
policy
|
||||
}
|
||||
|
||||
pub fn allows_reuse_at(
|
||||
&self,
|
||||
response_headers: &[(String, Vec<u8>)],
|
||||
stored_at_unix_ms: u64,
|
||||
expires_at_unix_ms: Option<u64>,
|
||||
force_validate: bool,
|
||||
now_unix_ms: u64,
|
||||
) -> bool {
|
||||
if self.no_store || self.no_cache || force_validate {
|
||||
return false;
|
||||
}
|
||||
let response = response_cache_policy_at(response_headers, stored_at_unix_ms);
|
||||
if !response.store || response.requires_validation {
|
||||
return false;
|
||||
}
|
||||
// Recalculate explicit freshness from the stored header facts. This
|
||||
// also handles persisted entries written before stale age was retained.
|
||||
let expiry = response
|
||||
.expires_at_unix_ms
|
||||
.or(expires_at_unix_ms.filter(|value| *value != 0));
|
||||
let Some(expiry) = expiry else {
|
||||
return false;
|
||||
};
|
||||
let fresh = now_unix_ms < expiry;
|
||||
if let Some(max_age) = self.max_age_seconds {
|
||||
let immutable_reload = max_age == 0
|
||||
&& fresh
|
||||
&& cached_response_is_fresh_immutable_at(
|
||||
now_unix_ms,
|
||||
response_headers,
|
||||
Some(expiry),
|
||||
);
|
||||
if !immutable_reload
|
||||
&& (max_age == 0
|
||||
|| current_age_ms(response_headers, stored_at_unix_ms, now_unix_ms)
|
||||
> max_age.saturating_mul(1000))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.min_fresh_seconds.is_some_and(|seconds| {
|
||||
now_unix_ms > expiry
|
||||
|| expiry.saturating_sub(now_unix_ms) < seconds.saturating_mul(1000)
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
fresh
|
||||
|| (!response.must_revalidate
|
||||
&& self.max_stale_seconds.is_some_and(|seconds| {
|
||||
now_unix_ms.saturating_sub(expiry) <= seconds.saturating_mul(1000)
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn delta_seconds(value: &str) -> Option<u64> {
|
||||
(!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()))
|
||||
.then(|| value.parse().unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
fn current_age_ms(headers: &[(String, Vec<u8>)], stored_at: u64, now: u64) -> u64 {
|
||||
let apparent_age = crate::response_date(headers)
|
||||
.and_then(|date| date.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|duration| {
|
||||
stored_at.saturating_sub(duration.as_millis().min(u128::from(u64::MAX)) as u64)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let age = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("age"))
|
||||
.filter_map(|(_, value)| {
|
||||
delta_seconds(moli_header_field::decode_header_value(value).trim())
|
||||
})
|
||||
.max()
|
||||
.unwrap_or_default()
|
||||
.saturating_mul(1000);
|
||||
apparent_age
|
||||
.max(age)
|
||||
.saturating_add(now.saturating_sub(stored_at))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn headers(fields: &[(&str, &str)]) -> Vec<(String, Vec<u8>)> {
|
||||
fields
|
||||
.iter()
|
||||
.map(|(name, value)| ((*name).into(), value.as_bytes().to_vec()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn request(value: &str) -> HttpCacheRequestPolicy {
|
||||
HttpCacheRequestPolicy::from_headers(&headers(&[("Cache-Control", value)]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_directives_merge_fields_without_reading_quoted_extensions() {
|
||||
let policy = HttpCacheRequestPolicy::from_headers(&headers(&[
|
||||
(
|
||||
"Cache-Control",
|
||||
"extension=\"only-if-cached, no-store\", MAX-AGE=90, max-stale",
|
||||
),
|
||||
(
|
||||
"cache-control",
|
||||
"max-age=30, max-stale=80, min-fresh=10, ONLY-IF-CACHED",
|
||||
),
|
||||
("Cache-Control", "max-age=60, max-stale=90, min-fresh=20"),
|
||||
]));
|
||||
assert!(!policy.no_store && !policy.no_cache);
|
||||
assert!(policy.only_if_cached);
|
||||
assert_eq!(policy.max_age_seconds, Some(30));
|
||||
assert_eq!(policy.max_stale_seconds, Some(80));
|
||||
assert_eq!(policy.min_fresh_seconds, Some(20));
|
||||
assert!(!request("extension=\"only-if-cached, no-store\"").only_if_cached);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_request_bounds_cannot_be_overridden_by_valid_duplicates() {
|
||||
for name in ["max-age", "min-fresh", "max-stale"] {
|
||||
for invalid in ["", "+1", "-1", "1.5", "one"] {
|
||||
let value = format!("{name}={invalid}, {name}=999, max-stale");
|
||||
assert!(request(&value).no_cache, "{value}");
|
||||
}
|
||||
}
|
||||
assert!(request("max-age").no_cache);
|
||||
assert!(request("min-fresh").no_cache);
|
||||
assert_eq!(request("max-stale").max_stale_seconds, Some(u64::MAX));
|
||||
assert_eq!(
|
||||
request("max-age=99999999999999999999999").max_age_seconds,
|
||||
Some(u64::MAX)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pragma_is_a_fallback_only_when_cache_control_is_absent() {
|
||||
assert!(HttpCacheRequestPolicy::from_headers(&headers(&[("Pragma", "no-cache")])).no_cache);
|
||||
assert!(
|
||||
!HttpCacheRequestPolicy::from_headers(&headers(&[
|
||||
("Pragma", "no-cache"),
|
||||
("Cache-Control", "max-age=60")
|
||||
]))
|
||||
.no_cache
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_stale_counts_age_before_storage_and_resident_time() {
|
||||
let response = headers(&[("Cache-Control", "max-age=1500"), ("Age", "2000")]);
|
||||
let stored = 3_000_000;
|
||||
// An older persisted expiry clamped to the storage time must not
|
||||
// hide the response's existing five hundred seconds of staleness.
|
||||
assert!(!request("max-stale=499").allows_reuse_at(
|
||||
&response,
|
||||
stored,
|
||||
Some(stored),
|
||||
false,
|
||||
stored
|
||||
));
|
||||
assert!(request("max-stale=500").allows_reuse_at(
|
||||
&response,
|
||||
stored,
|
||||
Some(stored),
|
||||
false,
|
||||
stored
|
||||
));
|
||||
assert!(!request("max-stale=500").allows_reuse_at(
|
||||
&response,
|
||||
stored,
|
||||
Some(stored),
|
||||
false,
|
||||
stored + 1
|
||||
));
|
||||
assert!(request("max-stale").allows_reuse_at(
|
||||
&response,
|
||||
stored,
|
||||
Some(stored),
|
||||
false,
|
||||
stored + 999_000
|
||||
));
|
||||
assert!(!request("").allows_reuse_at(&response, stored, Some(stored), false, stored));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_reuse_never_overrides_response_or_caller_validation_requirements() {
|
||||
for control in [
|
||||
"max-age=0, no-cache",
|
||||
"max-age=0, must-revalidate",
|
||||
"max-age=0, no-store",
|
||||
"max-age=0, private",
|
||||
] {
|
||||
let response = headers(&[("Cache-Control", control)]);
|
||||
assert!(
|
||||
!request("max-stale").allows_reuse_at(
|
||||
&response,
|
||||
100_000,
|
||||
Some(100_000),
|
||||
false,
|
||||
101_000
|
||||
),
|
||||
"{control}"
|
||||
);
|
||||
}
|
||||
let response = headers(&[("Cache-Control", "max-age=0")]);
|
||||
for control in [
|
||||
"no-cache, max-stale",
|
||||
"no-store, max-stale",
|
||||
"min-fresh=0, max-stale",
|
||||
] {
|
||||
assert!(
|
||||
!request(control).allows_reuse_at(
|
||||
&response,
|
||||
100_000,
|
||||
Some(100_000),
|
||||
false,
|
||||
101_000
|
||||
),
|
||||
"{control}"
|
||||
);
|
||||
}
|
||||
assert!(!request("max-stale").allows_reuse_at(
|
||||
&response,
|
||||
100_000,
|
||||
Some(100_000),
|
||||
true,
|
||||
101_000
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_age_and_min_fresh_use_current_age_not_time_since_storage() {
|
||||
let response = headers(&[("Cache-Control", "max-age=600"), ("Age", "100")]);
|
||||
for (control, expected) in [
|
||||
("max-age=130", true),
|
||||
("max-age=129", false),
|
||||
("min-fresh=470", true),
|
||||
("min-fresh=471", false),
|
||||
] {
|
||||
assert_eq!(
|
||||
request(control).allows_reuse_at(&response, 100_000, None, false, 130_000),
|
||||
expected,
|
||||
"{control}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn immutable_only_relaxes_soft_reload_while_fresh() {
|
||||
let response = headers(&[("Cache-Control", "max-age=600, immutable")]);
|
||||
assert!(request("max-age=0").allows_reuse_at(&response, 100_000, None, false, 130_000));
|
||||
assert!(!request("no-cache").allows_reuse_at(&response, 100_000, None, false, 130_000));
|
||||
assert!(!request("max-age=0").allows_reuse_at(&response, 100_000, None, true, 130_000));
|
||||
assert!(!request("max-age=0").allows_reuse_at(&response, 100_000, None, false, 701_000));
|
||||
assert!(request("max-stale=2").allows_reuse_at(&response, 100_000, None, false, 701_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_stale_does_not_invent_freshness_for_validator_only_entries() {
|
||||
let response = headers(&[("ETag", "\"validator\"")]);
|
||||
for expiry in [None, Some(0)] {
|
||||
assert!(
|
||||
!request("max-stale").allows_reuse_at(&response, 100_000, expiry, false, 130_000)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expires_lifetime_accounts_for_clock_skew_and_age() {
|
||||
let response = headers(&[
|
||||
("Date", "Thu, 01 Jan 1970 00:01:30 GMT"),
|
||||
("Expires", "Thu, 01 Jan 1970 00:02:30 GMT"),
|
||||
("Age", "20"),
|
||||
]);
|
||||
assert!(request("").allows_reuse_at(&response, 100_000, None, false, 139_999));
|
||||
assert!(!request("").allows_reuse_at(&response, 100_000, None, false, 140_000));
|
||||
assert!(request("max-stale=1").allows_reuse_at(&response, 100_000, None, false, 141_000));
|
||||
assert!(!request("max-stale=1").allows_reuse_at(&response, 100_000, None, false, 141_001));
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ fn cache_freshness_max_age_accounts_for_date_and_age() {
|
||||
);
|
||||
assert_eq!(
|
||||
cache_expires_at_unix_ms(100_000, Some(60), None, Some(99_000), 80),
|
||||
Some(100_000)
|
||||
Some(80_000)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ fn cache_freshness_max_age_accounts_for_date_and_age() {
|
||||
fn cache_freshness_uses_expires_when_max_age_is_absent() {
|
||||
assert_eq!(
|
||||
cache_expires_at_unix_ms(100_000, None, Some(150_000), Some(90_000), 20),
|
||||
Some(150_000)
|
||||
Some(140_000)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -13,8 +13,7 @@ use moli_fetch::{
|
||||
BrowserRequestMetadata, RawResponse, Request, RequestHeaders, RequestResourceType, Response,
|
||||
};
|
||||
use moli_http_cache::{
|
||||
cacheable_response_parts_policy, cached_response_is_fresh_immutable,
|
||||
request_cache_control_forbids_storage, request_header_requires_validation, response_date,
|
||||
HttpCacheRequestPolicy, cacheable_response_parts_policy, response_date,
|
||||
response_vary_header_names, unix_now_ms, validation_headers_from_headers,
|
||||
};
|
||||
use parking_lot::Mutex;
|
||||
@@ -82,7 +81,7 @@ pub(in crate::network) struct RawSubresourceCacheKey {
|
||||
}
|
||||
|
||||
pub(in crate::network) enum RawSubresourceCacheLookup {
|
||||
Fresh(Box<RawResponse>),
|
||||
Local(Box<RawResponse>),
|
||||
Revalidate(Box<RawResponse>),
|
||||
}
|
||||
|
||||
@@ -120,6 +119,7 @@ enum MemoryCacheEntry {
|
||||
|
||||
struct RawSubresourceVariant {
|
||||
response: Box<RawResponse>,
|
||||
stored_at_unix_ms: u64,
|
||||
expires_at_unix_ms: u64,
|
||||
response_time: std::time::SystemTime,
|
||||
vary_headers: Vec<(String, Option<Vec<u8>>)>,
|
||||
@@ -687,22 +687,30 @@ impl SharedMemoryResourceCache {
|
||||
)
|
||||
})
|
||||
.map(|(index, _)| index)?;
|
||||
let policy = HttpCacheRequestPolicy::from_headers(request_headers);
|
||||
let variant = &variants[index];
|
||||
let reusable = request
|
||||
.effective_cache_mode()
|
||||
.allows_stale_http_cache_response()
|
||||
|| policy.allows_reuse_at(
|
||||
&variant.response.headers,
|
||||
variant.stored_at_unix_ms,
|
||||
Some(variant.expires_at_unix_ms),
|
||||
request
|
||||
.effective_cache_mode()
|
||||
.requires_http_cache_validation(),
|
||||
unix_now_ms(),
|
||||
);
|
||||
if !reusable && policy.only_if_cached {
|
||||
// A cache-only miss must not retire a representation that a
|
||||
// subsequent request with different constraints can still use.
|
||||
return None;
|
||||
}
|
||||
let entry = self.remove_entry(&cache_key)?;
|
||||
let MemoryCacheEntry::RawSubresource { mut variants, .. } = entry else {
|
||||
unreachable!("raw cache key must map to a raw cache entry");
|
||||
};
|
||||
let force_validate = request
|
||||
.effective_cache_mode()
|
||||
.requires_http_cache_validation()
|
||||
|| (request_headers
|
||||
.to_byte_strings()
|
||||
.iter()
|
||||
.any(|(name, value)| request_header_requires_validation(name, value))
|
||||
&& !cached_response_is_fresh_immutable(
|
||||
&variants[index].response.headers,
|
||||
Some(variants[index].expires_at_unix_ms),
|
||||
));
|
||||
if force_validate || variants[index].expires_at_unix_ms <= unix_now_ms() {
|
||||
if !reusable {
|
||||
// Retire the old entry before fetching its replacement. Failure,
|
||||
// cancellation or a non-storable response must not resurrect it
|
||||
// as a fresh hit for a later consumer.
|
||||
@@ -713,7 +721,7 @@ impl SharedMemoryResourceCache {
|
||||
}
|
||||
let hit = variants[index].response.clone();
|
||||
self.insert_raw_subresource_variants(key.clone(), variants);
|
||||
Some(RawSubresourceCacheLookup::Fresh(hit))
|
||||
Some(RawSubresourceCacheLookup::Local(hit))
|
||||
}
|
||||
|
||||
pub(in crate::network) fn remove_raw_subresource(
|
||||
@@ -741,6 +749,7 @@ impl SharedMemoryResourceCache {
|
||||
return;
|
||||
};
|
||||
let variant = RawSubresourceVariant {
|
||||
stored_at_unix_ms: unix_now_ms(),
|
||||
response_time: response_date(&response.headers)
|
||||
.unwrap_or_else(std::time::SystemTime::now),
|
||||
response: Box::new(response),
|
||||
@@ -1028,14 +1037,10 @@ fn raw_subresource_memory_cacheable_resource_type(request: &Request) -> bool {
|
||||
}
|
||||
|
||||
fn raw_subresource_memory_cacheable_headers(headers: &RequestHeaders) -> bool {
|
||||
headers.to_byte_strings().iter().all(|(name, value)| {
|
||||
!memory_cache_sensitive_request_header(name)
|
||||
&& (!name.eq_ignore_ascii_case("cache-control")
|
||||
|| (request_header_requires_validation(name, value)
|
||||
&& !request_cache_control_forbids_storage(value)))
|
||||
&& (!name.eq_ignore_ascii_case("pragma")
|
||||
|| request_header_requires_validation(name, value))
|
||||
})
|
||||
!HttpCacheRequestPolicy::from_headers(headers).no_store
|
||||
&& headers
|
||||
.iter()
|
||||
.all(|(name, _)| !memory_cache_sensitive_request_header(name))
|
||||
}
|
||||
|
||||
fn memory_cache_sensitive_request_header(name: &str) -> bool {
|
||||
@@ -1105,9 +1110,9 @@ pub(in crate::network) fn raw_subresource_memory_cache_expiry(
|
||||
&response.headers,
|
||||
response.redirected,
|
||||
)?;
|
||||
let expires_at = policy.expires_at_unix_ms.unwrap_or(0);
|
||||
(expires_at > unix_now_ms() || !validation_headers_from_headers(&response.headers).is_empty())
|
||||
.then_some(expires_at)
|
||||
policy
|
||||
.expires_at_unix_ms
|
||||
.or_else(|| (!validation_headers_from_headers(&response.headers).is_empty()).then_some(0))
|
||||
}
|
||||
|
||||
fn script_text_memory_cache_expiry(request: &Request, response: &Response) -> Option<u64> {
|
||||
|
||||
@@ -14,7 +14,9 @@ use moli_fetch::{
|
||||
FetchCancelHandle, FetchConfig, NetworkFetchResult, RawResponse, Request, Response,
|
||||
ResponseHead, StreamingRawResponse,
|
||||
};
|
||||
use moli_http_cache::{merge_not_modified_headers, validation_headers_from_headers};
|
||||
use moli_http_cache::{
|
||||
HttpCacheRequestPolicy, merge_not_modified_headers, validation_headers_from_headers,
|
||||
};
|
||||
use moli_parkable_image::ParkableImageManager;
|
||||
|
||||
use crate::{protocol_types::OptionalResourceFetchMask, types::SubresourceResourceType};
|
||||
@@ -1003,6 +1005,8 @@ impl ResourceRequestClient {
|
||||
.resource_runtime
|
||||
.client()
|
||||
.cache_request_headers(&request);
|
||||
let cache_only =
|
||||
HttpCacheRequestPolicy::from_headers(&cache_request_headers).only_if_cached;
|
||||
let cache_key =
|
||||
raw_subresource_memory_cache_key(&request, &cache_request_headers).map(|key| {
|
||||
key.for_page_cache_partition(self.page_network_policy.memory_cache_partition_id())
|
||||
@@ -1015,7 +1019,7 @@ impl ResourceRequestClient {
|
||||
});
|
||||
let mut network_request = request.clone();
|
||||
let cached = match lookup {
|
||||
Some(RawSubresourceCacheLookup::Fresh(cached)) => {
|
||||
Some(RawSubresourceCacheLookup::Local(cached)) => {
|
||||
return Ok(NetworkFetchResult::without_request_observation(
|
||||
streaming_raw_response_from_cached_subresource(*cached)?,
|
||||
));
|
||||
@@ -1051,6 +1055,14 @@ impl ResourceRequestClient {
|
||||
};
|
||||
let (response, observation_journal) = observed.into_parts_with_observation_journal();
|
||||
let mut response = response.with_lifetime_lease(self.resource_runtime.clone());
|
||||
if cache_only && response.status == 504 {
|
||||
// The transport checked its disk cache before producing this
|
||||
// local miss. It must not evict any retained memory variant.
|
||||
return Ok(NetworkFetchResult::with_observation_journal(
|
||||
response,
|
||||
observation_journal,
|
||||
));
|
||||
}
|
||||
if response.status == 304
|
||||
&& (!response.redirected || !request.follow_redirects)
|
||||
&& response.final_url == request.url
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::*;
|
||||
mod directives;
|
||||
mod vary;
|
||||
use moli_fetch::{NetworkObservationJournal, RequestCacheMode, ResponseCacheState};
|
||||
|
||||
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
use super::*;
|
||||
|
||||
fn controlled(url: &str, control: &str) -> Result<Request> {
|
||||
let mut request = request(url)?;
|
||||
request.request_headers = vec![("Cache-Control".to_owned(), control.to_owned())].into();
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_max_stale_uses_age_and_cache_only_misses_keep_the_body() -> Result<()> {
|
||||
let (url, server) = server(vec![
|
||||
reply(
|
||||
200,
|
||||
&[("Cache-Control", "max-age=1500"), ("Age", "2000")],
|
||||
b"stale body",
|
||||
),
|
||||
reply(200, &[("Cache-Control", "max-age=600")], b"fresh body"),
|
||||
])
|
||||
.await?;
|
||||
let client = ResourceRequestClient::new(&FetchConfig::default())?;
|
||||
fetch(&client, request(&url)?, true).await?;
|
||||
for control in [
|
||||
"only-if-cached",
|
||||
"only-if-cached, max-stale=499",
|
||||
"only-if-cached, max-stale, no-store",
|
||||
] {
|
||||
let (miss, journal, cancel) = fetch(&client, controlled(&url, control)?, true).await?;
|
||||
assert_eq!(miss.status, 504, "{control}");
|
||||
assert!(miss.body_bytes().is_empty());
|
||||
assert_eq!(miss.cache_state, ResponseCacheState::Local);
|
||||
assert!(journal.is_empty());
|
||||
assert!(cancel.response_timing().request_started_at().is_none());
|
||||
assert!(miss.negotiated_http_version.is_none());
|
||||
}
|
||||
for control in ["only-if-cached, max-stale=1000", "max-stale"] {
|
||||
let (hit, journal, _) = fetch(&client, controlled(&url, control)?, true).await?;
|
||||
assert_eq!(hit.status, 200);
|
||||
assert_eq!(hit.body_bytes(), b"stale body");
|
||||
assert_eq!(hit.cache_state, ResponseCacheState::Local);
|
||||
assert!(journal.is_empty());
|
||||
}
|
||||
assert_eq!(
|
||||
fetch(&client, request(&url)?, false).await?.0.body_bytes(),
|
||||
b"fresh body"
|
||||
);
|
||||
assert_eq!(server.await??.len(), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_only_misses_preserve_all_matching_vary_variants() -> Result<()> {
|
||||
let (url, server) = server(vec![
|
||||
reply(
|
||||
200,
|
||||
&[
|
||||
("Cache-Control", "max-age=1500"),
|
||||
("Age", "2000"),
|
||||
("Vary", "Foo"),
|
||||
],
|
||||
b"one",
|
||||
),
|
||||
reply(
|
||||
200,
|
||||
&[
|
||||
("Cache-Control", "max-age=1500"),
|
||||
("Age", "2000"),
|
||||
("Vary", "Foo"),
|
||||
],
|
||||
b"two",
|
||||
),
|
||||
])
|
||||
.await?;
|
||||
let client = ResourceRequestClient::new(&FetchConfig::default())?;
|
||||
let variant = |value: &str, control: &str| -> Result<Request> {
|
||||
let mut request = controlled(&url, control)?;
|
||||
request
|
||||
.request_headers
|
||||
.push(("Foo".into(), value.as_bytes().to_vec()));
|
||||
Ok(request)
|
||||
};
|
||||
for value in ["one", "two"] {
|
||||
assert_eq!(
|
||||
fetch(&client, variant(value, "")?, false)
|
||||
.await?
|
||||
.0
|
||||
.body_bytes(),
|
||||
value.as_bytes()
|
||||
);
|
||||
}
|
||||
for value in ["one", "two"] {
|
||||
assert_eq!(
|
||||
fetch(&client, variant(value, "only-if-cached")?, true)
|
||||
.await?
|
||||
.0
|
||||
.status,
|
||||
504
|
||||
);
|
||||
}
|
||||
for value in ["two", "one"] {
|
||||
let (hit, journal, _) = fetch(
|
||||
&client,
|
||||
variant(value, "max-stale=1000, only-if-cached")?,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(hit.body_bytes(), value.as_bytes());
|
||||
assert_eq!(hit.cache_state, ResponseCacheState::Local);
|
||||
assert!(journal.is_empty());
|
||||
}
|
||||
assert_eq!(server.await??.len(), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_max_stale_obeys_response_validation_requirements() -> Result<()> {
|
||||
for policy in ["no-cache, max-age=600", "must-revalidate, max-age=0"] {
|
||||
let (url, server) = server(vec![
|
||||
reply(
|
||||
200,
|
||||
&[("Cache-Control", policy), ("ETag", "\"version\"")],
|
||||
b"original",
|
||||
),
|
||||
reply(304, &[("Cache-Control", "max-age=600")], b""),
|
||||
])
|
||||
.await?;
|
||||
let client = ResourceRequestClient::new(&FetchConfig::default())?;
|
||||
fetch(&client, request(&url)?, false).await?;
|
||||
let (miss, journal, _) = fetch(
|
||||
&client,
|
||||
controlled(&url, "only-if-cached, max-stale")?,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(miss.status, 504, "{policy}");
|
||||
assert!(journal.is_empty());
|
||||
let (validated, journal, _) = fetch(&client, controlled(&url, "max-stale")?, true).await?;
|
||||
assert_eq!(validated.body_bytes(), b"original");
|
||||
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
|
||||
assert_eq!(journal.final_response_observation().unwrap().status(), 304);
|
||||
let requests = server.await??;
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(
|
||||
header(&requests[1], "if-none-match"),
|
||||
Some(b"\"version\"".as_slice())
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_request_constraints_validate_before_reusing_fresh_immutable_entries()
|
||||
-> Result<()> {
|
||||
let (url, server) = server(vec![
|
||||
reply(
|
||||
200,
|
||||
&[
|
||||
("Cache-Control", "max-age=600, immutable"),
|
||||
("Age", "200"),
|
||||
("ETag", "\"original\""),
|
||||
],
|
||||
b"original",
|
||||
),
|
||||
reply(
|
||||
304,
|
||||
&[("Cache-Control", "max-age=600, immutable"), ("Age", "0")],
|
||||
b"",
|
||||
),
|
||||
reply(
|
||||
304,
|
||||
&[("Cache-Control", "max-age=600, immutable"), ("Age", "0")],
|
||||
b"",
|
||||
),
|
||||
])
|
||||
.await?;
|
||||
let client = ResourceRequestClient::new(&FetchConfig::default())?;
|
||||
fetch(&client, request(&url)?, false).await?;
|
||||
assert_eq!(
|
||||
fetch(&client, controlled(&url, "max-age=0")?, false)
|
||||
.await?
|
||||
.0
|
||||
.cache_state,
|
||||
ResponseCacheState::Local
|
||||
);
|
||||
for request in [
|
||||
controlled(&url, "only-if-cached, no-cache")?,
|
||||
controlled(&url, "only-if-cached")?.with_cache_mode(RequestCacheMode::Validate),
|
||||
controlled(&url, "only-if-cached")?.with_cache_mode(RequestCacheMode::Bypass),
|
||||
] {
|
||||
let (miss, journal, _) = fetch(&client, request, true).await?;
|
||||
assert_eq!(miss.status, 504);
|
||||
assert!(journal.is_empty());
|
||||
}
|
||||
for control in ["max-age=100", "min-fresh=1000"] {
|
||||
let validated = fetch(&client, controlled(&url, control)?, true).await?.0;
|
||||
assert_eq!(validated.body_bytes(), b"original");
|
||||
assert_eq!(validated.cache_state, ResponseCacheState::Validated);
|
||||
}
|
||||
assert_eq!(
|
||||
fetch(&client, controlled(&url, "only-if-cached")?, false)
|
||||
.await?
|
||||
.0
|
||||
.cache_state,
|
||||
ResponseCacheState::Local
|
||||
);
|
||||
let requests = server.await??;
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert!(
|
||||
requests[1..]
|
||||
.iter()
|
||||
.all(|request| header(request, "if-none-match") == Some(b"\"original\"".as_slice()))
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_only_redirects_stop_at_uncached_targets() -> Result<()> {
|
||||
let (url, server) = server(vec![reply(
|
||||
302,
|
||||
&[("Location", "/missing"), ("Cache-Control", "max-age=600")],
|
||||
b"",
|
||||
)])
|
||||
.await?;
|
||||
let client = ResourceRequestClient::new(&FetchConfig::default())?;
|
||||
let origin = Url::parse(&url)?;
|
||||
let request = || {
|
||||
Request::new_browser("GET", origin.clone(), None, Vec::new(), (&origin).into())
|
||||
.with_credentials_mode(RequestCredentialsMode::SameOrigin)
|
||||
.with_browser_request_metadata(BrowserRequestMetadata::Fetch)
|
||||
.with_page_network_policy()
|
||||
};
|
||||
// Prime the complete redirect representation independently. The browser
|
||||
// redirect loop may drop an unread body before its cache tee completes.
|
||||
assert_eq!(
|
||||
fetch(&client, request().with_follow_redirects(false), false)
|
||||
.await?
|
||||
.0
|
||||
.status,
|
||||
302
|
||||
);
|
||||
let mut request = request();
|
||||
request.request_headers =
|
||||
vec![("Cache-Control".to_owned(), "only-if-cached".to_owned())].into();
|
||||
let observed = crate::network_host::fetch_browser_subresource_raw_stream_for_owner(
|
||||
&client,
|
||||
request,
|
||||
None,
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let (stream, journal) = observed.into_parts_with_observation_journal();
|
||||
let response = stream.into_materialized_raw_response().await?;
|
||||
assert_eq!(response.status, 504);
|
||||
assert_eq!(response.final_url.path(), "/missing");
|
||||
assert_eq!(response.redirect_chain.len(), 1);
|
||||
assert!(response.redirected);
|
||||
assert!(journal.is_empty());
|
||||
assert_eq!(server.await??.len(), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn raw_memory_cache_only_miss_can_still_use_the_native_disk_cache() -> Result<()> {
|
||||
let (url, server) = server(vec![reply(
|
||||
200,
|
||||
&[("Cache-Control", "max-age=600")],
|
||||
b"disk body",
|
||||
)])
|
||||
.await?;
|
||||
let cache_dir = std::env::temp_dir().join(format!(
|
||||
"moli-renderer-cache-only-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_nanos()
|
||||
));
|
||||
let mut config = FetchConfig::default();
|
||||
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
|
||||
let native =
|
||||
moli_fetch::FetchClient::new(&config, moli_cookie_jar::new_shared_browser_cookie_store());
|
||||
assert_eq!(
|
||||
native.fetch_raw(request(&url)?).await?.body_bytes(),
|
||||
b"disk body"
|
||||
);
|
||||
native.shutdown();
|
||||
let client = ResourceRequestClient::new(&config)?;
|
||||
for _ in 0..2 {
|
||||
let (hit, journal, cancel) =
|
||||
fetch(&client, controlled(&url, "only-if-cached")?, true).await?;
|
||||
assert_eq!(hit.status, 200);
|
||||
assert_eq!(hit.body_bytes(), b"disk body");
|
||||
assert_eq!(hit.cache_state, ResponseCacheState::Local);
|
||||
assert!(journal.is_empty());
|
||||
assert!(cancel.response_timing().request_started_at().is_none());
|
||||
}
|
||||
assert_eq!(server.await??.len(), 1);
|
||||
drop(client);
|
||||
std::fs::remove_dir_all(cache_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user