diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rs index 1a2682ae62..4056bbc055 100644 --- a/moli-core/src/runtime/tests.rs +++ b/moli-core/src/runtime/tests.rs @@ -91,6 +91,7 @@ async fn external_raw_bridge_drop_after_body_eof_cancels_pending_fetch_completio redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_rx, diff --git a/moli-core/tests/fixtures/preload-consumption-sw.js b/moli-core/tests/fixtures/preload-consumption-sw.js new file mode 100644 index 0000000000..a4d28072f6 --- /dev/null +++ b/moli-core/tests/fixtures/preload-consumption-sw.js @@ -0,0 +1,24 @@ +async function () { +return (async()=>{ + const controlled=navigator.serviceWorker.controller?Promise.resolve():new Promise(resolve=>navigator.serviceWorker.addEventListener('controllerchange',resolve,{once:true})); + await navigator.serviceWorker.register('/probe-worker.js');await navigator.serviceWorker.ready;await controlled; + const rows=[]; + for(const destination of ['script','fetch']) for(const source of ['basic','cors','default','opaque']) for(const bad of [false,true]){ + const token='sw'+rows.length,url=new URL('/probe-worker-asset',location.href);Object.entries({source,token,as:'script'}).forEach(([k,v])=>url.searchParams.set(k,v)); + await fetch('/probe-release?token='+token); + const link=document.createElement('link');Object.assign(link,{rel:'preload',as:destination,href:url.href,integrity:bad?'sha384-AAAA':''}); + const linkEvent=await new Promise(resolve=>{link.onload=link.onerror=e=>resolve(e.type);document.head.append(link)}); + let consumer; + if(destination==='fetch'){ + try{const response=await fetch(url,{mode:'no-cors',credentials:'include'});consumer={event:'load',type:response.type,text:await response.text(),status:response.status};} + catch(e){consumer={event:'error',name:e.name};} + }else{ + const script=document.createElement('script');script.src=url; + consumer=await new Promise(resolve=>{script.onload=script.onerror=e=>resolve({event:e.type});document.body.append(script)});script.remove(); + } + const counts=await(await fetch('/probe-worker-counts')).json(); + rows.push({destination,source,bad,linkEvent,consumer,events:counts[url.href],timing:performance.getEntriesByName(url.href).map(e=>e.initiatorType)});link.remove(); + } + return {rows,failures:rows.filter(r=>r.events!==1||r.linkEvent!==(r.bad?'error':'load')||r.consumer.event!==(r.bad?'error':'load')||r.timing.length!==1 ||(!r.bad&&r.destination==='fetch'&&r.source==='opaque'&&(r.consumer.type!=='opaque'||r.consumer.text!==''||r.consumer.status!==0)))}; +})(); +} diff --git a/moli-core/tests/fixtures/preload-consumption-worker.js b/moli-core/tests/fixtures/preload-consumption-worker.js new file mode 100644 index 0000000000..a7e00749fd --- /dev/null +++ b/moli-core/tests/fixtures/preload-consumption-worker.js @@ -0,0 +1,13 @@ +self.addEventListener('install', e=>e.waitUntil(self.skipWaiting())); +self.addEventListener('activate', e=>e.waitUntil(clients.claim())); +const counts={}; +self.addEventListener('fetch', e=>{ + const u=new URL(e.request.url); + if(u.pathname==='/probe-worker-counts'){e.respondWith(new Response(JSON.stringify(counts),{headers:{'Content-Type':'application/json'}}));return;} + if(u.pathname!=='/probe-worker-asset')return; + counts[u.href]=(counts[u.href]||0)+1; + const kind=u.searchParams.get('source'); + const target=new URL('/probe-asset'+u.search,self.location.href); + if(kind==='cors'||kind==='opaque')target.hostname=target.hostname==='localhost'?'127.0.0.1':'localhost'; + e.respondWith(kind==='default'?new Response('globalThis.preloadExecuted=(globalThis.preloadExecuted||0)+1;',{headers:{'Content-Type':'text/javascript'}}):fetch(target,{mode:kind==='opaque'?'no-cors':'cors'})); +}); diff --git a/moli-core/tests/fixtures/preload-consumption.js b/moli-core/tests/fixtures/preload-consumption.js new file mode 100644 index 0000000000..1ade387609 --- /dev/null +++ b/moli-core/tests/fixtures/preload-consumption.js @@ -0,0 +1,63 @@ +async function () { + const basic = await (async () => {return (async () => { + const rows=[]; + for (const as of ['script','image']) for (const cache of ['max-age=60','no-store']) + for (const phase of ['pending','complete']) for (const cors of [null,'anonymous','use-credentials']) { + const token=String(rows.length), url=new URL('/probe-asset',location.href); + url.searchParams.set('token',token);url.searchParams.set('as',as);url.searchParams.set('cache',cache); + const link=document.createElement('link');Object.assign(link,{rel:'preload',as,href:url.href}); + if(cors!==null)link.crossOrigin=cors; + const linkDone=new Promise(resolve=>{link.onload=link.onerror=e=>resolve(e.type)}); + document.head.append(link); + await fetch('/probe-started?token='+token); + if(phase==='complete'){await fetch('/probe-release?token='+token);await linkDone;} + const consumer=document.createElement(as==='script'?'script':'img'); + if(cors!==null)consumer.crossOrigin=cors; + let timingAtConsumer; + const consumerDone=new Promise(resolve=>{consumer.onload=consumer.onerror=e=>{timingAtConsumer=performance.getEntriesByName(url.href).length;resolve(e.type)}}); + consumer.src=url.href;document.body.append(consumer); + if(phase==='pending')await fetch('/probe-release?token='+token); + const events=await Promise.all([linkDone,consumerDone]); + const actual=await (await fetch('/probe-stats?token='+token)).json(); + rows.push({as,cache,phase,cors,events,timingAtConsumer,...actual,timing:performance.getEntriesByName(url.href).map(e=>({initiator:e.initiatorType,transfer:e.transferSize,encoded:e.encodedBodySize}))}); + link.remove();consumer.remove(); + } + return {rows,executions:globalThis.preloadExecuted||0,failures:rows.filter(r=>r.count!==1 || r.timing.length!==1 || r.timingAtConsumer!==1 || r.events.some(e=>e!=='load'))}; +})(); +})(); + const integrity = await (async () => {return (async () => { + const rows=[]; + for (const cache of ['max-age=60','no-store']) for (const phase of ['pending','complete']) + for (const [name,preloadIntegrity,consumerIntegrity,preloadEvent,consumerEvent,requests] of [["good-empty", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "", "load", "load", 1], ["bad-empty", "sha384-AAAA", "", "error", "error", 1], ["bad-same", "sha384-AAAA", "sha384-AAAA", "error", "error", 1], ["bad-good", "sha384-AAAA", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "error", "load", 2], ["good-bad", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "sha384-AAAA", "load", "error", 2], ["good-options", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ?ignored=option", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "load", "load", 1], ["good-different-algorithm", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "sha256-cRlncoZ/tNV1cbKTR2cXiKEwVFkjvOz4kToIRLKBiSE=", "load", "load", 2], ["empty-good", "", "sha384-l+rtjjQWlmlLFL2PC8iIPwnXbiW45bZ6LU1ERjlAxYKUFswREZzY3pPyoT0RsvxJ", "load", "load", 2]]) { + const token='sri-'+rows.length,url=new URL('/probe-asset',location.href); + url.searchParams.set('token',token);url.searchParams.set('as','script');url.searchParams.set('cache',cache); + const link=document.createElement('link');Object.assign(link,{rel:'preload',as:'script',href:url.href,integrity:preloadIntegrity}); + const linkDone=new Promise(resolve=>{link.onload=link.onerror=e=>resolve(e.type)});document.head.append(link); + await fetch('/probe-started?token='+token); + if(phase==='complete'){await fetch('/probe-release?token='+token);await linkDone;} + const consumer=document.createElement('script');consumer.integrity=consumerIntegrity;consumer.src=url.href; + const consumerDone=new Promise(resolve=>{consumer.onload=consumer.onerror=e=>resolve(e.type)});document.body.append(consumer); + if(phase==='pending')await fetch('/probe-release?token='+token); + const events=await Promise.all([linkDone,consumerDone]); + const actual=await(await fetch('/probe-stats?token='+token)).json(); + rows.push({name,cache,phase,events,expectedEvents:[preloadEvent,consumerEvent],expectedRequests:requests,...actual,timing:performance.getEntriesByName(url.href).map(e=>({initiator:e.initiatorType,transfer:e.transferSize}))}); + link.remove();consumer.remove(); + } + return {rows,failures:rows.filter(r=>JSON.stringify(r.events)!==JSON.stringify(r.expectedEvents)||r.timing.length!==r.expectedRequests || (r.cache==='no-store' && r.count!==r.expectedRequests) || (r.expectedRequests===1 && r.count!==1))}; +})();})(); + const oneShot = await (async () => { + const url = new URL('/probe-asset?token=one-shot&as=script', location.href); + await fetch('/probe-release?token=one-shot'); + const link = document.createElement('link'); + Object.assign(link, {rel:'preload', as:'fetch', crossOrigin:'anonymous', href:url.href}); + await new Promise(resolve => { link.onload=link.onerror=resolve; document.head.append(link); }); + const count = async () => (await (await fetch('/probe-stats?token=one-shot')).json()).count; + const counts = [await count()]; + await (await fetch(url, {headers:{Accept:'text/javascript'}})).text(); counts.push(await count()); + await (await fetch(url)).text(); counts.push(await count()); + await (await fetch(url)).text(); counts.push(await count()); + link.remove(); + return counts; + })(); + return {basic,integrity,oneShot}; +} diff --git a/moli-core/tests/scripts.rs b/moli-core/tests/scripts.rs index 88372b4522..cc0c7e6ac6 100644 --- a/moli-core/tests/scripts.rs +++ b/moli-core/tests/scripts.rs @@ -6790,8 +6790,8 @@ async fn parse_time_lifecycle_queue_can_stop_cleanly_at_load_stage_after_load_mi Ok(()) } -#[path = "scripts/document_write_errors.rs"] -mod document_write_errors; #[path = "scripts/child_markup_insertion.rs"] mod child_markup_insertion; +#[path = "scripts/preload_consumption.rs"] +mod preload_consumption; diff --git a/moli-core/tests/scripts/preload_consumption.rs b/moli-core/tests/scripts/preload_consumption.rs new file mode 100644 index 0000000000..b81f453b0b --- /dev/null +++ b/moli-core/tests/scripts/preload_consumption.rs @@ -0,0 +1,176 @@ +use super::*; +use anyhow::Context; +use parking_lot::Mutex; +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Notify, + task::{JoinHandle, JoinSet}, +}; + +const SCRIPT: &str = "globalThis.preloadExecuted=(globalThis.preloadExecuted||0)+1;"; +const PROBE: &str = include_str!("../fixtures/preload-consumption.js"); + +#[derive(Default)] +struct Asset { + count: AtomicUsize, + started: Notify, + released: AtomicBool, + release: Notify, +} + +impl Asset { + async fn wait(&self, release: bool) { + loop { + let notified = if release { + self.release.notified() + } else { + self.started.notified() + }; + tokio::pin!(notified); + notified.as_mut().enable(); + if if release { + self.released.load(Ordering::Acquire) + } else { + self.count.load(Ordering::Acquire) != 0 + } { + return; + } + notified.await; + } + } +} + +struct PreloadServer { + origin: String, + task: JoinHandle<()>, +} + +impl Drop for PreloadServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl PreloadServer { + async fn spawn() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let task = tokio::spawn(async move { + let assets = Arc::new(Mutex::new(HashMap::>::new())); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let Ok((mut stream, _)) = accepted else { break }; + let assets = assets.clone(); + connections.spawn(async move { + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") && request.len() < 16384 { + let Ok(byte) = stream.read_u8().await else { return }; + request.push(byte); + } + let request = String::from_utf8_lossy(&request); + let path = request.split_whitespace().nth(1).unwrap_or("/"); + let url = url::Url::parse(&format!("http://test{path}")).unwrap(); + let query: HashMap<_,_> = url.query_pairs().into_owned().collect(); + let token = query.get("token").cloned().unwrap_or_default(); + let asset = assets.lock().entry(token).or_default().clone(); + let mut cache = "no-store".to_owned(); + let (mime, body) = match url.path() { + "/probe-worker.js" => ("text/javascript", include_str!("../fixtures/preload-consumption-worker.js").to_owned()), + "/probe-asset" => { + asset.count.fetch_add(1, Ordering::AcqRel); + asset.started.notify_waiters(); + asset.wait(true).await; + cache = query.get("cache").cloned().unwrap_or(cache); + if query.get("as").is_some_and(|kind| kind == "image") { + ("image/svg+xml", "".to_owned()) + } else { ("text/javascript", SCRIPT.to_owned()) } + } + "/probe-started" | "/probe-release" | "/probe-stats" => { + if url.path() == "/probe-started" { asset.wait(false).await; } + if url.path() == "/probe-release" { + asset.released.store(true, Ordering::Release); + asset.release.notify_waiters(); + } + ("application/json", serde_json::json!({"count":asset.count.load(Ordering::Acquire)}).to_string()) + } + _ => ("text/html", "preload consumption".to_owned()), + }; + let origin = request.lines().find_map(|line| line.split_once(':').filter(|(name,_)| name.eq_ignore_ascii_case("Origin")).map(|(_,value)| value.trim())).unwrap_or("*"); + let response = format!("HTTP/1.1 200 OK\r\nContent-Type: {mime}\r\nCache-Control: {cache}\r\nAccess-Control-Allow-Origin: {origin}\r\nAccess-Control-Allow-Credentials: true\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + let _ = stream.write_all(response.as_bytes()).await; + }); + } + _ = connections.join_next(), if !connections.is_empty() => {} + } + } + }); + Ok(Self { origin, task }) + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn document_preloads_supply_pending_and_completed_consumers_including_failed_integrity() +-> Result<()> { + let server = PreloadServer::spawn().await?; + let mut config = AppConfig::default(); + config.set_optional_resource_fetch_mask(moli_page_types::OptionalResourceFetchMask::ALL); + let browser = Browser::new(config)?; + let mut page = browser.fetch(&server.origin).await?; + let result = tokio::time::timeout( + Duration::from_secs(30), + page.evaluate_runtime_expression_with_await_async( + &format!("({PROBE})().then(JSON.stringify)"), + true, + ), + ) + .await??; + let value: serde_json::Value = serde_json::from_str( + result["value"] + .as_str() + .context("preload consumption results")?, + )?; + assert_eq!(value["basic"]["rows"].as_array().unwrap().len(), 24); + assert_eq!(value["integrity"]["rows"].as_array().unwrap().len(), 32); + assert_eq!(value["basic"]["failures"], serde_json::json!([])); + assert_eq!(value["integrity"]["failures"], serde_json::json!([])); + assert_eq!(value["basic"]["executions"], 12); + assert_eq!(value["oneShot"], serde_json::json!([1, 2, 2, 3])); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn document_preload_consumption_preserves_service_worker_filters_without_redispatch() +-> Result<()> { + let server = PreloadServer::spawn().await?; + let mut config = AppConfig::default(); + config.set_optional_resource_fetch_mask(moli_page_types::OptionalResourceFetchMask::ALL); + let browser = Browser::new(config)?; + let mut page = browser.fetch(&server.origin).await?; + let probe = include_str!("../fixtures/preload-consumption-sw.js"); + let result = tokio::time::timeout( + Duration::from_secs(30), + page.evaluate_runtime_expression_with_await_async( + &format!("({probe})().then(JSON.stringify)"), + true, + ), + ) + .await??; + let value: serde_json::Value = serde_json::from_str( + result["value"] + .as_str() + .context("Service Worker consumption results")?, + )?; + assert_eq!(value["rows"].as_array().unwrap().len(), 16); + assert_eq!(value["failures"], serde_json::json!([])); + Ok(()) +} diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs index ef8f8641a1..2c226d6600 100644 --- a/moli-fetch/src/blocking/collectors.rs +++ b/moli-fetch/src/blocking/collectors.rs @@ -538,6 +538,7 @@ impl StreamingResponseCollector { redirect_chain: self.redirect_chain.clone(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: self.negotiated_http_version, network_request_extra_info: self.network_request_extra_info.clone(), })); @@ -942,6 +943,7 @@ impl RawStreamingResponseCollector { redirect_chain: self.redirect_chain.clone(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: self.negotiated_http_version, network_request_extra_info: self.network_request_extra_info.clone(), })); diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs index 8f0b66d5c1..3d857a1324 100644 --- a/moli-fetch/src/blocking/mod.rs +++ b/moli-fetch/src/blocking/mod.rs @@ -74,6 +74,7 @@ pub struct StreamingHtmlResponseStart { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, pub network_request_extra_info: Option, } @@ -91,6 +92,7 @@ impl StreamingHtmlResponseStart { redirect_chain: self.redirect_chain, from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs index a5e028316e..df5758260e 100644 --- a/moli-fetch/src/lib.rs +++ b/moli-fetch/src/lib.rs @@ -82,8 +82,9 @@ pub use request::{ }; pub use request_policy::{is_bad_port, should_request_be_blocked_due_to_bad_port}; pub use response::{ - NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, RawResponse, - RedirectInfo, RedirectSource, Response, ResponseBody, ResponseCacheState, ResponseHead, + FetchResponseFilter, NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, + RawResponse, RedirectInfo, RedirectSource, Response, ResponseBody, ResponseCacheState, + ResponseHead, ResponsePreloadState, }; pub use runtime::PendingStreamingRawResponse; pub use runtime::{ diff --git a/moli-fetch/src/response.rs b/moli-fetch/src/response.rs index 8e8802758f..9b94279131 100644 --- a/moli-fetch/src/response.rs +++ b/moli-fetch/src/response.rs @@ -16,6 +16,59 @@ pub enum ResponseCacheState { Validated, } +/// The actual Fetch response filter, including Service Worker responses whose +/// type cannot be reconstructed from the request or final URL. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum FetchResponseFilter { + Basic, + Cors(Vec), + Opaque, + OpaqueRedirect, +} + +impl FetchResponseFilter { + pub fn is_readable(&self) -> bool { + matches!(self, Self::Basic | Self::Cors(_)) + } +} + +/// A consumed preload retains its response filter and reports Resource Timing +/// only for the producer. This is independent of HTTP cache state: a no-store +/// response can also be preloaded. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResponsePreloadState { + #[default] + None, + Consumed { + filter: Option, + from_service_worker: bool, + }, +} + +impl ResponsePreloadState { + pub fn is_consumed(&self) -> bool { + matches!(self, Self::Consumed { .. }) + } + + pub fn from_service_worker(&self) -> bool { + matches!( + self, + Self::Consumed { + from_service_worker: true, + .. + } + ) + } + + pub fn response_filter(&self) -> Option<&FetchResponseFilter> { + match self { + Self::Consumed { filter, .. } => filter.as_ref(), + Self::None => None, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NegotiatedHttpVersion { Http09, @@ -64,6 +117,7 @@ pub struct ResponseHead { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, } @@ -341,6 +395,7 @@ pub struct Response { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, network_request_extra_info: Option, } @@ -362,6 +417,7 @@ impl Clone for Response { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, network_request_extra_info: self.network_request_extra_info.clone(), } @@ -417,6 +473,7 @@ impl Response { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } @@ -456,6 +513,7 @@ impl Response { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, }) @@ -475,6 +533,7 @@ impl Response { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, }) @@ -492,6 +551,7 @@ impl Response { redirect_chain: self.redirect_chain, from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, }; let (body, body_bytes) = self @@ -530,6 +590,7 @@ impl Response { redirect_chain: self.redirect_chain, from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, }; (head, self.body) @@ -557,6 +618,7 @@ pub struct RawResponse { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, network_request_extra_info: Option, } @@ -578,6 +640,7 @@ impl Clone for RawResponse { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, network_request_extra_info: self.network_request_extra_info.clone(), } @@ -627,6 +690,7 @@ impl RawResponse { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } @@ -652,6 +716,7 @@ impl RawResponse { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, }) @@ -671,6 +736,7 @@ impl RawResponse { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, }) @@ -688,6 +754,7 @@ impl RawResponse { redirect_chain: self.redirect_chain, from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, }; (head, self.body) diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs index 6191d3c077..822b175f3c 100644 --- a/moli-fetch/src/runtime.rs +++ b/moli-fetch/src/runtime.rs @@ -2345,6 +2345,7 @@ impl RuntimeOwner { redirect_chain: job.request.redirect_chain, from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version, network_request_extra_info: request_extra_info, })); @@ -2922,6 +2923,7 @@ fn complete_streaming_html_job(job: StreamingRuntimeJob, response: Response) { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info, })); @@ -2977,6 +2979,7 @@ fn complete_cached_streaming_html_job( redirect_chain, from_cache: true, cache_state, + preload_state: Default::default(), negotiated_http_version: None, network_request_extra_info: None, })); @@ -3101,6 +3104,7 @@ fn complete_cached_streaming_raw_job( redirect_chain, from_cache: true, cache_state, + preload_state: Default::default(), negotiated_http_version: None, network_request_extra_info: None, })); @@ -3544,6 +3548,7 @@ fn proxy_connect_response_start( redirect_chain: redirect_chain.to_vec(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, network_request_extra_info: None, } @@ -3648,6 +3653,7 @@ fn collect_buffered_response( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version, }, body, @@ -4309,6 +4315,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "cached".to_owned(), diff --git a/moli-fetch/src/streaming_response.rs b/moli-fetch/src/streaming_response.rs index 0b13184a18..0924a7e767 100644 --- a/moli-fetch/src/streaming_response.rs +++ b/moli-fetch/src/streaming_response.rs @@ -32,6 +32,7 @@ pub struct StreamingHtmlResponse { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, network_request_extra_info: Option, body_chunks: mpsc::UnboundedReceiver, @@ -60,6 +61,7 @@ impl StreamingHtmlResponse { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_chunks, @@ -85,6 +87,7 @@ impl StreamingHtmlResponse { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, body_chunks, @@ -105,6 +108,7 @@ impl StreamingHtmlResponse { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } @@ -179,6 +183,7 @@ pub struct StreamingRawResponse { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: crate::ResponseCacheState, + pub preload_state: crate::ResponsePreloadState, pub negotiated_http_version: Option, network_request_extra_info: Option, body_chunks: mpsc::UnboundedReceiver>, @@ -211,6 +216,7 @@ impl StreamingRawResponse { redirect_chain, from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, network_request_extra_info: None, body_chunks, @@ -237,6 +243,7 @@ impl StreamingRawResponse { redirect_chain: head.redirect_chain, from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, body_chunks, @@ -329,6 +336,7 @@ impl StreamingRawResponse { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } @@ -396,6 +404,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, } } diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs index c17e69a535..bfe9e81f22 100644 --- a/moli-fetch/src/tests/mod.rs +++ b/moli-fetch/src/tests/mod.rs @@ -75,6 +75,7 @@ fn sample_response_head() -> ResponseHead { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, } } diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rs index 3a5b37e94f..84c057d5d4 100644 --- a/moli-page-types/src/lib.rs +++ b/moli-page-types/src/lib.rs @@ -327,6 +327,7 @@ pub struct NavigationResponse { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: moli_fetch::ResponseCacheState, + pub preload_state: moli_fetch::ResponsePreloadState, pub negotiated_http_version: Option, network_request_headers: Option>, } @@ -348,6 +349,7 @@ impl Clone for NavigationResponse { redirect_chain: self.redirect_chain.clone(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, network_request_headers: self.network_request_headers.clone(), } @@ -410,6 +412,7 @@ impl NavigationResponse { .collect(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } @@ -452,6 +455,7 @@ impl NavigationResponse { redirect_chain: head.redirect_chain.into_iter().map(Into::into).collect(), from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_headers: None, } @@ -479,6 +483,7 @@ impl NavigationResponse { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body, @@ -512,6 +517,7 @@ impl NavigationResponse { redirect_chain: self.redirect_chain.into_iter().map(Into::into).collect(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, }; let (body, body_bytes) = self @@ -550,6 +556,7 @@ impl NavigationResponse { redirect_chain: self.redirect_chain.into_iter().map(Into::into).collect(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, }; (head, self.body) @@ -2439,6 +2446,7 @@ pub struct PendingSubresourceResponseInfo { pub response_body: SubresourceResponseBody, pub from_cache: bool, pub cache_state: moli_fetch::ResponseCacheState, + pub preload_state: moli_fetch::ResponsePreloadState, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -2459,6 +2467,7 @@ pub struct PendingSubresourceAuthInfo { pub response_body: SubresourceResponseBody, pub response_from_cache: bool, pub response_cache_state: moli_fetch::ResponseCacheState, + pub response_preload_state: moli_fetch::ResponsePreloadState, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -3321,6 +3330,8 @@ pub struct ChildFrameDocumentNetworkSnapshot { pub from_cache: bool, #[serde(default)] pub cache_state: moli_fetch::ResponseCacheState, + #[serde(skip)] + pub preload_state: moli_fetch::ResponsePreloadState, } /// A completed child main-resource request whose Network facts remain @@ -4045,6 +4056,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "hello".to_owned(), @@ -4073,6 +4085,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "é".to_owned(), @@ -4112,6 +4125,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, String::new(), diff --git a/moli-protocol/src/conn/fetch_support.rs b/moli-protocol/src/conn/fetch_support.rs index b2fbe5d708..16c7c14c31 100644 --- a/moli-protocol/src/conn/fetch_support.rs +++ b/moli-protocol/src/conn/fetch_support.rs @@ -274,6 +274,7 @@ impl PendingFetchAuthNavigation { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, b"auth required".to_vec(), diff --git a/moli-protocol/src/conn/page_state/fetch_state.rs b/moli-protocol/src/conn/page_state/fetch_state.rs index 9d64029dde..c9e8ec66c5 100644 --- a/moli-protocol/src/conn/page_state/fetch_state.rs +++ b/moli-protocol/src/conn/page_state/fetch_state.rs @@ -409,6 +409,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body.to_vec(), diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index 6ff2bfce30..dcaf19e3a9 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -160,6 +160,7 @@ async fn prepare_browser_owned_error_page_navigation_with_engine_async( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; prepare_captured_document_response_with_engine_async( @@ -1217,6 +1218,7 @@ fn decoded_data_url_navigation_response( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, decoded.body, @@ -1314,6 +1316,7 @@ async fn load_inline_html_navigation_with_engine_async( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; prepare_navigation_from_captured_raw_response_with_engine_async( @@ -3007,6 +3010,7 @@ impl CdpConnection { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; self.build_navigation_from_captured_raw_response_with_load_inputs_async( diff --git a/moli-protocol/src/domains/fetch/navigation.rs b/moli-protocol/src/domains/fetch/navigation.rs index 4fbc78ced9..03da3f0d3f 100644 --- a/moli-protocol/src/domains/fetch/navigation.rs +++ b/moli-protocol/src/domains/fetch/navigation.rs @@ -630,6 +630,7 @@ fn pause_data_url_response_stage_navigation_into_buffer( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body, diff --git a/moli-protocol/src/domains/network/activity.rs b/moli-protocol/src/domains/network/activity.rs index 1e4c1e61fe..b0f6bf4a96 100644 --- a/moli-protocol/src/domains/network/activity.rs +++ b/moli-protocol/src/domains/network/activity.rs @@ -593,6 +593,7 @@ mod tests { response_body: SubresourceResponseBody::from_bytes(b"prepared".to_vec()), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }); let action = crate::domains::activity::PreparedSubresourceContinueAction::capture_for_test( &mut conn, @@ -671,6 +672,7 @@ mod tests { response_body: SubresourceResponseBody::from_bytes(b"old".to_vec()), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }); let old_action = crate::domains::activity::PreparedSubresourceContinueAction::capture_for_test( diff --git a/moli-protocol/src/domains/network/tests/cache.rs b/moli-protocol/src/domains/network/tests/cache.rs index c264dbbb3e..cd3f31cf6c 100644 --- a/moli-protocol/src/domains/network/tests/cache.rs +++ b/moli-protocol/src/domains/network/tests/cache.rs @@ -690,6 +690,7 @@ async fn clear_browser_cache_keeps_pending_response_navigation_transfer() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, b"body".to_vec(), diff --git a/moli-protocol/src/domains/page/producer_tests.rs b/moli-protocol/src/domains/page/producer_tests.rs index 0cc9115717..88f11f4d9e 100644 --- a/moli-protocol/src/domains/page/producer_tests.rs +++ b/moli-protocol/src/domains/page/producer_tests.rs @@ -1644,6 +1644,7 @@ async fn child_frame_activity_emits_document_network_events_from_prepared_load() response_body: Some(SubresourceResponseBody::from_bytes(vec![0x00, 0xff, b'a'])), from_cache: true, cache_state: moli_fetch::ResponseCacheState::Local, + preload_state: Default::default(), }), }], "https://example.test".to_owned(), @@ -1776,6 +1777,7 @@ async fn stale_child_document_response_emits_network_without_navigation_or_lifec )), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }, }], Vec::new(), @@ -1878,6 +1880,7 @@ async fn child_document_network_without_body_records_known_no_data() { response_body: None, from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }; let mut background_events = Vec::new(); diff --git a/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs b/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs index ef0c4cedec..e337a011ec 100644 --- a/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs +++ b/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs @@ -243,6 +243,7 @@ fn record_performance_load_event_for_window( } pub(crate) struct ResourcePerformanceEntry { + preload_state: moli_fetch::ResponsePreloadState, name: String, initiator_type: String, start_unix_millis: Option, @@ -299,6 +300,7 @@ impl ResourcePerformanceEntry { &response.headers, body_size as f64, response.cache_state, + response.preload_state.clone(), ) } @@ -317,6 +319,7 @@ impl ResourcePerformanceEntry { &response.headers, body_size as f64, response.cache_state, + response.preload_state.clone(), ) } @@ -334,6 +337,7 @@ impl ResourcePerformanceEntry { &response.headers, response.body_bytes().len() as f64, response.cache_state, + response.preload_state.clone(), ) } @@ -345,6 +349,7 @@ impl ResourcePerformanceEntry { response_headers: &[(String, Vec)], body_size: f64, cache_state: moli_fetch::ResponseCacheState, + preload_state: moli_fetch::ResponsePreloadState, ) -> Self { // Resource Timing deliberately uses a fixed overhead rather than // exposing response-header sizes, which could reveal cookies. @@ -357,6 +362,7 @@ impl ResourcePerformanceEntry { .and_then(|value| moli_web_mime::mime_essence(&value)) .unwrap_or_default(); Self { + preload_state, name: name.into(), initiator_type: initiator_type.to_owned(), start_unix_millis, @@ -385,6 +391,7 @@ impl ResourcePerformanceEntry { &network.response_headers, network.encoded_data_length as f64, network.cache_state, + network.preload_state.clone(), ) } @@ -402,6 +409,7 @@ impl ResourcePerformanceEntry { start_unix_millis: Option, ) -> Self { Self { + preload_state: Default::default(), name: name.into(), initiator_type: initiator_type.to_owned(), start_unix_millis, @@ -419,6 +427,9 @@ pub(crate) fn record_resource_performance_entry( scope: &mut v8::PinScope<'_, '_>, entry: ResourcePerformanceEntry, ) { + if entry.preload_state.is_consumed() { + return; + } if !url::Url::parse(&entry.name).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) { return; } diff --git a/moli-renderer-v8/src/context_bootstrap/performance_runtime/window_state/pending.rs b/moli-renderer-v8/src/context_bootstrap/performance_runtime/window_state/pending.rs index b149d648ef..ea19dab855 100644 --- a/moli-renderer-v8/src/context_bootstrap/performance_runtime/window_state/pending.rs +++ b/moli-renderer-v8/src/context_bootstrap/performance_runtime/window_state/pending.rs @@ -117,6 +117,7 @@ pub(in crate::context_bootstrap::performance_runtime) fn take_pending_resource_e .get_index(scope, index) .and_then(|value| v8::Local::::try_from(value).ok())?; Some(ResourcePerformanceEntry { + preload_state: Default::default(), name: array_string(scope, record, 0)?, initiator_type: array_string(scope, record, 1)?, start_unix_millis: record diff --git a/moli-renderer-v8/src/document_script_scheduler.rs b/moli-renderer-v8/src/document_script_scheduler.rs index 0550c57260..6d7183a7cd 100644 --- a/moli-renderer-v8/src/document_script_scheduler.rs +++ b/moli-renderer-v8/src/document_script_scheduler.rs @@ -1109,6 +1109,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: true, cache_state: moli_fetch::ResponseCacheState::Local, + preload_state: Default::default(), negotiated_http_version: None, }, "export default 1;".to_owned(), diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index e709110ea2..849351406a 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -889,6 +889,7 @@ fn child_document_load_outcome_from_response( response_body: Some(response_body), from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), }), }, ))) @@ -958,6 +959,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes(body_bytes.clone()), diff --git a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs index ad665b2930..397b976e43 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs @@ -3086,6 +3086,7 @@ impl JsContextHost { redirect_chain: Vec::new(), from_cache: info.from_cache, cache_state: info.cache_state, + preload_state: info.preload_state.clone(), negotiated_http_version: None, }, ) @@ -3132,6 +3133,7 @@ impl JsContextHost { redirect_chain: Vec::new(), from_cache: info.response_from_cache, cache_state: info.response_cache_state, + preload_state: info.response_preload_state.clone(), negotiated_http_version: None, }, ) diff --git a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs index adc5fd61a4..0252b96f5e 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs @@ -1773,6 +1773,44 @@ impl JsContextHost { &self, dispatch: ServiceWorkerFetchDispatch, ) -> bool { + use crate::service_worker_runtime::ServiceWorkerRequestDestination as Destination; + use moli_fetch::BrowserRequestMetadata as Metadata; + let metadata = match dispatch.request.destination { + Destination::Script => Some(Metadata::Script), + Destination::Style => Some(Metadata::Style), + Destination::Image => Some(Metadata::Image), + Destination::Font => Some(Metadata::Font), + Destination::Track => Some(Metadata::TextTrack), + Destination::Empty => Some(Metadata::Fetch), + _ => None, + }; + if let Some(metadata) = metadata { + let request = moli_fetch::Request::new_browser( + &dispatch.request.method, + dispatch.request.url.clone(), + dispatch.request.body.clone(), + moli_fetch::RequestHeaders::from_byte_strings(&dispatch.request.headers) + .expect("service worker request headers are ByteStrings"), + dispatch.network_context.request_origin.clone(), + ) + .with_initiator_url(&dispatch.network_context.document_url) + .with_request_mode(dispatch.request.request_mode) + .with_credentials_mode(dispatch.request.credentials_mode) + .with_browser_request_metadata(metadata) + .with_subresource_request_metadata( + moli_fetch::SubresourceRequestMetadata { + integrity: Some(dispatch.request.metadata.integrity.clone()), + ..Default::default() + }, + ); + if let Some(preload) = dispatch.request_client.consume_document_preload(&request) { + dispatch.resource_task_runner.spawn(async move { + let completion = preload.into_completion(dispatch.internal_id, request).await; + let _ = dispatch.completion_tx.send_async_subresource(completion); + }); + return true; + } + } self.browser_context_runtime .dispatch_service_worker_fetch(dispatch) } diff --git a/moli-renderer-v8/src/native_bridge/context_host/websockets/registration.rs b/moli-renderer-v8/src/native_bridge/context_host/websockets/registration.rs index 92245f3f1a..14330be85f 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/websockets/registration.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/websockets/registration.rs @@ -307,6 +307,7 @@ impl JsContextHost { response_body: SubresourceResponseBody::from_bytes(Vec::new()), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }), ); true diff --git a/moli-renderer-v8/src/network/backend/memory_cache.rs b/moli-renderer-v8/src/network/backend/memory_cache.rs index d607e96fe8..d4c976b3cb 100644 --- a/moli-renderer-v8/src/network/backend/memory_cache.rs +++ b/moli-renderer-v8/src/network/backend/memory_cache.rs @@ -837,6 +837,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body.to_owned(), @@ -856,6 +857,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body.to_vec(), @@ -1061,6 +1063,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "x".repeat(64), diff --git a/moli-renderer-v8/src/network/context/document.rs b/moli-renderer-v8/src/network/context/document.rs index e25e304c07..d69185361d 100644 --- a/moli-renderer-v8/src/network/context/document.rs +++ b/moli-renderer-v8/src/network/context/document.rs @@ -1,6 +1,6 @@ use std::sync::{ Arc, - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }; use moli_cookie_jar::BrowserCookieFacadeContext; @@ -51,6 +51,8 @@ struct DocumentResourceLoaderAuthority { id: u64, lifecycle: Mutex, loads: ResourceLoadRegistry, + preloads: crate::network::preloads::DocumentPreloads, + owns_preloads: AtomicBool, } struct DocumentResourceLoaderLifecycle { @@ -64,6 +66,9 @@ impl Drop for DocumentResourceLoaderAuthority { // boundary. This final guard covers construction failures and runtime // teardown paths that drop the authority before publishing it. self.loads.begin_detach(); + if self.owns_preloads.load(Ordering::Acquire) { + self.preloads.retire(); + } } } @@ -143,9 +148,12 @@ impl DocumentResourceLoader { let loads = ResourceLoadRegistry::new(task_runner); // The Fetch client origin is independent of a script's referrer/base // URL, including cross-origin dependencies and inherited/sandboxed Documents. - let request_client = request_client.with_script_request_origin( - moli_url::WebOrigin::from_ascii_serialization(context.origin()), - ); + let preloads = crate::network::preloads::DocumentPreloads::default(); + let request_client = request_client + .with_script_request_origin(moli_url::WebOrigin::from_ascii_serialization( + context.origin(), + )) + .with_document_preloads(preloads.clone()); Self { request_client, csp_reports: Default::default(), @@ -158,6 +166,8 @@ impl DocumentResourceLoader { context, }), loads, + preloads, + owns_preloads: AtomicBool::new(true), }), } } @@ -204,6 +214,18 @@ impl DocumentResourceLoader { // document.open() rotates Moli's load owner, but keeps the Document's // CSP. In-flight requests and new requests must share its report history. replacement.csp_reports = previous.csp_reports.clone(); + // document.open() replaces the load owner, not the Document. Transfer + // retirement authority along with its one-shot preload map. + previous + .authority + .owns_preloads + .store(false, Ordering::Release); + Arc::get_mut(&mut replacement.authority) + .expect("new Document resource authority is unshared") + .preloads = previous.authority.preloads.clone(); + replacement.request_client = replacement + .request_client + .with_document_preloads(previous.authority.preloads.clone()); replacement } @@ -238,11 +260,11 @@ impl DocumentResourceLoader { if let Some(browser_site_context) = self.request_client.shared_browser_site_context() { request_client = request_client.with_shared_browser_site_context(browser_site_context); } - request_client = request_client.with_script_request_origin( - moli_url::WebOrigin::from_ascii_serialization( + request_client = request_client + .with_script_request_origin(moli_url::WebOrigin::from_ascii_serialization( self.authority.lifecycle.lock().context.origin(), - ), - ); + )) + .with_document_preloads(self.authority.preloads.clone()); Self { request_client, authority: Arc::clone(&self.authority), @@ -258,6 +280,9 @@ impl DocumentResourceLoader { lifecycle.state = DocumentResourceLoaderState::Detaching; drop(lifecycle); self.authority.loads.begin_detach(); + if self.authority.owns_preloads.load(Ordering::Acquire) { + self.authority.preloads.retire(); + } true } diff --git a/moli-renderer-v8/src/network/context/tests.rs b/moli-renderer-v8/src/network/context/tests.rs index d17741afc8..dba7617861 100644 --- a/moli-renderer-v8/src/network/context/tests.rs +++ b/moli-renderer-v8/src/network/context/tests.rs @@ -528,3 +528,53 @@ async fn dropping_unpublished_worker_authority_cancels_ordinary_loads() { assert!(cancel.is_cancelled()); assert!(load.is_cancelled()); } + +#[tokio::test] +async fn document_preloads_survive_transport_replacement_and_open_but_not_new_documents() { + let transport = ResourceRequestClient::new(&FetchConfig::default()).unwrap(); + let first = document_loader(transport.clone(), 1, "https://example.test/page"); + let request = moli_fetch::Request::new("GET", "https://example.test/asset", None, vec![]) + .unwrap() + .with_page_network_policy() + .with_request_mode(moli_fetch::RequestMode::NoCors) + .with_browser_request_metadata(moli_fetch::BrowserRequestMetadata::Script); + let producer = first + .request_client() + .register_document_preload(&request) + .unwrap(); + // A preload lookup is independent of whether the network transport can + // currently make a request, both on a miss and when consuming a response. + transport.set_network_offline(true); + let other = first.fork_for_document(context(2, "https://example.test/page")); + assert!( + other + .request_client() + .consume_document_preload(&request) + .is_none() + ); + let replaced = first.with_replacement_transport(transport.handle()); + let opened = DocumentResourceLoader::for_document_open( + context(3, "https://example.test/page"), + super::document::DocumentResourceAuthoritySource::Inherited(replaced.clone()), + &replaced, + ); + first.begin_detach(); + let captured = opened.request_client().frozen_request_client(); + let consumer = captured.consume_document_preload(&request).unwrap(); + producer.complete(Err("retained preload failure".to_owned())); + assert_eq!( + consumer.response().await.unwrap_err(), + "retained preload failure" + ); + let producer = opened + .request_client() + .register_document_preload(&request) + .unwrap(); + let consumer = captured.consume_document_preload(&request).unwrap(); + opened.begin_detach(); + assert_eq!( + consumer.response().await.unwrap_err(), + "preload Document retired" + ); + drop(producer); +} diff --git a/moli-renderer-v8/src/network/devtools_resource_load.rs b/moli-renderer-v8/src/network/devtools_resource_load.rs index 27eeeecf86..ac06a67e14 100644 --- a/moli-renderer-v8/src/network/devtools_resource_load.rs +++ b/moli-renderer-v8/src/network/devtools_resource_load.rs @@ -69,6 +69,7 @@ impl RendererPreparedNetworkResourceLoad { redirect_chain: head.redirect_chain.into_iter().map(Into::into).collect(), from_cache: head.from_cache, cache_state: head.cache_state, + preload_state: head.preload_state.clone(), negotiated_http_version: head.negotiated_http_version, network_request_headers, }, @@ -92,6 +93,7 @@ pub struct RendererNetworkResourceLoadResponse { pub redirect_chain: Vec, pub from_cache: bool, pub cache_state: moli_fetch::ResponseCacheState, + pub preload_state: moli_fetch::ResponsePreloadState, pub negotiated_http_version: Option, pub network_request_headers: Option>, } @@ -114,6 +116,7 @@ impl RendererNetworkResourceLoadResponse { .collect(), from_cache: self.from_cache, cache_state: self.cache_state, + preload_state: self.preload_state.clone(), negotiated_http_version: self.negotiated_http_version, } } diff --git a/moli-renderer-v8/src/network/mod.rs b/moli-renderer-v8/src/network/mod.rs index bcba300d2f..5efaa536ba 100644 --- a/moli-renderer-v8/src/network/mod.rs +++ b/moli-renderer-v8/src/network/mod.rs @@ -14,6 +14,7 @@ mod devtools_resource_load; pub(crate) mod loads; pub mod navigation; mod policy; +pub(crate) mod preloads; mod request_client; mod task_runner; @@ -29,6 +30,7 @@ pub use devtools_resource_load::{ RendererNetworkResourceLoadResponse, RendererPreparedNetworkResourceLoad, }; pub use policy::{PageNetworkPolicy, PageNetworkPolicySnapshot}; +pub(crate) use request_client::streaming_raw_response_from_local_response; pub use request_client::{ResourceRequestClient, ResourceRequestClientOwner}; pub use task_runner::RendererResourceTaskRunner; diff --git a/moli-renderer-v8/src/network/preloads.rs b/moli-renderer-v8/src/network/preloads.rs new file mode 100644 index 0000000000..e2599a6e2d --- /dev/null +++ b/moli-renderer-v8/src/network/preloads.rs @@ -0,0 +1,377 @@ +use std::{ + collections::HashMap, + sync::{Arc, Weak}, +}; + +use moli_fetch::{BrowserRequestMetadata, Request, RequestCredentialsMode, RequestMode}; +use parking_lot::Mutex; +use tokio::sync::Notify; +use url::Url; + +use crate::{ + protocol_types::NavigationResponse, + subresource_integrity::integrity_metadata_allows_preload_consumption, + types::AsyncSubresourceFetchResponseFilter, +}; + +#[derive(Clone, Debug)] +pub(crate) struct DocumentPreloadResponse { + pub(crate) response: NavigationResponse, + pub(crate) from_service_worker: bool, + pub(crate) response_filter: Option, +} + +#[derive(Debug)] +pub(crate) struct ConsumedPreloadError(pub(crate) String); + +impl std::fmt::Display for ConsumedPreloadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ConsumedPreloadError {} + +type PreloadResult = Result; + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct PreloadKey { + url: Url, + destination: &'static str, + mode: RequestMode, + credentials: RequestCredentialsMode, +} + +impl PreloadKey { + fn from_request(request: &Request) -> Option { + if request.method != "GET" + || request.body.is_some() + || !matches!( + request.request_mode, + RequestMode::NoCors | RequestMode::Cors | RequestMode::SameOrigin + ) + || !matches!(request.url.scheme(), "http" | "https") + { + return None; + } + let destination = match request.browser_request_metadata()? { + BrowserRequestMetadata::Script => "script", + BrowserRequestMetadata::Style | BrowserRequestMetadata::StyleModule => "style", + BrowserRequestMetadata::Image => "image", + BrowserRequestMetadata::Font => "font", + BrowserRequestMetadata::TextTrack => "track", + BrowserRequestMetadata::Fetch | BrowserRequestMetadata::Xhr => "", + _ => return None, + }; + Some(Self { + url: request.url.clone(), + destination, + mode: request.request_mode, + credentials: request.credentials_mode, + }) + } +} + +struct PreloadEntry { + integrity: String, + result: Mutex>, + available: Notify, +} + +impl PreloadEntry { + fn complete(&self, result: PreloadResult) { + let mut current = self.result.lock(); + if current.is_some() { + return; + } + *current = Some(result); + drop(current); + self.available.notify_waiters(); + } +} + +#[derive(Default)] +struct PreloadMap { + retired: bool, + entries: HashMap>, + // Includes consumed entries, so retiring the Document also wakes consumers + // waiting for a response that has not yet reached the loader. + pending: Vec>, +} + +#[derive(Clone, Default)] +pub(crate) struct DocumentPreloads(Arc>); + +impl DocumentPreloads { + pub(crate) fn register(&self, request: &Request) -> Option { + let key = PreloadKey::from_request(request)?; + let mut map = self.0.lock(); + if map.retired { + return None; + } + let entry = Arc::new(PreloadEntry { + integrity: request + .subresource_request_metadata() + .and_then(|metadata| metadata.integrity.as_deref()) + .unwrap_or_default() + .to_owned(), + result: Mutex::new(None), + available: Notify::new(), + }); + map.pending.retain(|entry| entry.strong_count() != 0); + map.pending.push(Arc::downgrade(&entry)); + map.entries.insert(key, entry.clone()); + Some(DocumentPreloadProducer { entry }) + } + + pub(crate) fn consume(&self, request: &Request) -> Option { + if request.priority_hints.link_preload { + return None; + } + // Fetch/XHR author headers make the request ineligible for preload + // consumption. DOM-initiated requests do not set the unsafe-request flag. + if matches!( + request.browser_request_metadata(), + Some(BrowserRequestMetadata::Fetch | BrowserRequestMetadata::Xhr) + ) && !request.request_headers.is_empty() + { + return None; + } + let key = PreloadKey::from_request(request)?; + let mut map = self.0.lock(); + if map.retired { + return None; + } + let entry = map.entries.get(&key)?; + if !integrity_metadata_allows_preload_consumption( + &entry.integrity, + request + .subresource_request_metadata() + .and_then(|metadata| metadata.integrity.as_deref()), + ) { + return None; + } + Some(DocumentPreloadConsumer { + entry: map.entries.remove(&key)?, + }) + } + + pub(crate) fn retire(&self) { + let pending = { + let mut map = self.0.lock(); + map.retired = true; + map.entries.clear(); + std::mem::take(&mut map.pending) + }; + for entry in pending.into_iter().filter_map(|entry| entry.upgrade()) { + entry.complete(Err("preload Document retired".to_owned())); + } + } +} + +pub(crate) struct DocumentPreloadProducer { + entry: Arc, +} + +impl DocumentPreloadProducer { + pub(crate) fn complete(self, result: PreloadResult) { + self.entry.complete(result); + } +} + +impl Drop for DocumentPreloadProducer { + fn drop(&mut self) { + self.entry + .complete(Err("preload response was cancelled".to_owned())); + } +} + +pub(crate) struct DocumentPreloadConsumer { + entry: Arc, +} + +impl DocumentPreloadConsumer { + pub(crate) async fn into_completion( + self, + internal_id: u64, + request: Request, + ) -> crate::types::AsyncSubresourceFetchCompletion { + use crate::types::{AsyncSubresourceFetchCompletion, AsyncSubresourceFetchResult}; + let (result, response_filter, from_service_worker) = match self.response().await { + Ok(preload) => ( + AsyncSubresourceFetchResult::Response(preload.response), + preload.response_filter, + preload.from_service_worker, + ), + Err(error) => ( + AsyncSubresourceFetchResult::PreloadFailure(error), + None, + false, + ), + }; + AsyncSubresourceFetchCompletion { + internal_id, + request_url: request.url, + request_method: request.method, + request_headers: request.request_headers, + request_body: request + .body + .as_ref() + .map(|body| String::from_utf8_lossy(body).into_owned()), + response_status_text: None, + skip_fetch_security_validation: from_service_worker, + response_filter, + network_error_text: None, + result, + } + } + + pub(crate) async fn response(self) -> PreloadResult { + loop { + let notified = self.entry.available.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if let Some(result) = self.entry.result.lock().clone() { + return result.map(|mut preload| { + preload.response.preload_state = moli_fetch::ResponsePreloadState::Consumed { + filter: preload.response_filter.clone(), + from_service_worker: preload.from_service_worker, + }; + preload + }); + } + notified.await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use moli_fetch::{ResponseCacheState, SubresourceRequestMetadata}; + + fn request() -> Request { + Request::new("GET", "https://example.test/asset", None, vec![]) + .unwrap() + .with_browser_request_metadata(BrowserRequestMetadata::Script) + .with_request_mode(RequestMode::NoCors) + .with_credentials_mode(RequestCredentialsMode::Include) + } + + fn response() -> DocumentPreloadResponse { + DocumentPreloadResponse { + response: NavigationResponse::from_text_body( + request().url, + 200, + vec![], + "body".to_owned(), + ), + response_filter: Some(AsyncSubresourceFetchResponseFilter::Opaque), + from_service_worker: true, + } + } + + #[tokio::test] + async fn pending_preloads_are_consumed_once_and_retain_response_provenance() { + let preloads = DocumentPreloads::default(); + let request = request(); + let producer = preloads.register(&request).unwrap(); + let consumer = preloads.consume(&request).unwrap(); + assert!(preloads.consume(&request).is_none()); + let pending = tokio::spawn(consumer.response()); + tokio::task::yield_now().await; + assert!(!pending.is_finished()); + producer.complete(Ok(response())); + let actual = pending.await.unwrap().unwrap(); + assert_eq!(actual.response.body_bytes(), b"body"); + assert_eq!(actual.response.cache_state, ResponseCacheState::None); + assert!(actual.response.preload_state.is_consumed()); + assert!(actual.response.preload_state.from_service_worker()); + assert_eq!( + actual.response.preload_state.response_filter(), + Some(&AsyncSubresourceFetchResponseFilter::Opaque) + ); + } + + #[tokio::test] + async fn integrity_failure_is_reused_and_incompatible_metadata_does_not_remove_it() { + let preloads = DocumentPreloads::default(); + let producer_request = + request().with_subresource_request_metadata(SubresourceRequestMetadata { + integrity: Some("sha384-AAAA".to_owned()), + ..Default::default() + }); + let producer = preloads.register(&producer_request).unwrap(); + producer.complete(Err("integrity mismatch".to_owned())); + let other = request().with_subresource_request_metadata(SubresourceRequestMetadata { + integrity: Some("sha384-BBBB".to_owned()), + ..Default::default() + }); + assert!(preloads.consume(&other).is_none()); + assert_eq!( + preloads + .consume(&request()) + .unwrap() + .response() + .await + .unwrap_err(), + "integrity mismatch" + ); + assert!(preloads.consume(&request()).is_none()); + } + + #[tokio::test] + async fn incompatible_fetch_keys_and_author_headers_leave_preload_available() { + let preloads = DocumentPreloads::default(); + let producer = preloads.register(&request()).unwrap(); + let mut post = request(); + post.method = "POST".to_owned(); + for other in [ + post, + request().with_request_mode(RequestMode::Navigate), + request().with_request_mode(RequestMode::Cors), + request().with_credentials_mode(RequestCredentialsMode::Omit), + request().with_browser_request_metadata(BrowserRequestMetadata::Image), + ] { + assert!(preloads.consume(&other).is_none()); + } + let mut producer_request = request(); + producer_request.priority_hints.link_preload = true; + assert!(preloads.consume(&producer_request).is_none()); + drop(producer); + assert!( + preloads + .consume(&request()) + .unwrap() + .response() + .await + .is_err() + ); + + let fetch = request().with_browser_request_metadata(BrowserRequestMetadata::Fetch); + let _producer = preloads.register(&fetch).unwrap(); + let mut with_header = fetch.clone(); + with_header.request_headers = moli_fetch::RequestHeaders::from_byte_strings(&[( + "X-Test".to_owned(), + "yes".to_owned(), + )]) + .unwrap(); + assert!(preloads.consume(&with_header).is_none()); + assert!(preloads.consume(&fetch).is_some()); + } + + #[tokio::test] + async fn retirement_wakes_reserved_consumers_and_blocks_late_publication() { + let preloads = DocumentPreloads::default(); + let producer = preloads.register(&request()).unwrap(); + let pending = tokio::spawn(preloads.consume(&request()).unwrap().response()); + preloads.retire(); + producer.complete(Ok(response())); + assert_eq!( + pending.await.unwrap().unwrap_err(), + "preload Document retired" + ); + assert!(preloads.register(&request()).is_none()); + assert!(preloads.consume(&request()).is_none()); + } +} diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rs index 9ca0732e73..1a15814b12 100644 --- a/moli-renderer-v8/src/network/request_client.rs +++ b/moli-renderer-v8/src/network/request_client.rs @@ -36,6 +36,7 @@ pub struct ResourceRequestClient { page_network_policy: PageNetworkPolicy, browser_site_context: Option>, script_request_origin: Option>, + document_preloads: Option, } /// Thread-affine lifetime root for a standalone resource request client. @@ -121,6 +122,7 @@ impl ResourceRequestClient { page_network_policy, browser_site_context: None, script_request_origin: None, + document_preloads: None, } } @@ -129,6 +131,31 @@ impl ResourceRequestClient { self } + pub(crate) fn with_document_preloads( + mut self, + preloads: super::preloads::DocumentPreloads, + ) -> Self { + self.document_preloads = Some(preloads); + self + } + + // Looking up a Document response must not apply the transport's offline + // policy: a miss can still be handled by a Service Worker. The request + // entry points retain their normal policy and response validation. + pub(crate) fn consume_document_preload( + &self, + request: &Request, + ) -> Option { + self.document_preloads.as_ref()?.consume(request) + } + + pub(crate) fn register_document_preload( + &self, + request: &Request, + ) -> Option { + self.document_preloads.as_ref()?.register(request) + } + fn script_request_with_client_origin(&self, request: Request) -> Request { if request.request_origin().is_none() && let Some(origin) = &self.script_request_origin @@ -174,6 +201,7 @@ impl ResourceRequestClient { ); client.browser_site_context = self.browser_site_context.clone(); client.script_request_origin = self.script_request_origin.clone(); + client.document_preloads = self.document_preloads.clone(); client } @@ -318,6 +346,13 @@ impl ResourceRequestClient { request: Request, ) -> Result { let request = self.apply_network_policy(self.script_request_with_client_origin(request))?; + if let Some(preload) = self.consume_document_preload(&request) { + return preload + .response() + .await + .map(|preload| preload.response.into()) + .map_err(|error| super::preloads::ConsumedPreloadError(error).into()); + } if let Some(result) = local_text_response(&request) { return result; } @@ -485,6 +520,21 @@ impl ResourceRequestClient { F: FnOnce(Result) + Send + 'static, { let request = self.apply_network_policy(self.script_request_with_client_origin(request))?; + if let Some(preload) = self.consume_document_preload(&request) { + let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel(); + resource_load.attach_consumer_cancel(move || { + let _ = cancel_tx.send(()); + }); + resource_load.task_runner().spawn(async move { + let result = tokio::select! { + result = preload.response() => result.map(|preload| preload.response.into()).map_err(|error| super::preloads::ConsumedPreloadError(error).into()), + _ = cancel_rx => Err(super::preloads::ConsumedPreloadError("preload consumer cancelled".to_owned()).into()), + }; + resource_load.finish(); + callback(result); + }); + return Ok(()); + } if let Some(result) = local_text_response(&request) { let task_runner = resource_load.task_runner(); task_runner.spawn(async move { @@ -1225,7 +1275,9 @@ fn local_text_response(request: &Request) -> Option> { .map(|result| result.map_err(anyhow::Error::msg)) } -fn streaming_raw_response_from_local_response(response: Response) -> Result { +pub(crate) fn streaming_raw_response_from_local_response( + response: Response, +) -> Result { let raw_response = response.into_materialized_raw_response(); streaming_raw_response_from_head_and_body(raw_response.head(), raw_response.clone_body_bytes()) } diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rs index 3df1d2017b..8ae4a492e0 100644 --- a/moli-renderer-v8/src/network/request_client/tests.rs +++ b/moli-renderer-v8/src/network/request_client/tests.rs @@ -289,6 +289,7 @@ async fn memory_cache_tee_drop_after_body_eof_cancels_pending_completion_and_rel redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_rx, @@ -1610,6 +1611,7 @@ async fn cached_raw_subresource_marks_redirect_hops_from_cache() -> Result<()> { }], from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, b"cached-final-body".to_vec(), diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rs index efeb63e464..5adcd34884 100644 --- a/moli-renderer-v8/src/network_host/async_fetch.rs +++ b/moli-renderer-v8/src/network_host/async_fetch.rs @@ -99,6 +99,10 @@ async fn fetch_browser_subresource_with_preflight_headers_and_observer( preflight_request_headers: Vec<(String, String)>, preflight_observer: Option<&CorsPreflightNetworkObserver>, ) -> Result, String> { + if let Some(preload) = loader.consume_document_preload(&request) { + let response = preload.response().await?.response.into(); + return Ok(NetworkFetchResult::without_request_observation(response)); + } if browser_request_needs_cors_redirect_checks(&request) { return fetch_browser_subresource_with_cors_redirect_checks( loader, @@ -515,6 +519,13 @@ async fn fetch_browser_subresource_raw_stream_with_preflight_headers_and_observe preflight_request_headers: Vec<(String, String)>, preflight_observer: Option<&CorsPreflightNetworkObserver>, ) -> Result, String> { + if let Some(preload) = loader.consume_document_preload(&request) { + let response = crate::network::streaming_raw_response_from_local_response( + preload.response().await?.response.into(), + ) + .map_err(format_network_error)?; + return Ok(NetworkFetchResult::without_request_observation(response)); + } if browser_request_needs_cors_redirect_checks(&request) { return fetch_browser_subresource_raw_stream_with_cors_redirect_checks( loader, @@ -680,6 +691,11 @@ pub(crate) fn spawn_async_subresource_fetch( .then(|| loader.parkable_image_manager(&task_runner)); let request = observe_async_xhr_upload(request, &completion_tx, internal_id); task_runner.spawn(async move { + if let Some(preload) = loader.consume_document_preload(&request) { + let completion = preload.into_completion(internal_id, request).await; + let _ = completion_tx.send_async_subresource(completion); + return; + } let preflight_observer = CorsPreflightNetworkObserver::new(completion_tx.clone(), network_context); let auth_requires_buffered_transport = request.auth_requires_buffered_transport(); @@ -1090,6 +1106,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, true, diff --git a/moli-renderer-v8/src/network_host/async_fetch/redirect_tests.rs b/moli-renderer-v8/src/network_host/async_fetch/redirect_tests.rs index c449789ff8..e5e4dcef5d 100644 --- a/moli-renderer-v8/src/network_host/async_fetch/redirect_tests.rs +++ b/moli-renderer-v8/src/network_host/async_fetch/redirect_tests.rs @@ -188,6 +188,7 @@ fn cors_redirect_referrer_never_recovers_stripped_information() -> Result<()> { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; assert_eq!( @@ -241,6 +242,7 @@ fn cors_redirect_modes_precede_location_parsing() -> Result<()> { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let result = redirects.advance(head, false, None); diff --git a/moli-renderer-v8/src/network_host/browser_response.rs b/moli-renderer-v8/src/network_host/browser_response.rs index 69c3d2db90..53ccb7655f 100644 --- a/moli-renderer-v8/src/network_host/browser_response.rs +++ b/moli-renderer-v8/src/network_host/browser_response.rs @@ -130,6 +130,7 @@ pub(super) fn blob_response( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_bytes.to_vec(), @@ -154,6 +155,7 @@ pub(crate) fn data_url_response(url: &url::Url) -> Option { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_bytes, diff --git a/moli-renderer-v8/src/network_host/image.rs b/moli-renderer-v8/src/network_host/image.rs index 9241b789b2..90bcffd3fc 100644 --- a/moli-renderer-v8/src/network_host/image.rs +++ b/moli-renderer-v8/src/network_host/image.rs @@ -115,8 +115,8 @@ pub(crate) fn start_scanned_image_preload( let decode_runner = task_runner.clone(); let cancel_handle = load.cancel_handle(); task_runner.spawn(async move { - let result = match loader - .fetch_raw_stream_with_cancel_and_network_metadata(request, cancel_handle) + let result = match crate::network_host::fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata( + &loader, request, Some(cancel_handle), Vec::new()) .await { Ok(observed) => { diff --git a/moli-renderer-v8/src/network_host/response/cors.rs b/moli-renderer-v8/src/network_host/response/cors.rs index bb8103ddfe..6847ffc092 100644 --- a/moli-renderer-v8/src/network_host/response/cors.rs +++ b/moli-renderer-v8/src/network_host/response/cors.rs @@ -627,6 +627,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, } } @@ -659,6 +660,7 @@ mod tests { }], from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let document_url = url("https://document.test/page.html"); diff --git a/moli-renderer-v8/src/network_host/response/materialize.rs b/moli-renderer-v8/src/network_host/response/materialize.rs index 4c72632bbd..ed86bd077d 100644 --- a/moli-renderer-v8/src/network_host/response/materialize.rs +++ b/moli-renderer-v8/src/network_host/response/materialize.rs @@ -24,6 +24,9 @@ impl FetchResponseRequest<'_> { head: &moli_fetch::ResponseHead, credentials_mode: moli_fetch::RequestCredentialsMode, ) -> crate::types::AsyncSubresourceFetchResponseFilter { + if let Some(filter) = head.preload_state.response_filter() { + return filter.clone(); + } use crate::types::AsyncSubresourceFetchResponseFilter as Filter; let filter = response_filter(request_origin, head, self); match compute_fetch_response_type(request_origin, head, filter) { @@ -123,6 +126,10 @@ pub(crate) fn network_response_filter( let request_origin = request_origin.into(); use crate::types::AsyncSubresourceFetchResponseFilter; + if let Some(filter) = head.preload_state.response_filter() { + return Some(filter.clone()); + } + if redirect_mode == RequestRedirectMode::Manual && is_redirect_status(head.status) { Some(AsyncSubresourceFetchResponseFilter::OpaqueRedirect) } else if request_mode == RequestMode::NoCors diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs index eec72d86a2..5d27c1cba4 100644 --- a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs +++ b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs @@ -96,6 +96,7 @@ fn xhr_complete_callback( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, pending_body_value, diff --git a/moli-renderer-v8/src/planning.rs b/moli-renderer-v8/src/planning.rs index 446f682e2a..4fb2a6101d 100644 --- a/moli-renderer-v8/src/planning.rs +++ b/moli-renderer-v8/src/planning.rs @@ -352,11 +352,13 @@ pub(crate) async fn load_prepared_script_source_outcome_with_document_character_ ) } Err(error) => { + let consumed_preload = + error.is::(); let error = format!("failed to fetch script `{}`: {error}", script.url); PreparedScriptSourceLoadOutcome { source_result: Err(error.clone()), source_bytes: None, - network_result: Some(Arc::new(Err(error))), + network_result: (!consumed_preload).then(|| Arc::new(Err(error))), muted_errors: false, } } @@ -450,11 +452,12 @@ pub(crate) async fn load_service_worker_aware_external_script_source_outcome( .await } Err(error) => { + let consumed_preload = error.is::(); let message = format!("failed to fetch script `{}`: {error}", script.url); PreparedScriptSourceLoadOutcome { source_result: Err(message.clone()), source_bytes: None, - network_result: Some(Arc::new(Err(message))), + network_result: (!consumed_preload).then(|| Arc::new(Err(message))), muted_errors: false, } } diff --git a/moli-renderer-v8/src/runtime/browser_context_runtime.rs b/moli-renderer-v8/src/runtime/browser_context_runtime.rs index 07205fb4e1..f1263fd858 100644 --- a/moli-renderer-v8/src/runtime/browser_context_runtime.rs +++ b/moli-renderer-v8/src/runtime/browser_context_runtime.rs @@ -313,6 +313,7 @@ impl DetachedParserScriptFetchContinuation { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, ResponseBody::materialized_text(text, response_body), diff --git a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs index dafc46b63e..7d9e554b88 100644 --- a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs +++ b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs @@ -256,6 +256,17 @@ impl RendererBrowserContextRuntime { destination: ServiceWorkerRequestDestination, resource_type: SubresourceResourceType, ) -> Result> { + if let Some(preload) = request_client.consume_document_preload(request) { + let preload = preload + .response() + .await + .map_err(crate::network::preloads::ConsumedPreloadError)?; + return Ok(Some(ServiceWorkerDirectFetchResponse { + response: Box::new(preload.response), + response_filter: preload.response_filter, + from_network_fallback: !preload.from_service_worker, + })); + } if !matches!(request.url.scheme(), "http" | "https") { return Ok(None); } diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rs index f3cf4273da..9aca400add 100644 --- a/moli-renderer-v8/src/runtime/page_surface.rs +++ b/moli-renderer-v8/src/runtime/page_surface.rs @@ -733,6 +733,7 @@ fn dedicated_worker_navigation_response_eq( && left.redirect_chain == right.redirect_chain && left.from_cache == right.from_cache && left.cache_state == right.cache_state + && left.preload_state == right.preload_state && left.negotiated_http_version == right.negotiated_http_version && left.network_request_headers() == right.network_request_headers() } diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs index eb77ea87f2..7ff7948109 100644 --- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs +++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs @@ -492,6 +492,7 @@ fn about_blank_navigation_response(url: &Url) -> Option { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, Vec::new(), diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs b/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs index 92f1b90511..5c5aff07b0 100644 --- a/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs +++ b/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs @@ -246,6 +246,7 @@ fetch({:?}) redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }, @@ -462,6 +463,7 @@ async fn streaming_finish_requires_matching_request_and_body_source_identity() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }, diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/child_document_completion.rs b/moli-renderer-v8/src/runtime/page_vm/tests/child_document_completion.rs index b6a67a561a..5b5c680b59 100644 --- a/moli-renderer-v8/src/runtime/page_vm/tests/child_document_completion.rs +++ b/moli-renderer-v8/src/runtime/page_vm/tests/child_document_completion.rs @@ -739,6 +739,7 @@ pub(super) fn stale_loaded_completion( response_body: None, from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), }), }, ))), diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/extracted/main_module_graphs.rs b/moli-renderer-v8/src/runtime/page_vm/tests/extracted/main_module_graphs.rs index 83f5eeb3e4..5b2c3e34f7 100644 --- a/moli-renderer-v8/src/runtime/page_vm/tests/extracted/main_module_graphs.rs +++ b/moli-renderer-v8/src/runtime/page_vm/tests/extracted/main_module_graphs.rs @@ -634,6 +634,7 @@ fn module_graph_network_result_records_staged_response_started_with_cache_state( redirect_chain: Vec::new(), from_cache: true, cache_state: moli_fetch::ResponseCacheState::Local, + preload_state: Default::default(), negotiated_http_version: None, }, "export default 1;".to_owned(), diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch.rs index ce2d5c8ff1..13de22bec5 100644 --- a/moli-renderer-v8/src/script_vm/subresource_fetch.rs +++ b/moli-renderer-v8/src/script_vm/subresource_fetch.rs @@ -331,6 +331,7 @@ enum ImageSubresourceTerminal<'a> { encoded: &'a moli_parkable_image::ParkableImage, }, Failure, + PreloadFailure, } impl ImageSubresourceTerminal<'_> { @@ -348,7 +349,7 @@ impl ImageSubresourceTerminal<'_> { encoded.len(), ) } - Self::Failure => { + Self::Failure | Self::PreloadFailure => { crate::context_bootstrap::ResourcePerformanceEntry::from_network_failure( request_url.as_str(), "img", @@ -383,7 +384,7 @@ fn apply_image_subresource_terminal( ); (completion.accepted(), completion.followup()) } - ImageSubresourceTerminal::Failure => { + ImageSubresourceTerminal::Failure | ImageSubresourceTerminal::PreloadFailure => { let followup = context_host .borrow_mut() .complete_pending_image_load_network_request_if_matches( @@ -395,7 +396,7 @@ fn apply_image_subresource_terminal( (followup.is_some(), followup) } }; - if accepted { + if accepted && !matches!(terminal, ImageSubresourceTerminal::PreloadFailure) { crate::context_bootstrap::record_resource_performance_entry( scope, terminal.resource_performance_entry(request_url), diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch/fetch_dispatch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch/fetch_dispatch.rs index 0aa1649fb6..8b963410ea 100644 --- a/moli-renderer-v8/src/script_vm/subresource_fetch/fetch_dispatch.rs +++ b/moli-renderer-v8/src/script_vm/subresource_fetch/fetch_dispatch.rs @@ -411,12 +411,21 @@ impl ScriptVm { network_error_text: Option, completion_result: AsyncSubresourceFetchResult, ) -> Result { + let consumed_preload = match &completion_result { + AsyncSubresourceFetchResult::PreloadFailure(_) => true, + AsyncSubresourceFetchResult::Response(response) + | AsyncSubresourceFetchResult::Image { response, .. } => { + response.preload_state.is_consumed() + } + AsyncSubresourceFetchResult::Failure(_) => false, + }; let (supplied_parkable_image, result) = match completion_result { AsyncSubresourceFetchResult::Response(response) => (None, Ok(response)), AsyncSubresourceFetchResult::Image { response, encoded } => { (Some(encoded), Ok(response)) } - AsyncSubresourceFetchResult::Failure(error) => (None, Err(error)), + AsyncSubresourceFetchResult::Failure(error) + | AsyncSubresourceFetchResult::PreloadFailure(error) => (None, Err(error)), }; let trace_started = moli_trace::cdp_runtime_trace_enabled().then(Instant::now); let trace_fields = async_subresource_trace_fields_for_pending( @@ -889,7 +898,7 @@ impl ScriptVm { sequence, pending.info.internal_id, &pending.info.url, - ImageSubresourceTerminal::Failure, + if consumed_preload { ImageSubresourceTerminal::PreloadFailure } else { ImageSubresourceTerminal::Failure }, ), PendingSubresourceContinuation::Media { media_handle, diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch/pending_fetch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch/pending_fetch.rs index 3021209847..2caf954019 100644 --- a/moli-renderer-v8/src/script_vm/subresource_fetch/pending_fetch.rs +++ b/moli-renderer-v8/src/script_vm/subresource_fetch/pending_fetch.rs @@ -860,6 +860,7 @@ impl ScriptVm { response_body: SubresourceResponseBody::from_navigation_response(&response), from_cache: response.from_cache, cache_state: response.cache_state, + preload_state: response.preload_state.clone(), }; self._context_host .borrow_mut() @@ -1117,6 +1118,7 @@ impl ScriptVm { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let pending = match continuation { @@ -1864,6 +1866,7 @@ impl ScriptVm { redirect_chain: Vec::new(), from_cache: pending.response.from_cache, cache_state: pending.response.cache_state, + preload_state: pending.response.preload_state.clone(), negotiated_http_version: pending.response.negotiated_http_version, }); let result = if pending.pending.request_mode == moli_fetch::RequestMode::NoCors { @@ -1929,6 +1932,7 @@ impl ScriptVm { redirect_chain: Vec::new(), from_cache: pending.response.from_cache, cache_state: pending.response.cache_state, + preload_state: pending.response.preload_state.clone(), negotiated_http_version: pending.response.negotiated_http_version, }); let result = if pending.pending.request_mode == moli_fetch::RequestMode::NoCors { @@ -2003,6 +2007,7 @@ impl ScriptVm { redirect_chain: Vec::new(), from_cache: pending.response.from_cache, cache_state: pending.response.cache_state, + preload_state: pending.response.preload_state.clone(), negotiated_http_version: pending.response.negotiated_http_version, }), ), diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch/streaming.rs b/moli-renderer-v8/src/script_vm/subresource_fetch/streaming.rs index d9d923abdc..ed8c0e6be3 100644 --- a/moli-renderer-v8/src/script_vm/subresource_fetch/streaming.rs +++ b/moli-renderer-v8/src/script_vm/subresource_fetch/streaming.rs @@ -139,6 +139,7 @@ impl ScriptVm { response_body: SubresourceResponseBody::from_navigation_response(&response), response_from_cache: response.from_cache, response_cache_state: response.cache_state, + response_preload_state: response.preload_state.clone(), }; trace_async_subresource_stage( "async_subresource_complete_running_auth_required", @@ -193,6 +194,7 @@ impl ScriptVm { response_body: SubresourceResponseBody::from_navigation_response(&response), from_cache: response.from_cache, cache_state: response.cache_state, + preload_state: response.preload_state.clone(), }; self._context_host .borrow_mut() diff --git a/moli-renderer-v8/src/script_vm/tests/browser_api/navigation/performance.rs b/moli-renderer-v8/src/script_vm/tests/browser_api/navigation/performance.rs index 01167cab38..9a318d4c2c 100644 --- a/moli-renderer-v8/src/script_vm/tests/browser_api/navigation/performance.rs +++ b/moli-renderer-v8/src/script_vm/tests/browser_api/navigation/performance.rs @@ -589,6 +589,7 @@ fn parser_script_network_results_populate_buffered_resource_timing_snapshots() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "void 0;".to_owned(), diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/null_body.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/null_body.rs index df4a9c9fae..b46d9237f0 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/null_body.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/null_body.rs @@ -180,6 +180,7 @@ fn xhr_null_body_statuses_discard_buffered_and_streamed_bytes() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; if streaming { @@ -346,6 +347,7 @@ fn xhr_null_body_late_completion_preserves_a_replacement_requests_progress() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/opaque_response.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/opaque_response.rs index ad8e4e4438..043f200d1c 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/opaque_response.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/opaque_response.rs @@ -65,6 +65,7 @@ fn window_service_worker_response_filter_survives_streaming_clone_and_cache() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let response_filter = Some(if cors { @@ -235,6 +236,7 @@ fn window_filtered_fetch_preserves_internal_head_through_clone_and_cache() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; if streaming { diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/redirect_filter.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/redirect_filter.rs index dea6bb63fb..7a1bac5e33 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/redirect_filter.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/redirect_filter.rs @@ -58,6 +58,7 @@ fn redirect_filter_completion_keeps_status_text_and_explicit_filters() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "body".to_owned(), diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/response_headers.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/response_headers.rs index a0467f6b4f..bb34a360b2 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/response_headers.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/response_headers.rs @@ -36,6 +36,7 @@ fn xhr_response_headers_are_filtered_combined_and_sorted_before_events() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; if streaming { diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs index 349ccb5150..bbc694d574 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs @@ -1201,6 +1201,7 @@ fn xml_http_request_blob_response_uses_final_mime_type() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes(b"body".to_vec()), @@ -1302,6 +1303,7 @@ fn xml_http_request_default_response_type_parses_response_xml_for_document_mime( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes( @@ -1330,6 +1332,7 @@ fn xml_http_request_default_response_type_parses_response_xml_for_document_mime( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes(b"plain text".to_vec()), @@ -1409,6 +1412,7 @@ fn xml_http_request_response_document_uses_response_url_and_requester_origin() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes(br#"link"#.to_vec()), @@ -1507,6 +1511,7 @@ fn xml_http_request_document_response_requires_an_eligible_mime_and_well_formed_ redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; if streaming { @@ -1662,6 +1667,7 @@ fn xhr_streamed_response_documents_keep_distinct_source_modification_times() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, @@ -1780,6 +1786,7 @@ fn xhr_response_decoding_uses_headers_received_overrides_for_buffered_and_stream redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; if streaming { @@ -2966,6 +2973,7 @@ __streamingXhr.send(); redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; @@ -3153,6 +3161,7 @@ async fn streaming_subresource_finish_preserves_response_head_cache_state() { }], from_cache: true, cache_state: moli_fetch::ResponseCacheState::Local, + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, @@ -3434,6 +3443,7 @@ async fn streaming_fetch_body_error_records_response_started_then_body_failed() redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, @@ -3536,6 +3546,7 @@ fn install_streaming_fetch_response_fixture( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_source_id, @@ -3599,6 +3610,7 @@ fn install_streaming_fetch_response_fixture( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, @@ -3712,6 +3724,7 @@ async fn streaming_fetch_body_cancel_aborts_streaming_subresource() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, @@ -4350,6 +4363,7 @@ async fn streaming_xhr_materialization_failure_errors_body_source_before_close() redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, diff --git a/moli-renderer-v8/src/script_vm/tests/extracted/child_documents.rs b/moli-renderer-v8/src/script_vm/tests/extracted/child_documents.rs index d2f79a237f..857ee13374 100644 --- a/moli-renderer-v8/src/script_vm/tests/extracted/child_documents.rs +++ b/moli-renderer-v8/src/script_vm/tests/extracted/child_documents.rs @@ -656,6 +656,7 @@ async fn child_navigation_keeps_accepted_beacon_network_only_and_rejects_stale_s redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }) diff --git a/moli-renderer-v8/src/script_vm/tests/extracted/page_runtime.rs b/moli-renderer-v8/src/script_vm/tests/extracted/page_runtime.rs index 1e9f9cdb34..0bd3b827bd 100644 --- a/moli-renderer-v8/src/script_vm/tests/extracted/page_runtime.rs +++ b/moli-renderer-v8/src/script_vm/tests/extracted/page_runtime.rs @@ -326,6 +326,7 @@ async fn child_csp_report_keeps_exact_violation_document_without_v8_after_naviga redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }) diff --git a/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs b/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs index f774e68dda..c0eb8d1ada 100644 --- a/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs +++ b/moli-renderer-v8/src/script_vm/tests/extracted/realms_and_teardown.rs @@ -1968,6 +1968,7 @@ fn isolated_realm_destruction_aborts_fetch_and_detaches_keepalive() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }) diff --git a/moli-renderer-v8/src/script_vm/tests/fetch_request_guard.rs b/moli-renderer-v8/src/script_vm/tests/fetch_request_guard.rs index 52faf95a3c..8df70c4e62 100644 --- a/moli-renderer-v8/src/script_vm/tests/fetch_request_guard.rs +++ b/moli-renderer-v8/src/script_vm/tests/fetch_request_guard.rs @@ -32,6 +32,7 @@ fn fetch_request_initializers_use_request_header_guards() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; vm.complete_async_subresource_fetch(crate::types::AsyncSubresourceFetchCompletion { diff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rs index 22b367771b..deb94c8bcc 100644 --- a/moli-renderer-v8/src/script_vm/tests/mod.rs +++ b/moli-renderer-v8/src/script_vm/tests/mod.rs @@ -375,6 +375,7 @@ fn register_pending_window_fetch_for_test( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, network_request_headers: None, diff --git a/moli-renderer-v8/src/script_vm/tests/post_parse.rs b/moli-renderer-v8/src/script_vm/tests/post_parse.rs index 579edb345a..fc5479dccf 100644 --- a/moli-renderer-v8/src/script_vm/tests/post_parse.rs +++ b/moli-renderer-v8/src/script_vm/tests/post_parse.rs @@ -358,6 +358,7 @@ fn external_script_redirect_final_url_obeys_script_src_csp() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "globalThis.__redirectScriptRan = true;".to_owned(), diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/url_and_initializer_semantics.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/url_and_initializer_semantics.rs index 26505c85b7..ba3786d413 100644 --- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/url_and_initializer_semantics.rs +++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/url_and_initializer_semantics.rs @@ -726,6 +726,7 @@ fn response_clone_tees_pending_network_body_after_parent_consumption() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_source_id, @@ -833,6 +834,7 @@ fn pending_fetch_body_pipe_through_text_decoder_stream_pulls_future_chunks() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, body_source_id, @@ -957,6 +959,7 @@ fn materialize_response_object_preserves_redirected_slot() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, "redirected-body".to_owned(), @@ -1340,6 +1343,7 @@ redirect_mode: moli_fetch::RequestRedirectMode::Follow, redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let id = crate::network_host::new_network_body_source_id(); @@ -1444,6 +1448,7 @@ fn opaque_window_fetch_keeps_blocked_bytes_out_of_internal_clone_consumers() { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, }, @@ -1557,6 +1562,7 @@ redirect_mode: moli_fetch::RequestRedirectMode::Follow, method: "GET", mode: mol redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, moli_fetch::ResponseBody::materialized_bytes(Vec::new()), diff --git a/moli-renderer-v8/src/service_worker_runtime/script_loading.rs b/moli-renderer-v8/src/service_worker_runtime/script_loading.rs index fd742035ea..381537c461 100644 --- a/moli-renderer-v8/src/service_worker_runtime/script_loading.rs +++ b/moli-renderer-v8/src/service_worker_runtime/script_loading.rs @@ -555,6 +555,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; @@ -952,6 +953,7 @@ mod tests { redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let mut resource = diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs index c270b26189..28395f9803 100644 --- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs +++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs @@ -116,6 +116,7 @@ fn service_worker_fetch_stream_response_head( redirect_chain: job.request.redirect_chain().to_vec(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, } } @@ -640,6 +641,7 @@ impl ServiceWorkerRuntimeService { redirect_chain: job.request.redirect_chain().to_vec(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }, String::from_utf8_lossy(&response.body).into_owned(), diff --git a/moli-renderer-v8/src/stylesheet_runtime/connected.rs b/moli-renderer-v8/src/stylesheet_runtime/connected.rs index 5a45dc9821..b3d9daf363 100644 --- a/moli-renderer-v8/src/stylesheet_runtime/connected.rs +++ b/moli-renderer-v8/src/stylesheet_runtime/connected.rs @@ -21,22 +21,29 @@ struct ConnectedLinkReadinessFetchResponse { response: crate::protocol_types::NavigationResponse, origin_clean: bool, load_event_successful: bool, + response_filter: Option, + integrity_matches: bool, + from_service_worker: bool, } impl ConnectedLinkReadinessFetchResponse { fn from_response( response: crate::protocol_types::NavigationResponse, - origin_clean: bool, + response_filter: Option, load_event_successful: bool, integrity: Option<&str>, + from_service_worker: bool, ) -> Self { // Hash the original body, including binary responses. A matching digest // must not make an opaque response eligible, including SW responses to // an otherwise same-origin request. // Retain the network response even on integrity failure: its downloaded // bytes still contribute to the preload's Resource Timing entry. - let load_event_successful = load_event_successful - && crate::subresource_integrity::response_matches_subresource_integrity_metadata( + let origin_clean = response_filter + .as_ref() + .is_none_or(|filter| filter.is_readable()); + let integrity_matches = + crate::subresource_integrity::response_matches_subresource_integrity_metadata( response.body_bytes(), integrity, origin_clean, @@ -44,9 +51,25 @@ impl ConnectedLinkReadinessFetchResponse { Self { response, origin_clean, - load_event_successful, + load_event_successful: load_event_successful && integrity_matches, + response_filter, + integrity_matches, + from_service_worker, } } + + fn preload_response( + &self, + ) -> Result { + if !self.integrity_matches { + return Err("preload response failed its integrity check".to_owned()); + } + Ok(crate::network::preloads::DocumentPreloadResponse { + response: self.response.clone(), + from_service_worker: self.from_service_worker, + response_filter: self.response_filter.clone(), + }) + } } #[derive(Debug)] @@ -1179,6 +1202,18 @@ impl DocumentRuntime { let start_unix_millis = moli_time::unix_epoch_millis(); let resource_task_runner = resource_loader.task_runner(); let request_origin = resource_loader.fetch_context().request_origin(); + let preload = fetch_options + .link_preload + .then(|| { + let request = connected_link_readiness_request( + &document_url, + &request_origin, + &url, + &fetch_options, + ); + loader.register_document_preload(&request) + }) + .flatten(); let document_owner = operation.document_owner(); resource_loader.spawn_resource_task(async move { let result = fetch_connected_link_readiness_with_service_worker( @@ -1191,6 +1226,13 @@ impl DocumentRuntime { service_worker_context, ) .await; + let preload = preload.map(|producer| { + let response = result + .as_ref() + .map_err(Clone::clone) + .and_then(ConnectedLinkReadinessFetchResponse::preload_response); + (producer, response) + }); let completion = ConnectedLoadCompletion { operation, successful: result @@ -1212,6 +1254,9 @@ impl DocumentRuntime { }], }; let _ = task_producer.send_connected_completion(completion); + if let Some((producer, response)) = preload { + producer.complete(response); + } }); return result; } @@ -2302,19 +2347,16 @@ async fn fetch_connected_link_readiness_with_service_worker( .await { Ok(Some(response)) => { - let origin_clean = response - .response_filter - .as_ref() - .is_none_or(|filter| filter.is_readable()); let load_event_successful = connected_link_load_event_successful( &response.response, - response.response_filter, + response.response_filter.clone(), ); return Ok(ConnectedLinkReadinessFetchResponse::from_response( *response.response, - origin_clean, + response.response_filter, load_event_successful, integrity, + !response.from_network_fallback, )); } Ok(None) => {} @@ -2330,18 +2372,18 @@ async fn fetch_connected_link_readiness_with_service_worker( .await .map(|response| { let load_event_successful = connected_link_load_event_successful(&response, None); - let origin_clean = crate::network_host::network_response_filter( + let response_filter = crate::network_host::network_response_filter( &request_origin, &response.head(), request_mode, moli_fetch::RequestRedirectMode::Follow, - ) - .is_none_or(|filter| filter.is_readable()); + ); ConnectedLinkReadinessFetchResponse::from_response( response, - origin_clean, + response_filter, load_event_successful, integrity, + false, ) }) } diff --git a/moli-renderer-v8/src/subresource_integrity.rs b/moli-renderer-v8/src/subresource_integrity.rs index 630f2223d8..e34466667c 100644 --- a/moli-renderer-v8/src/subresource_integrity.rs +++ b/moli-renderer-v8/src/subresource_integrity.rs @@ -1,11 +1,29 @@ use base64::Engine; use moli_crypto::DigestAlgorithm; +#[derive(PartialEq, Eq)] pub(crate) struct IntegrityMetadataHash<'a> { pub(crate) algorithm: DigestAlgorithm, pub(crate) digest: &'a str, } +/// Preload consumption compares parsed metadata sets, including weaker hashes, +/// while ignoring unknown algorithms, duplicate tokens, and unknown options. +/// A consumer with no supported metadata can consume any matching preload, +/// including one whose integrity check failed. +pub(crate) fn integrity_metadata_allows_preload_consumption( + preload: &str, + consumer: Option<&str>, +) -> bool { + let consumer: Vec<_> = integrity_metadata_hashes(consumer.unwrap_or_default()).collect(); + if consumer.is_empty() { + return true; + } + let preload: Vec<_> = integrity_metadata_hashes(preload).collect(); + consumer.iter().all(|hash| preload.contains(hash)) + && preload.iter().all(|hash| consumer.contains(hash)) +} + struct ParsedIntegrityMetadata { tokens: Vec, } @@ -191,6 +209,33 @@ mod tests { )); } + #[test] + fn preload_integrity_compares_sets_including_weaker_hashes() { + let preload = "sha384-AAAA sha256-BBBB"; + for compatible in [ + None, + Some(""), + Some("unknown-AAAA"), + Some("sha256-BBBB sha384-AAAA"), + Some("sha384-AAAA?ignored sha256-BBBB sha384-AAAA unknown-AAAA"), + ] { + assert!( + integrity_metadata_allows_preload_consumption(preload, compatible), + "{compatible:?}" + ); + } + for incompatible in ["sha384-AAAA", "sha384-AAAA sha256-CCCC", "sha512-AAAA"] { + assert!( + !integrity_metadata_allows_preload_consumption(preload, Some(incompatible)), + "{incompatible}" + ); + } + assert!(!integrity_metadata_allows_preload_consumption( + "", + Some("sha384-AAAA") + )); + } + fn response_body_matches_subresource_integrity_metadata( body: &[u8], integrity: Option<&str>, diff --git a/moli-renderer-v8/src/types.rs b/moli-renderer-v8/src/types.rs index 1eb5c5ebe9..0ca7630863 100644 --- a/moli-renderer-v8/src/types.rs +++ b/moli-renderer-v8/src/types.rs @@ -673,6 +673,8 @@ pub(super) enum AsyncSubresourceFetchResult { encoded: moli_parkable_image::ParkableImage, }, Failure(String), + /// The preload producer already reported this failed fetch. + PreloadFailure(String), } impl AsyncSubresourceFetchResult { @@ -702,7 +704,7 @@ impl AsyncSubresourceFetchResult { pub(super) fn into_result(self) -> std::result::Result { match self { Self::Response(response) | Self::Image { response, .. } => Ok(response), - Self::Failure(error) => Err(error), + Self::Failure(error) | Self::PreloadFailure(error) => Err(error), } } @@ -710,7 +712,7 @@ impl AsyncSubresourceFetchResult { pub(super) fn as_ref(&self) -> std::result::Result<&NavigationResponse, &String> { match self { Self::Response(response) | Self::Image { response, .. } => Ok(response), - Self::Failure(error) => Err(error), + Self::Failure(error) | Self::PreloadFailure(error) => Err(error), } } @@ -718,19 +720,19 @@ impl AsyncSubresourceFetchResult { pub(super) fn encoded(&self) -> Option<&moli_parkable_image::ParkableImage> { match self { Self::Image { encoded, .. } => Some(encoded), - Self::Response(_) | Self::Failure(_) => None, + Self::Response(_) | Self::Failure(_) | Self::PreloadFailure(_) => None, } } #[cfg(test)] pub(super) fn is_ok(&self) -> bool { - !matches!(self, Self::Failure(_)) + !matches!(self, Self::Failure(_) | Self::PreloadFailure(_)) } #[cfg(test)] pub(super) fn err(self) -> Option { match self { - Self::Failure(error) => Some(error), + Self::Failure(error) | Self::PreloadFailure(error) => Some(error), Self::Response(_) | Self::Image { .. } => None, } } @@ -739,14 +741,14 @@ impl AsyncSubresourceFetchResult { pub(super) fn expect(self, message: &str) -> NavigationResponse { match self { Self::Response(response) | Self::Image { response, .. } => response, - Self::Failure(error) => panic!("{message}: {error:?}"), + Self::Failure(error) | Self::PreloadFailure(error) => panic!("{message}: {error:?}"), } } #[cfg(test)] pub(super) fn expect_err(self, message: &str) -> String { match self { - Self::Failure(error) => error, + Self::Failure(error) | Self::PreloadFailure(error) => error, Self::Response(response) | Self::Image { response, .. } => { panic!("{message}: {response:?}") } @@ -763,19 +765,7 @@ impl From> for AsyncSubresourceF } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub(super) enum AsyncSubresourceFetchResponseFilter { - Basic, - Cors(Vec), - Opaque, - OpaqueRedirect, -} - -impl AsyncSubresourceFetchResponseFilter { - pub(super) fn is_readable(&self) -> bool { - matches!(self, Self::Basic | Self::Cors(_)) - } -} +pub(super) use moli_fetch::FetchResponseFilter as AsyncSubresourceFetchResponseFilter; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(crate) struct SubresourcePolicyContext { diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rs index 2607cba135..a7fdf0618c 100644 --- a/moli-renderer-v8/src/worker/global_scope/fetch.rs +++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs @@ -2678,6 +2678,7 @@ pub(in crate::worker) fn worker_response_from_body( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }) } @@ -3090,6 +3091,7 @@ pub(in crate::worker) fn drain_worker_fetch_completion_result( response_body, response_from_cache: response_head.from_cache, response_cache_state: response_head.cache_state, + response_preload_state: response_head.preload_state.clone(), }) } else { None @@ -3125,6 +3127,7 @@ pub(in crate::worker) fn drain_worker_fetch_completion_result( response_body: response_body.clone(), from_cache: response_head.from_cache, cache_state: response_head.cache_state, + preload_state: response_head.preload_state.clone(), }; pending.paused_response = Some(PausedWorkerSubresourceResponse { response_filter: completion.response_filter.clone(), diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rs index b3077b08a9..bbbb9dfb00 100644 --- a/moli-renderer-v8/src/worker/global_scope/xhr.rs +++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs @@ -1070,6 +1070,7 @@ pub(in crate::worker) fn drain_worker_xhr_completion( response_body, response_from_cache: response_head.from_cache, response_cache_state: response_head.cache_state, + response_preload_state: response_head.preload_state.clone(), }) } else { None @@ -1106,6 +1107,7 @@ pub(in crate::worker) fn drain_worker_xhr_completion( response_body: response_body.clone(), from_cache: response_head.from_cache, cache_state: response_head.cache_state, + preload_state: response_head.preload_state.clone(), }; pending.paused_response = Some(PausedWorkerSubresourceResponse { response_filter: None, diff --git a/moli-renderer-v8/src/worker/thread/dispatch.rs b/moli-renderer-v8/src/worker/thread/dispatch.rs index 9f41772f86..de9261d387 100644 --- a/moli-renderer-v8/src/worker/thread/dispatch.rs +++ b/moli-renderer-v8/src/worker/thread/dispatch.rs @@ -3447,6 +3447,7 @@ fn start_service_worker_navigation_preload_response_in_context( redirect_chain: Vec::new(), from_cache: false, cache_state: Default::default(), + preload_state: Default::default(), negotiated_http_version: None, }; let response_object = build_navigation_preload_response_object_from_stream_for_request_mode(