diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs index e26484ccfc7..68777506185 100644 --- a/moli-core/src/runtime/mod.rs +++ b/moli-core/src/runtime/mod.rs @@ -1470,6 +1470,10 @@ fn external_raw_document_body_from_streaming_response_with_observer_and_progress ), None => ExternalRawDocumentBodyStream::channel(completion_rx), }; + let body_stream = match response.navigation_mime_type() { + Some(mime) => body_stream.with_navigation_mime_type(mime.to_owned()), + None => body_stream, + }; tokio::spawn(async move { let result = async { loop { diff --git a/moli-fetch/src/streaming_response.rs b/moli-fetch/src/streaming_response.rs index 28e11797ce2..50efd0f37ea 100644 --- a/moli-fetch/src/streaming_response.rs +++ b/moli-fetch/src/streaming_response.rs @@ -1,4 +1,4 @@ -use std::{any::Any, fmt}; +use std::{any::Any, collections::VecDeque, fmt}; use anyhow::{Result, anyhow}; use moli_cookie_jar::{StoredCookieQueryReport, StoredCookieSetReport}; @@ -171,6 +171,12 @@ impl Drop for StreamingHtmlResponse { } } +#[derive(Debug, Default)] +struct NavigationBodyState { + buffered_chunks: VecDeque>, + mime_type: Option, +} + struct BodyPrefixObserver { limit: usize, bytes: Vec, @@ -203,6 +209,7 @@ pub struct StreamingRawResponse { pub negotiated_http_version: Option, network_request_extra_info: Option, body_chunks: mpsc::UnboundedReceiver>, + navigation: Option>, cancel_handle: FetchCancelHandle, completion: Option>>, lifetime_lease: Option, @@ -247,6 +254,7 @@ impl StreamingRawResponse { negotiated_http_version: None, network_request_extra_info: None, body_chunks, + navigation: None, cancel_handle, completion: Some(completion), lifetime_lease: None, @@ -275,6 +283,7 @@ impl StreamingRawResponse { negotiated_http_version: head.negotiated_http_version, network_request_extra_info: None, body_chunks, + navigation: None, cancel_handle, completion: Some(completion), lifetime_lease: None, @@ -332,11 +341,63 @@ impl StreamingRawResponse { } pub async fn next_chunk(&mut self) -> Option> { - let chunk = self.body_chunks.recv().await; + let buffered = self + .navigation + .as_mut() + .and_then(|navigation| navigation.buffered_chunks.pop_front()); + let chunk = match buffered { + Some(chunk) => Some(chunk), + None => self.body_chunks.recv().await, + }; self.observe_body_prefix(chunk.as_deref()); chunk } + /// Peeks the bounded navigation prefix without consuming response bytes. + /// Header overrides are supplied by the navigation owner after Fetch's + /// response-stage decision. Cancelling this future retains already read + /// chunks, and ordinary consumers observe every byte exactly once. + pub async fn sniff_navigation_mime_type( + &mut self, + headers: &[(String, Vec)], + ) -> Result { + let mut sniffer = + moli_web_mime::NavigationMimeSniffer::new(self.final_url.scheme(), headers); + let navigation = self.navigation.get_or_insert_with(Default::default); + navigation.mime_type = None; + for chunk in &navigation.buffered_chunks { + sniffer.push(chunk); + } + let (mime_type, eof) = loop { + if let Some(mime) = sniffer.mime_type() { + break (mime.to_owned(), false); + } + match self.body_chunks.recv().await { + Some(chunk) => { + sniffer.push(&chunk); + if !chunk.is_empty() { + navigation.buffered_chunks.push_back(chunk); + } + } + None => break (sniffer.finish().to_owned(), true), + } + }; + if eof { + self.finish_transfer().await?; + } + self.navigation + .as_mut() + .expect("initialized navigation prefix state") + .mime_type = Some(mime_type.clone()); + Ok(mime_type) + } + + pub fn navigation_mime_type(&self) -> Option<&str> { + self.navigation + .as_ref() + .and_then(|navigation| navigation.mime_type.as_deref()) + } + pub fn with_network_request_extra_info( mut self, network_request_extra_info: Option, @@ -358,6 +419,14 @@ impl StreamingRawResponse { pub fn try_next_chunk(&mut self) -> Option> { // See StreamingHtmlResponse::try_next_chunk. + if let Some(chunk) = self + .navigation + .as_mut() + .and_then(|navigation| navigation.buffered_chunks.pop_front()) + { + self.observe_body_prefix(Some(&chunk)); + return Some(chunk); + } match self.body_chunks.try_recv() { Ok(chunk) => { self.observe_body_prefix(Some(&chunk)); @@ -379,16 +448,25 @@ impl StreamingRawResponse { /// a streaming consumer distinguish an open, temporarily empty body from /// an already exhausted one without waiting. pub fn body_chunk_stream_is_exhausted(&self) -> bool { - self.body_chunks.is_closed() && self.body_chunks.is_empty() + self.navigation + .as_ref() + .is_none_or(|navigation| navigation.buffered_chunks.is_empty()) + && self.body_chunks.is_closed() + && self.body_chunks.is_empty() } pub async fn finish(&mut self) -> Result<()> { while self.next_chunk().await.is_some() {} - let completion = self - .completion - .as_mut() - .expect("streaming raw completion should only be awaited once") - .await; + self.finish_transfer().await + } + + async fn finish_transfer(&mut self) -> Result<()> { + // A short sniffed response can already have reached terminal success + // before its retained prefix is replayed to the document consumer. + let Some(completion) = self.completion.as_mut() else { + return Ok(()); + }; + let completion = completion.await; // See StreamingHtmlResponse::finish: taking the receiver before the // await would make a cancelled finish future look completed to Drop. self.completion = None; @@ -636,6 +714,109 @@ mod tests { )); } + #[tokio::test] + async fn navigation_sniff_replays_a_split_signature_without_duplicate_observation() { + let (body_tx, body_rx) = mpsc::unbounded_channel(); + let (completion_tx, completion_rx) = oneshot::channel(); + let (prefix_tx, prefix_rx) = std::sync::mpsc::channel(); + let mut response = StreamingRawResponse::new_with_head( + sample_response_head(), + body_rx, + FetchCancelHandle::new(), + completion_rx, + ) + .with_body_prefix_observer(8, move |bytes| prefix_tx.send(bytes).unwrap()); + let first = b"\x89PNG\r\n\x1a".to_vec(); + body_tx.send(first.clone()).unwrap(); + let mut sniff = Box::pin(response.sniff_navigation_mime_type(&[])); + std::future::poll_fn(|cx| { + assert!(sniff.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + body_tx.send(b"\n".to_vec()).unwrap(); + assert_eq!(sniff.await.unwrap(), "image/png"); + assert!(!response.body_chunk_stream_is_exhausted()); + assert!(matches!( + prefix_rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + assert_eq!(response.next_chunk().await, Some(first)); + assert_eq!(response.try_next_chunk(), Some(b"\n".to_vec())); + assert_eq!(prefix_rx.try_recv().unwrap(), b"\x89PNG\r\n\x1a\n"); + body_tx.send(b"tail".to_vec()).unwrap(); + drop(body_tx); + completion_tx.send(Ok(())).unwrap(); + assert_eq!(response.next_chunk().await, Some(b"tail".to_vec())); + assert_eq!(response.next_chunk().await, None); + response.finish().await.unwrap(); + assert_eq!(response.head().headers, sample_response_head().headers); + assert_eq!(response.navigation_mime_type(), Some("image/png")); + assert!(matches!( + prefix_rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Disconnected) + )); + } + + #[tokio::test] + async fn cancelling_navigation_sniff_retains_bytes_and_transfer_cancellation() { + let (body_tx, body_rx) = mpsc::unbounded_channel(); + let (completion_tx, completion_rx) = oneshot::channel(); + let cancel = FetchCancelHandle::new(); + let mut response = StreamingRawResponse::new_with_head( + sample_response_head(), + body_rx, + cancel.clone(), + completion_rx, + ); + body_tx.send(b"GIF89".to_vec()).unwrap(); + let mut sniff = Box::pin(response.sniff_navigation_mime_type(&[])); + std::future::poll_fn(|cx| { + assert!(sniff.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(sniff); + assert_eq!(response.try_next_chunk(), Some(b"GIF89".to_vec())); + assert!(!cancel.is_cancelled()); + drop(response); + assert!(cancel.is_cancelled()); + assert!(completion_tx.send(Ok(())).is_err()); + } + + #[tokio::test] + async fn navigation_sniff_preserves_short_eof_bytes_and_transfer_errors() { + for fails in [false, true] { + let (body_tx, body_rx) = mpsc::unbounded_channel(); + let (completion_tx, completion_rx) = oneshot::channel(); + let mut response = StreamingRawResponse::new_with_head( + sample_response_head(), + body_rx, + FetchCancelHandle::new(), + completion_rx, + ); + body_tx.send(b"short text".to_vec()).unwrap(); + drop(body_tx); + completion_tx + .send(if fails { + Err(anyhow!("sniff transfer failed")) + } else { + Ok(()) + }) + .unwrap(); + let result = response.sniff_navigation_mime_type(&[]).await; + if fails { + assert_eq!(result.unwrap_err().to_string(), "sniff transfer failed"); + } else { + assert_eq!(result.unwrap(), "text/plain"); + assert!(!response.body_chunk_stream_is_exhausted()); + assert_eq!(response.next_chunk().await, Some(b"short text".to_vec())); + response.finish().await.unwrap(); + assert!(response.body_chunk_stream_is_exhausted()); + } + } + } + #[tokio::test] async fn raw_prefix_observation_distinguishes_pending_input_from_body_eof() { let (body_tx, body_rx) = mpsc::unbounded_channel(); diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs index 27073b0e136..a7cdb530cac 100644 --- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs +++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs @@ -508,7 +508,19 @@ async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() for (name, mime, _, _, payload) in cases { fixture = fixture.route( &format!("/{name}"), - axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }), + axum::routing::get(move || async move { + let mut response = ([(header::CONTENT_TYPE, mime)], payload).into_response(); + if name == "plain.xml" { + // This payload contains NUL. Keep its literal-text parser + // coverage explicit: without nosniff, browsers classify + // the bytes as binary and download instead of navigating. + response.headers_mut().insert( + header::X_CONTENT_TYPE_OPTIONS, + axum::http::HeaderValue::from_static("nosniff"), + ); + } + response + }), ); } fixture = fixture.route( @@ -572,6 +584,80 @@ async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; } +#[tokio::test] +async fn webdriver_classic_binary_plain_text_download_retains_committed_document() { + const PAYLOAD: &str = "\n<&\r\nbeta\rgamma\0"; + let cases = [ + ("plain", "text/plain", false), + ("plain-utf8", "text/plain; charset=utf-8", false), + ("nosniff-plain", "text/plain", true), + ("nosniff-plain-utf8", "text/plain; charset=utf-8", true), + ]; + let mut fixture = axum::Router::new().route( + "/seed", + axum::routing::get(|| async { + ( + [(header::CONTENT_TYPE, "text/html")], + "seedold document", + ) + }), + ); + for (name, mime, nosniff) in cases { + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get(move || async move { + let mut response = ([(header::CONTENT_TYPE, mime)], PAYLOAD).into_response(); + if nosniff { + response.headers_mut().insert( + header::X_CONTENT_TYPE_OPTIONS, + axum::http::HeaderValue::from_static("nosniff"), + ); + } + response + }), + ); + } + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "binary-plain-navigation"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, _, nosniff) in cases { + for path in ["seed", name] { + let navigated = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{path}")}), + ) + .await; + assert_eq!(navigated, json!({"value":null}), "{name}: {path}"); + } + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({"script":"return [location.pathname,document.contentType,document.body.textContent,window.executed??null,document.querySelectorAll('script').length];","args":[]}), + ) + .await; + let expected = if nosniff { + json!([ + format!("/{name}"), + "text/plain", + PAYLOAD + .replace("\r\n", "\n") + .replace('\r', "\n") + .replace('\0', "\u{fffd}"), + null, + 0 + ]) + } else { + json!(["/seed", "text/html", "old document", null, 0]) + }; + assert_eq!(observed["value"], expected, "{name}"); + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + #[tokio::test] async fn webdriver_classic_page_load_strategy_maps_navigation_wait_policy() { let app = build_router(test_state()); diff --git a/moli-protocol/src/conn/browser_context/target_session_owner.rs b/moli-protocol/src/conn/browser_context/target_session_owner.rs index 399aa84e94f..90d0e4853ec 100644 --- a/moli-protocol/src/conn/browser_context/target_session_owner.rs +++ b/moli-protocol/src/conn/browser_context/target_session_owner.rs @@ -1466,6 +1466,20 @@ impl CdpConnection { } } + pub(crate) fn cancel_uncommitted_navigation_for_owner(&mut self, owner: &CommandOwnerScope) { + let Some(mut resolved) = self.target_session_owner_mut_for_owner(owner) else { + return; + }; + let target = resolved.target_mut(); + if target + .runtime_slot + .page_slot_mut() + .cancel_uncommitted_document_navigation() + { + target.owner_state.clear_pending_navigation_history_update(); + } + } + pub(crate) fn take_crashed_page_for_owner( &mut self, owner: &CommandOwnerScope, diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index 6d1bc5ba66b..041b6c851ee 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -1195,11 +1195,17 @@ impl BackgroundNavigationLoadJob { .into_parts_with_observation_journal(); let reserved_service_worker_client = navigation_response.reserved_service_worker_client; let document_fetch_context_seed = navigation_response.document_fetch_context_seed; - let defer_early_result_for_http_error_body = - response_status_may_use_http_error_page(response.status); + let defer_early_result_until_document_decision = + response_status_may_use_http_error_page(response.status) + || moli_web_mime::NavigationMimeSniffer::new( + response.final_url.scheme(), + &response.headers, + ) + .mime_type() + .is_none_or(|mime| !moli_web_mime::is_supported_document_mime_type(mime)); if !super::downloads::response_headers_indicate_download(&response.headers) && !response_status_has_no_content(response.status) - && !defer_early_result_for_http_error_body + && !defer_early_result_until_document_decision && let Some(early_result) = early_result.take() { early_result_sent = early_result.emit(); @@ -1222,7 +1228,7 @@ impl BackgroundNavigationLoadJob { reply_boundary, ) .await; - if defer_early_result_for_http_error_body { + if defer_early_result_until_document_decision { early_result_sent = Self::emit_early_result_for_successful_document(&mut early_result, &navigation); } @@ -1552,6 +1558,11 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( .into_iter() .map(Into::into) .collect::>(); + let navigation_mime_type = response + .sniff_navigation_mime_type(&response_headers) + .await?; + let body_progress_source = + body_progress_source.with_response_mime_type(navigation_mime_type.clone()); let network_events = CompletedMainDocumentNetworkEvents::new( initial_request_cookie_report.clone(), response_status, @@ -1561,10 +1572,13 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( network_extra_info_available, response_from_cache, ) + .with_response_mime_type(navigation_mime_type.clone()) .with_negotiated_http_version(negotiated_http_version) .with_network_observation_journal(network_observation_journal.clone()); - if super::downloads::response_headers_indicate_download(&response_headers) { + if super::downloads::response_headers_indicate_download(&response_headers) + || !moli_web_mime::is_supported_document_mime_type(&navigation_mime_type) + { let mut head = response.head(); head.status = response_status; head.headers = response_headers; @@ -1632,7 +1646,8 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( // including XML charsets and BOMs. let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY); let (completion_tx, completion_rx) = oneshot::channel(); - let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx); + let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx) + .with_navigation_mime_type(navigation_mime_type); let page_storage = load_inputs.page_storage_handles(); let main_document_commit = load_inputs .main_document_commit_for_final_url(&final_url, None) @@ -3504,6 +3519,27 @@ impl CdpConnection { } } +fn captured_navigation_mime_type( + head: &ResponseHead, + body: &CapturedBody, +) -> anyhow::Result { + let mut sniffer = + moli_web_mime::NavigationMimeSniffer::new(head.final_url.scheme(), &head.headers); + if let Some(mime) = sniffer.mime_type() { + return Ok(mime.to_owned()); + } + let mut reader = body.chunk_reader(moli_web_mime::NAVIGATION_MIME_PREFIX_LIMIT)?; + loop { + match reader.next_chunk()? { + Some(chunk) => sniffer.push(&chunk), + None => return Ok(sniffer.finish().to_owned()), + } + if let Some(mime) = sniffer.mime_type() { + return Ok(mime.to_owned()); + } + } +} + async fn prepare_navigation_from_captured_raw_response_with_engine_async( engine: &mut NavigationEngine, page_reservation: RendererDocumentPreparationTarget, @@ -3586,6 +3622,9 @@ async fn prepare_captured_document_response_with_engine_async( synthetic_body: bool, reply_boundary: RendererReplyBoundary, ) -> Result { + let navigation_mime_type = captured_navigation_mime_type(&head, &body)?; + let body_progress_source = + body_progress_source.with_response_mime_type(navigation_mime_type.clone()); let auxiliary_document_response = load_inputs .auxiliary_document_response .as_ref() @@ -3601,7 +3640,8 @@ async fn prepare_captured_document_response_with_engine_async( let redirect_count = head.redirect_chain.len(); let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY); let (completion_tx, completion_rx) = oneshot::channel(); - let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx); + let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx) + .with_navigation_mime_type(navigation_mime_type); let page_storage = load_inputs.page_storage_handles(); let main_document_commit = load_inputs .main_document_commit_for_final_url(&final_url, network_error_page.as_ref()) diff --git a/moli-protocol/src/conn/runtime_load/captured_response.rs b/moli-protocol/src/conn/runtime_load/captured_response.rs index af675d74277..30acd0b453e 100644 --- a/moli-protocol/src/conn/runtime_load/captured_response.rs +++ b/moli-protocol/src/conn/runtime_load/captured_response.rs @@ -62,7 +62,11 @@ impl PreparedCapturedNavigationResponse { &self.body_progress_source, )); } - if super::super::downloads::response_headers_indicate_download(&self.head.headers) { + if super::super::downloads::response_headers_indicate_download(&self.head.headers) + || !moli_web_mime::is_supported_document_mime_type(&captured_navigation_mime_type( + &self.head, &self.body, + )?) + { complete_auxiliary_download_response(&self.load_inputs, self.head.clone()); let body = self .body diff --git a/moli-protocol/src/domains/network/main_document_progress/emit.rs b/moli-protocol/src/domains/network/main_document_progress/emit.rs index c26459e1dc8..b44af41500f 100644 --- a/moli-protocol/src/domains/network/main_document_progress/emit.rs +++ b/moli-protocol/src/domains/network/main_document_progress/emit.rs @@ -289,6 +289,7 @@ pub(super) fn emit_main_document_response_received( from_cache: bool, negotiated_http_version: Option, has_extra_info: bool, + response_mime_type: Option<&str>, ) { if from_cache { emit_request_served_from_cache(output, session_id, request_id); @@ -326,7 +327,7 @@ pub(super) fn emit_main_document_response_received( status: Some(status), status_text: None, response_headers: response_headers.to_vec(), - response_mime_type: None, + response_mime_type: response_mime_type.map(str::to_owned), response_protocol: negotiated_http_version .map(|version| version.protocol_name().to_owned()), encoded_data_length: Some(encoded_data_length), diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rs index 124ff785225..8777ab3ac32 100644 --- a/moli-protocol/src/domains/network/main_document_progress/mod.rs +++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs @@ -631,6 +631,7 @@ enum MainDocumentResponseVisibility { pub struct MainDocumentBodyProgressSource { live_source: Option, response_visibility: MainDocumentResponseVisibility, + response_mime_type: Option, } impl MainDocumentBodyProgressSource { @@ -641,9 +642,15 @@ impl MainDocumentBodyProgressSource { Self { live_source, response_visibility, + response_mime_type: None, } } + pub(crate) fn with_response_mime_type(mut self, mime_type: String) -> Self { + self.response_mime_type = Some(mime_type); + self + } + pub(crate) fn body_network_progress_for_response_head( &self, head: &ResponseHead, @@ -715,6 +722,7 @@ impl MainDocumentBodyProgressSource { network_extra_info_available, response_from_cache, negotiated_http_version, + self.response_mime_type.as_deref(), ); } MainDocumentResponseVisibility::AfterResponseStageContinue => { @@ -725,6 +733,7 @@ impl MainDocumentBodyProgressSource { network_extra_info_available, response_from_cache, negotiated_http_version, + self.response_mime_type.as_deref(), ); } } @@ -749,6 +758,7 @@ impl MainDocumentBodyProgressSource { final_url: final_url.clone(), status: response_status, headers: response_headers.to_vec(), + mime_type: self.response_mime_type.clone(), cookie_set_reports: Vec::new(), extra_info_status: response_status, extra_info_headers: response_headers.to_vec(), @@ -894,6 +904,9 @@ impl MainDocumentBodyProgressSource { events.response_stage_metadata_already_emitted = self.response_visibility == MainDocumentResponseVisibility::AfterResponseStageContinue; events + .response_mime_type + .clone_from(&self.response_mime_type); + events } } @@ -902,6 +915,7 @@ pub(crate) struct CompletedMainDocumentNetworkEvents { pub(crate) final_request_cookie_report: Option, pub(crate) response_status: u16, pub(crate) response_headers: Vec<(String, Vec)>, + response_mime_type: Option, pub(crate) response_cookie_reports: Vec, pub(crate) redirect_chain: Vec, pub(crate) network_extra_info_available: bool, @@ -912,6 +926,11 @@ pub(crate) struct CompletedMainDocumentNetworkEvents { } impl CompletedMainDocumentNetworkEvents { + pub(crate) fn with_response_mime_type(mut self, mime_type: String) -> Self { + self.response_mime_type = Some(mime_type); + self + } + pub(crate) fn new( final_request_cookie_report: Option, response_status: u16, @@ -925,6 +944,7 @@ impl CompletedMainDocumentNetworkEvents { final_request_cookie_report, response_status, response_headers, + response_mime_type: None, response_cookie_reports, redirect_chain, network_extra_info_available, @@ -1201,6 +1221,7 @@ impl MainDocumentLiveNetworkProgressSource { network_extra_info_available: bool, response_from_cache: bool, negotiated_http_version: Option, + response_mime_type: Option<&str>, ) { let (extra_info_status, extra_info_headers) = observed_response_metadata( network_observation_journal, @@ -1221,6 +1242,7 @@ impl MainDocumentLiveNetworkProgressSource { response_from_cache, negotiated_http_version, network_extra_info_available && !response_from_cache, + response_mime_type, ); } @@ -1232,6 +1254,7 @@ impl MainDocumentLiveNetworkProgressSource { network_extra_info_available: bool, response_from_cache: bool, negotiated_http_version: Option, + response_mime_type: Option<&str>, ) { self.emit_response_received_with_extra_info_policy( final_url, @@ -1245,6 +1268,7 @@ impl MainDocumentLiveNetworkProgressSource { response_from_cache, negotiated_http_version, network_extra_info_available && !response_from_cache, + response_mime_type, ); } @@ -1261,6 +1285,7 @@ impl MainDocumentLiveNetworkProgressSource { response_from_cache: bool, negotiated_http_version: Option, has_extra_info: bool, + response_mime_type: Option<&str>, ) { self.send_progress_events( MainDocumentProgressPhase::ResponseReceived, @@ -1269,6 +1294,7 @@ impl MainDocumentLiveNetworkProgressSource { final_url: final_url.clone(), status: response_status, headers: response_headers.to_vec(), + mime_type: response_mime_type.map(str::to_owned), cookie_set_reports: response_cookie_reports.to_vec(), extra_info_status, extra_info_headers, @@ -1467,6 +1493,7 @@ pub(crate) fn emit_child_document_navigation_network_background_events( final_url, status: network.status, headers: network.response_headers.clone(), + mime_type: None, cookie_set_reports: Vec::new(), extra_info_status: network.status, extra_info_headers: network.response_headers.clone(), @@ -1904,6 +1931,7 @@ impl CompletedMainDocumentProgressContext { final_url: final_url.clone(), status: events.response_status, headers: events.response_headers.clone(), + mime_type: events.response_mime_type.clone(), cookie_set_reports: events.response_cookie_reports.clone(), extra_info_status, extra_info_headers, @@ -1984,6 +2012,7 @@ enum MainDocumentNavigationProgressEvent { final_url: Url, status: u16, headers: Vec<(String, Vec)>, + mime_type: Option, cookie_set_reports: Vec, extra_info_status: u16, extra_info_headers: Vec<(String, Vec)>, @@ -2208,6 +2237,7 @@ impl MainDocumentNavigationProgressEvent { final_url, status, headers, + mime_type, cookie_set_reports, extra_info_status, extra_info_headers, @@ -2238,6 +2268,7 @@ impl MainDocumentNavigationProgressEvent { from_cache, negotiated_http_version, has_extra_info, + mime_type.as_deref(), ); } } diff --git a/moli-protocol/src/domains/network/main_document_progress/tests.rs b/moli-protocol/src/domains/network/main_document_progress/tests.rs index f9633c91bd6..22f0c738605 100644 --- a/moli-protocol/src/domains/network/main_document_progress/tests.rs +++ b/moli-protocol/src/domains/network/main_document_progress/tests.rs @@ -394,7 +394,9 @@ fn drain_gate_until_body_finished_visible_into_protocol_messages( protocol_messages_from_background_events(events) } -fn response_progress_batch_for_output_target() -> MainDocumentProgressEventBatch { +fn response_progress_batch_for_output_target( + mime_type: Option, +) -> MainDocumentProgressEventBatch { MainDocumentProgressEventBatch::from_events(vec![ MainDocumentNavigationProgressEvent::ResponseReceived { target: MainDocumentProgressEventTarget { @@ -407,6 +409,7 @@ fn response_progress_batch_for_output_target() -> MainDocumentProgressEventBatch final_url: Url::parse("http://example.test/target").unwrap(), status: 200, headers: vec![("Content-Type".to_owned(), b"text/html".to_vec())], + mime_type, cookie_set_reports: Vec::new(), extra_info_status: 200, extra_info_headers: vec![("Content-Type".to_owned(), b"text/html".to_vec())], @@ -435,16 +438,41 @@ fn pending_document_progress_transfer_keeps_body_unmaterialized() { )); } +#[test] +fn response_progress_preserves_computed_mime_separately_from_raw_headers() { + let mut events = Vec::new(); + MainDocumentProgressOutputTarget::background_events(&mut events).emit_batch( + response_progress_batch_for_output_target(Some("image/png".to_owned())), + ); + assert_eq!(events.len(), 2); + for event in events { + let (message, automation) = event.into_parts(); + assert_eq!(message["params"]["response"]["mimeType"], "image/png"); + assert_eq!( + message["params"]["response"]["headers"]["Content-Type"], + "text/html" + ); + let Some(AutomationEvent::NetworkResponseStarted(network)) = automation else { + panic!("response progress must retain its automation event"); + }; + assert_eq!(network.response_mime_type.as_deref(), Some("image/png")); + assert_eq!( + network.response_headers, + vec![("Content-Type".to_owned(), b"text/html".to_vec())] + ); + } +} + #[test] fn progress_output_target_serializes_background_targets_consistently() { let (sender, mut receiver) = unbounded_channel(); let mut background = MainDocumentProgressOutputTarget::background_sender(&sender); - background.emit_batch(response_progress_batch_for_output_target()); + background.emit_batch(response_progress_batch_for_output_target(None)); drop(sender); let mut background_events = Vec::new(); MainDocumentProgressOutputTarget::background_events(&mut background_events) - .emit_batch(response_progress_batch_for_output_target()); + .emit_batch(response_progress_batch_for_output_target(None)); let mut background_parts = Vec::new(); while let Ok(value) = receiver.try_recv() { @@ -598,6 +626,7 @@ fn live_progress_source_serializes_through_progress_emissions() { true, false, None, + None, ); source.emit_body_finished(17); @@ -1379,7 +1408,7 @@ fn live_ready_progress_emissions_reuse_streaming_output_queue() { MainDocumentProgressPhase::ResponseReceived, super::gate::MainDocumentProgressEmission::new( MainDocumentProgressPhase::ResponseReceived, - response_progress_batch_for_output_target(), + response_progress_batch_for_output_target(None), ), ); let out = drain_into_protocol_messages(&mut drain); @@ -1458,7 +1487,7 @@ fn progress_output_queue_buffers_body_finished_until_response_received() { MainDocumentProgressPhase::ResponseReceived, super::gate::MainDocumentProgressEmission::new( MainDocumentProgressPhase::ResponseReceived, - response_progress_batch_for_output_target(), + response_progress_batch_for_output_target(None), ), ); diff --git a/moli-protocol/src/domains/page/termination.rs b/moli-protocol/src/domains/page/termination.rs index c9955947afc..ce0a6d67019 100644 --- a/moli-protocol/src/domains/page/termination.rs +++ b/moli-protocol/src/domains/page/termination.rs @@ -196,6 +196,7 @@ impl PreparedPageStopLoading { // original Native stop waits must not become part of this cleanup. let pending = take_pending_fetch_state_for_owner(conn, owner); let cleanup = crate::domains::fetch::prepare_page_stop_loading(conn, owner, pending); + conn.cancel_uncommitted_navigation_for_owner(owner); // A request waiting for response headers need not be in a Fetch pause // map. Stop that exact pending request in this finite admission turn, // before the Native lifecycle wait can admit a successor navigation. diff --git a/moli-protocol/src/domains/page/tests/navigation/requests.rs b/moli-protocol/src/domains/page/tests/navigation/requests.rs index 85bffeaddd0..5e65f3527cf 100644 --- a/moli-protocol/src/domains/page/tests/navigation/requests.rs +++ b/moli-protocol/src/domains/page/tests/navigation/requests.rs @@ -239,6 +239,48 @@ async fn stop_loading_aborts_paused_request_stage_navigation() { } } +#[tokio::test(flavor = "multi_thread")] +async fn stop_loading_cancels_uncommitted_navigation_outside_the_fetch_pause_registry() { + let mut ctx = TestContext::new(); + load_bc_with_session(&mut ctx, "BID-1", "TID-1", "SID-1", "about:blank"); + let (committed, cancellation) = ctx.with_fixture(|conn| { + let bc = conn.browser_context.as_mut().unwrap(); + let committed = bc + .start_document_navigation_for_active_target("LOADER-committed".to_owned()) + .unwrap(); + bc.commit_document_navigation_if_matches(&committed); + let provisional = bc + .start_document_navigation_for_active_target("LOADER-provisional".to_owned()) + .unwrap(); + let cancellation = bc + .active_page_target() + .runtime_slot + .document_navigation_cancellation_handle(&provisional) + .unwrap(); + assert!(!cancellation.is_cancelled()); + + (committed, cancellation) + }); + + ctx.process_async(json!({ + "id": 159, + "method": "Page.stopLoading", + "sessionId": "SID-1" + })) + .await; + ctx.expect_result(159, json!({}), Some("SID-1")); + assert!(cancellation.is_cancelled()); + ctx.with_fixture(|conn| { + let bc = conn.browser_context.as_ref().unwrap(); + assert!( + !bc.active_page_target() + .runtime_slot + .has_pending_document_navigation() + ); + assert!(bc.accepts_document_body_completion_event(&committed)); + }); +} + #[tokio::test(flavor = "multi_thread")] async fn stop_loading_without_browser_context_returns_empty_result() { let mut ctx = TestContext::new(); diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index 2515c577484..791a43ba3d6 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -1624,7 +1624,7 @@ fn child_document_load_outcome_from_response( let body_bytes = response_body .try_bytes() .map_err(|error| format!("failed to read child document response body: {error}"))?; - let mime = navigation_response_content_type(&head.headers, &body_bytes); + let mime = navigation_response_content_type(&head.final_url, &head.headers, &body_bytes); if !moli_web_mime::is_supported_document_mime_type(&mime) { return Ok(ChildDocumentLoadOutcome::IgnoredNavigation(Box::new( network, diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rs index 012fe8f2ba1..972b3e16325 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs @@ -3113,7 +3113,11 @@ impl PageScriptHost { response.status, false, )?; - let mime = navigation_response_content_type(&head.headers, response.body_bytes()); + let mime = navigation_response_content_type( + &head.final_url, + &head.headers, + response.body_bytes(), + ); if !moli_web_mime::is_supported_document_mime_type(&mime) { return Ok(PopupDocumentLoadOutcome::IgnoredNavigation); } diff --git a/moli-renderer-v8/src/navigation_document.rs b/moli-renderer-v8/src/navigation_document.rs index b39b510db9e..0e7c72e6a6b 100644 --- a/moli-renderer-v8/src/navigation_document.rs +++ b/moli-renderer-v8/src/navigation_document.rs @@ -11,19 +11,16 @@ pub(crate) fn is_image_document_mime(mime: &str) -> bool { moli_web_mime::is_image_mime(mime) && !moli_web_mime::is_xml_document_mime(mime) } -/// An absent HTTP Content-Type is resolved from the response, not its URL's -/// extension. Keep an explicitly supplied type for the Document's metadata. +/// Uses the same MIME decision for complete child responses and main streams. +/// The URL scheme controls sniffing; a path extension does not select a parser. pub(crate) fn navigation_response_content_type( + final_url: &Url, headers: &[(String, Vec)], bytes: &[u8], ) -> String { - moli_web_mime::navigation_response_mime_type_from_headers(headers).unwrap_or_else(|| { - moli_web_mime::computed_response_mime_type( - headers, - moli_web_mime::MimeSniffingContext::Browsing, - bytes, - ) - }) + let mut sniffer = moli_web_mime::NavigationMimeSniffer::new(final_url.scheme(), headers); + sniffer.push(bytes); + sniffer.finish().to_owned() } #[derive(Clone, Debug, PartialEq, Eq)] @@ -153,7 +150,7 @@ mod tests { (&[][..], gif.as_slice(), "image/gif", true), (supplied.as_slice(), html.as_slice(), "image/png", true), ] { - let mime = navigation_response_content_type(headers, bytes); + let mime = navigation_response_content_type(&url, headers, bytes); assert_eq!(mime, expected); let (content, _) = decode_navigation_document_response(bytes, headers, Some(&mime), &url, None); @@ -168,7 +165,11 @@ mod tests { fn unsupplied_navigation_mime_with_nosniff_never_becomes_an_image_document() { let headers = vec![("X-Content-Type-Options".to_owned(), b"nosniff".to_vec())]; let bytes = b"GIF89a\x01\x00\x01\x00"; - let mime = navigation_response_content_type(&headers, bytes); + let mime = navigation_response_content_type( + &Url::parse("https://example.test/picture.gif").unwrap(), + &headers, + bytes, + ); assert_eq!(mime, "text/plain"); let (content, _) = decode_navigation_document_response( bytes, diff --git a/moli-renderer-v8/src/runtime/page.rs b/moli-renderer-v8/src/runtime/page.rs index 5bd8801b997..7e286427c8f 100644 --- a/moli-renderer-v8/src/runtime/page.rs +++ b/moli-renderer-v8/src/runtime/page.rs @@ -564,7 +564,7 @@ impl JsRuntime { response_headers: Vec<(String, Vec)>, loader: &ResourceRequestClient, web_storage: crate::RendererWebStorageHandles, - raw_body: ExternalRawDocumentBodyStream, + mut raw_body: ExternalRawDocumentBodyStream, wpt_extensions_enabled: bool, stage: PageVmInitStage, reply_boundary: crate::RendererReplyBoundary, @@ -574,6 +574,27 @@ impl JsRuntime { lifecycle_decider: Option, options: RendererDocumentOptions, ) -> Result { + let page_target = page_target.into(); + // Callers with an already reserved token transfer cancellation + // ownership before the first await. An existing-Page target is still + // non-owning until MIME classification accepts the response. + let reserved_preparation = match &page_target { + super::RendererDocumentPreparationTarget::NewPage(token) => { + Some(RendererDocumentPreparation { + runtime: self.clone(), + token: *token, + cancel_on_drop: true, + _document_replacement: None, + }) + } + super::RendererDocumentPreparationTarget::ExistingPage(_) => None, + }; + // Resolve outside the owner lane and before reserving a replacement. + // An undecided response must leave the old Document and its scripts + // alive, and dropping this future must still cancel the body producer. + raw_body + .resolve_navigation_mime_type(&final_url, &response_headers) + .await?; let mut request = self .inner .renderer_owner @@ -602,17 +623,19 @@ impl JsRuntime { // dispatching owner work: queuing the wake itself would deadlock behind // a paused script. Downloads and intercepted responses never get here. let (page_reservation, replacement) = page_target - .into() .begin_and_reserve(&self.inner.renderer_owner) .await?; request.document_replacement = replacement.clone(); // Own cancellation before awaiting the prepare reply, so abandoning // the future also retires a queued or already stored preparation. - let preparation = RendererDocumentPreparation { - runtime: self.clone(), - token: page_reservation, - cancel_on_drop: true, - _document_replacement: replacement, + let preparation = match reserved_preparation { + Some(preparation) => preparation, + None => RendererDocumentPreparation { + runtime: self.clone(), + token: page_reservation, + cancel_on_drop: true, + _document_replacement: replacement, + }, }; let reply = self .inner diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs index 0d2f4a0f7ba..84305f9531b 100644 --- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs +++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs @@ -125,7 +125,11 @@ pub(super) async fn load_followed_location_navigation( response.finish().await?; return Ok(LoadedFollowedLocationNavigation::NoDocument); } - if response_headers_indicate_download(&response.headers) { + let headers = response.headers.clone(); + let mime_type = response.sniff_navigation_mime_type(&headers).await?; + if response_headers_indicate_download(&headers) + || !moli_web_mime::is_supported_document_mime_type(&mime_type) + { let mut body = Vec::new(); while let Some(chunk) = response.next_chunk().await { body.extend_from_slice(&chunk); diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index 6a041fd097a..fc502e1f7c7 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -8,6 +8,7 @@ use crate::document_response_decoder::{decode_document_response, new_document_re use crate::types::SubresourceInterceptionClasses; use moli_encoding::DocumentStreamingDecoder; use moli_web_mime::response_headers_indicate_attachment_download; +use std::collections::VecDeque; use tokio::sync::{mpsc, oneshot}; pub(in crate::runtime) struct StreamingHtmlPageCreationResult { @@ -90,81 +91,24 @@ impl ConcurrentParseTimeRuntime { ) -> Result { let response_status = response.status; let response_headers = response.headers.clone(); - debug_assert!(!response_headers_indicate_download(&response_headers)); - let response_final_url = response.final_url.clone(); - let mut env = env.clone(); - env.apply_navigation_response_headers(&response_final_url, &response_headers); - - let mut body_source = RawDocumentBodySource::fetch_response(response); - let (mut state, mut decoder) = - response_document_parser(response_final_url, &response_headers, &env); - state - .buffered_document_preloads - .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( - &env, - )); - state - .buffered_document_preloads - .set_response_csp_requires_parser_admission( - !env.document_policy_container - .response_content_security_policies - .is_empty(), - ); - state - .buffered_document_preloads - .set_response_referrer_policy(env.document_policy_container.referrer_policy.clone()); - state.buffered_document_preloads.bind_resource_runtime( - runtime_hooks.owner_wake(), - runtime_hooks.resource_task_runner(), - ); - let service_worker_preload_context = - env.reserved_service_worker_client_id.map(|client_id| { - ServiceWorkerScriptPreloadContext::new( - runtime_hooks.browser_context_runtime.clone(), - client_id, - state.final_url.clone(), - runtime_hooks.owner_wake(), - ) - }); - state.service_worker_preload_context = service_worker_preload_context.clone(); - // Raw navigation bodies are decoded during prebootstrap scan. The decoder - // is carried forward so split multibyte sequences are not decoded twice or - // lost between the scan and parser handoff. - let prebootstrap_chunks = prebootstrap_scan_ready_streaming_raw_chunks( - &mut state, - &mut body_source, - &mut decoder, - loader, - service_worker_preload_context.as_ref(), - ); - let bootstrap_outcome = Self::start_creation_from_streaming_html_bootstrap_with_state( + let final_url = response.final_url.clone(); + let content_type = response.navigation_mime_type().map(str::to_owned); + Self::create_from_committed_raw_document_response( page_id, local_executor, loader, - &env, + env, runtime_hooks, - state, stage, started, - ) - .await?; - let outcome = settle_streaming_raw_response_at_boundary( - bootstrap_outcome, - body_source, - prebootstrap_chunks, - decoder, - service_worker_preload_context, - started, + final_url, + response_status, + response_headers, + RawDocumentBodySource::fetch_response(response), + content_type, boundary, ) - .await?; - Ok(StreamingNavigationPageCreationResult::Html(Box::new( - StreamingHtmlPageCreationResult { - response_status, - response_headers, - outcome, - }, - ))) + .await } pub(in crate::runtime) async fn create_external_raw_document_response_at_reply_boundary( @@ -296,19 +240,57 @@ impl ConcurrentParseTimeRuntime { raw_body: ExternalRawDocumentBodyStream, boundary: CommittedNavigationBootstrapBoundary, ) -> Result { - debug_assert!(!response_headers_indicate_download(&response_headers)); let runtime_hooks = match raw_body.page_creation_progress() { Some(progress) => runtime_hooks.with_page_creation_progress(progress), None => runtime_hooks, }; + let content_type = raw_body.navigation_mime_type.clone(); + Self::create_from_committed_raw_document_response( + page_id, + local_executor, + loader, + env, + runtime_hooks, + stage, + started, + final_url, + response_status, + response_headers, + RawDocumentBodySource::External(raw_body), + content_type, + boundary, + ) + .await + } + + async fn create_from_committed_raw_document_response( + page_id: PageId, + local_executor: JsLocalExecutor, + loader: &ResourceRequestClient, + env: &PageVmEnvConfig, + runtime_hooks: PageVmRuntimeHooks, + stage: PageVmInitStage, + started: Instant, + final_url: Url, + response_status: u16, + response_headers: Vec<(String, Vec)>, + mut body_source: RawDocumentBodySource, + content_type: Option, + boundary: CommittedNavigationBootstrapBoundary, + ) -> Result { + debug_assert!(!response_headers_indicate_download(&response_headers)); let mut env = env.clone(); env.apply_navigation_response_headers(&final_url, &response_headers); - let mut body_source = RawDocumentBodySource::External(raw_body); - if response_headers_indicate_xml_document(&response_headers) { + let content_type = content_type.or_else(|| { + moli_web_mime::navigation_response_mime_type_from_headers(&response_headers) + }); + if content_type + .as_deref() + .is_some_and(moli_web_mime::is_xml_document_mime) + { let body = collect_streaming_raw_body(&mut body_source).await?; - let content_type = moli_web_mime::response_document_content_type(&response_headers) - .unwrap_or_else(|| "application/xml".to_owned()); + let content_type = content_type.expect("checked XML MIME type"); let (source, character_set) = decode_document_response( &body, &response_headers, @@ -341,7 +323,12 @@ impl ConcurrentParseTimeRuntime { }, ))); } - let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env); + let (mut state, mut decoder) = response_document_parser_with_mime( + final_url, + &response_headers, + &env, + content_type.as_deref(), + ); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -488,15 +475,12 @@ impl ConcurrentParseTimeRuntime { } } -fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { - moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime)) -} - const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8; pub struct ExternalRawDocumentBodyStream { body_chunks: mpsc::Receiver>, + buffered_body_chunks: VecDeque>, + navigation_mime_type: Option, completion: Option>>, page_creation_progress: Option, } @@ -523,6 +507,8 @@ impl ExternalRawDocumentBodyStream { body_tx, Self { body_chunks, + buffered_body_chunks: VecDeque::new(), + navigation_mime_type: None, completion: Some(completion), page_creation_progress: Some(page_creation_progress), }, @@ -535,6 +521,8 @@ impl ExternalRawDocumentBodyStream { ) -> Self { Self { body_chunks, + buffered_body_chunks: VecDeque::new(), + navigation_mime_type: None, completion: Some(completion), page_creation_progress: None, } @@ -549,8 +537,70 @@ impl ExternalRawDocumentBodyStream { body_stream } + pub fn with_navigation_mime_type(mut self, mime_type: String) -> Self { + self.navigation_mime_type = Some(mime_type); + self + } + + pub(crate) async fn resolve_navigation_mime_type( + &mut self, + final_url: &Url, + headers: &[(String, Vec)], + ) -> Result<()> { + if self.navigation_mime_type.is_some() { + return Ok(()); + } + let mut sniffer = moli_web_mime::NavigationMimeSniffer::new(final_url.scheme(), headers); + for chunk in &self.buffered_body_chunks { + sniffer.push(chunk); + } + loop { + if let Some(mime_type) = sniffer.mime_type() { + self.navigation_mime_type = Some(mime_type.to_owned()); + return Ok(()); + } + match self.body_chunks.recv().await { + Some(chunk) => { + sniffer.push(&chunk); + if !chunk.is_empty() { + self.buffered_body_chunks.push_back(chunk); + } + } + None => { + self.finish_transfer().await?; + self.navigation_mime_type = Some(sniffer.finish().to_owned()); + return Ok(()); + } + } + } + } + + async fn next_chunk(&mut self) -> Option> { + match self.buffered_body_chunks.pop_front() { + Some(chunk) => Some(chunk), + None => self.body_chunks.recv().await, + } + } + + fn try_next_chunk(&mut self) -> Option> { + self.buffered_body_chunks + .pop_front() + .or_else(|| self.body_chunks.try_recv().ok()) + } + + async fn finish_transfer(&mut self) -> Result<()> { + let Some(completion) = self.completion.as_mut() else { + return Ok(()); + }; + let result = completion.await; + self.completion = None; + result.map_err(|_| anyhow!("external raw document body completion channel closed"))? + } + fn body_chunk_stream_is_exhausted(&self) -> bool { - self.body_chunks.is_closed() && self.body_chunks.is_empty() + self.buffered_body_chunks.is_empty() + && self.body_chunks.is_closed() + && self.body_chunks.is_empty() } fn page_creation_progress(&self) -> Option { @@ -571,7 +621,7 @@ impl RawDocumentBodySource { pub(super) async fn next_chunk(&mut self) -> Option> { match self { Self::FetchResponse(response) => response.next_chunk().await, - Self::External(source) => source.body_chunks.recv().await, + Self::External(source) => source.next_chunk().await, } } @@ -581,7 +631,7 @@ impl RawDocumentBodySource { // from the response source into the parser loop. match self { Self::FetchResponse(response) => response.try_next_chunk(), - Self::External(source) => source.body_chunks.try_recv().ok(), + Self::External(source) => source.try_next_chunk(), } } @@ -595,12 +645,7 @@ impl RawDocumentBodySource { pub(super) async fn finish(&mut self) -> Result<()> { match self { Self::FetchResponse(response) => response.finish().await, - Self::External(source) => source - .completion - .take() - .expect("external raw document body completion should only be awaited once") - .await - .map_err(|_| anyhow!("external raw document body completion channel closed"))?, + Self::External(source) => source.finish_transfer().await, } } } @@ -628,20 +673,27 @@ fn scan_prebootstrap_html_chunk_into_state( ); } +#[cfg(test)] fn response_document_parser( final_url: Url, headers: &[(String, Vec)], env: &PageVmEnvConfig, ) -> (ParseTimeDriverState, DocumentStreamingDecoder) { let content_type = moli_web_mime::navigation_response_mime_type_from_headers(headers); - let text_type = content_type - .as_deref() - .filter(|mime| moli_web_mime::is_text_document_mime(mime)); + response_document_parser_with_mime(final_url, headers, env, content_type.as_deref()) +} + +fn response_document_parser_with_mime( + final_url: Url, + headers: &[(String, Vec)], + env: &PageVmEnvConfig, + content_type: Option<&str>, +) -> (ParseTimeDriverState, DocumentStreamingDecoder) { + let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)); let mut state = if let Some(mime) = text_type { ParseTimeDriverState::new_text(final_url, mime) - } else if let Some(mime) = content_type - .as_deref() - .filter(|mime| crate::navigation_document::is_image_document_mime(mime)) + } else if let Some(mime) = + content_type.filter(|mime| crate::navigation_document::is_image_document_mime(mime)) { ParseTimeDriverState::new_image(final_url, mime, headers) } else { @@ -650,8 +702,7 @@ fn response_document_parser( main_document_parser_scripting_enabled(env), ) }; - let decoder = - new_document_response_decoder(headers, content_type.as_deref(), &state.final_url, None); + let decoder = new_document_response_decoder(headers, content_type, &state.final_url, None); state.response_links = crate::response_links::DocumentResponseLinks::parse(&state.final_url, headers); // Headers/defaults are observable even when no body chunk is ready yet. diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs index c60a0a5b17f..3dd31abafd3 100644 --- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs +++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs @@ -2936,3 +2936,147 @@ async fn live_page_replacement_stage_reply_drains_large_body_and_finishes_script producer.await.unwrap(); page.close_async().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn sniffed_plain_navigation_replays_every_byte_without_executing_late_markup() { + let runtime = JsRuntime::initialize(); + let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap(); + let source = format!( + "ordinary text {}", + "a".repeat(1600) + ); + let prepared = prepare_test_external_raw_document_with_content_type( + &runtime, + &loader, + url::Url::parse("https://example.test/mime/plain").unwrap(), + "", + ExternalRawDocumentBodyStream::from_bytes(source.as_bytes().to_vec()), + ) + .await; + let permit = prepared.issue_commit_permit(); + let (mut page, state, _, _, download) = prepared.commit(permit).await.unwrap(); + assert!(download.is_none()); + assert_eq!(state.document_content_type, "text/plain"); + assert_eq!( + state.headers, + vec![("content-type".to_owned(), Vec::::new())] + ); + let expression = format!( + "JSON.stringify([document.contentType,typeof sneaked,document.body.textContent==={}])", + serde_json::to_string(&source).unwrap(), + ); + let (reply, _) = page + .run_async_command(RendererPageCommand::EvaluateExpression { + expression, + await_promise: false, + }) + .await + .unwrap(); + assert_eq!( + renderer_json_value(reply), + Some(serde_json::json!("[\"text/plain\",\"undefined\",true]")) + ); + page.close_async().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn undecided_navigation_mime_keeps_the_old_document_live_on_cancel_and_failure() { + for fail in [false, true] { + let runtime = JsRuntime::initialize(); + let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap(); + let mut page = create_test_html_page( + &runtime, &loader, + url::Url::parse("https://example.test/mime/original").unwrap(), + "

original

", + ).await; + let testing = RendererPageTestingHandle::new_for_testing(&page); + let before = testing.renderer_page_view_async().await.unwrap(); + let accounting = runtime.document_isolate_accounting_for_diagnostics(); + let (completion_tx, completion_rx) = oneshot::channel(); + let (body_tx, raw_body) = ExternalRawDocumentBodyStream::channel(completion_rx); + body_tx.send(b"\x89PNG\r\n\x1a".to_vec()).await.unwrap(); + let target = page.document_replacement_target(moli_fetch::FetchCancelHandle::new()); + let url = url::Url::parse("https://example.test/mime/undecided").unwrap(); + let mut prepare = Box::pin(runtime.prepare_streaming_raw_document_from_external_body( + target, + url.clone(), + url, + None, + false, + 0, + Vec::new(), + 200, + Vec::new(), + &loader, + crate::RendererWebStorageHandles::ephemeral(), + raw_body, + false, + PageVmInitStage::Load, + RendererReplyBoundary::DocumentCommit, + RendererTopLevelNavigationDispatch::DelegateToBrowser, + RendererNavigationReplyPolicy::ReturnWithPendingNavigation, + None, + None, + crate::RendererDocumentOptions::default(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + tokio::select! { + _ = &mut prepare => panic!("an incomplete PNG signature must not prepare a Document"), + _ = tokio::task::yield_now() => {}, + } + if body_tx.capacity() == body_tx.max_capacity() { + break; + } + } + }).await.unwrap(); + let (reply, _) = page.run_async_command(RendererPageCommand::EvaluateExpression { + expression: "JSON.stringify([retained, document.querySelector('p').textContent, ++oldCounter])".to_owned(), + await_promise: false, + }).await.unwrap(); + assert_eq!( + renderer_json_value(reply), + Some(serde_json::json!("[17,\"original\",1]")) + ); + assert_eq!( + runtime.document_isolate_accounting_for_diagnostics(), + accounting + ); + if fail { + drop(body_tx); + completion_tx + .send(Err(anyhow::anyhow!("sniff transport failed"))) + .unwrap(); + let error = prepare + .await + .err() + .expect("failed sniff must reject preparation"); + assert!( + error.to_string().contains("sniff transport failed"), + "{error:#}" + ); + } else { + drop(prepare); + assert!( + body_tx.is_closed(), + "cancelled sniff must release the producer" + ); + assert!(completion_tx.send(Ok(())).is_err()); + } + let after = testing.renderer_page_view_async().await.unwrap(); + assert_eq!(after.vm_creation_id, before.vm_creation_id); + assert_eq!( + runtime.document_isolate_accounting_for_diagnostics(), + accounting + ); + let (reply, _) = page + .run_async_command(RendererPageCommand::EvaluateExpression { + expression: "String(++oldCounter)".to_owned(), + await_promise: false, + }) + .await + .unwrap(); + assert_eq!(renderer_json_value(reply), Some(serde_json::json!("2"))); + page.close_async().await.unwrap(); + } +} diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index 45fa80a62c9..848f67f3ad2 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -5,6 +5,7 @@ mod file; mod headers; mod media; mod media_capabilities; +mod navigation; mod parse; mod resource; mod response_policy; @@ -35,6 +36,7 @@ pub use headers::{ }; pub use media::{MediaMimeSupport, is_media_source_type_supported, media_mime_support}; pub use media_capabilities::{MediaKind, is_valid_media_capabilities_mime_type}; +pub use navigation::{NAVIGATION_MIME_PREFIX_LIMIT, NavigationMimeSniffer}; pub use parse::{ mime_charset, mime_essence, mime_parameter, normalize_web_api_mime_type, parse_mime, parse_mime_type, request_header_content_type_essence, diff --git a/moli-web-mime/src/navigation.rs b/moli-web-mime/src/navigation.rs new file mode 100644 index 00000000000..0cb23c0752a --- /dev/null +++ b/moli-web-mime/src/navigation.rs @@ -0,0 +1,298 @@ +use crate::{ + determine_nosniff, extract_response_mime_essence, + response_headers_indicate_attachment_download, sniff_archive_mime_type, + sniff_audio_video_mime_type, sniff_image_mime_type, sniff_text_or_binary, +}; + +pub const NAVIGATION_MIME_PREFIX_LIMIT: usize = 1024; +const HTML_PREFIX_LIMIT: usize = 512; +const MAGIC_PREFIX_LIMIT: usize = 42; + +/// Resolves a navigation MIME type before a new Document is committed. +/// +/// A positive signature can settle the decision before EOF. A provisional +/// text/binary default must not choose a parser while more prefix bytes can +/// still change it. The retained prefix is bounded independently of chunk size. +#[derive(Debug)] +pub struct NavigationMimeSniffer { + mime_type: Option, + sniff_scriptable: bool, + prefix: Vec, +} + +impl NavigationMimeSniffer { + pub fn new(scheme: &str, headers: &[(String, Vec)]) -> Self { + let supplied = extract_response_mime_essence(headers); + let unknown = supplied + .as_deref() + .is_none_or(|mime| matches!(mime, "unknown/unknown" | "application/unknown" | "*/*")); + let no_sniff = determine_nosniff(headers); + let needs_prefix = matches!(scheme, "http" | "https" | "file" | "filesystem") + && !no_sniff + && !response_headers_indicate_attachment_download(headers) + && (unknown || supplied.as_deref() == Some("text/plain")); + Self { + mime_type: if needs_prefix { + None + } else { + Some(supplied.unwrap_or_else(|| { + if no_sniff || scheme == "blob" { + "text/plain".to_owned() + } else { + "text/html".to_owned() + } + })) + }, + sniff_scriptable: unknown, + prefix: Vec::new(), + } + } + + pub fn mime_type(&self) -> Option<&str> { + self.mime_type.as_deref() + } + + pub fn push(&mut self, bytes: &[u8]) { + if self.mime_type.is_some() { + return; + } + let count = (NAVIGATION_MIME_PREFIX_LIMIT - self.prefix.len()).min(bytes.len()); + self.prefix.extend_from_slice(&bytes[..count]); + let header = self.prefix.as_slice(); + if self.sniff_scriptable + && let Some(mime) = + sniff_navigation_scriptable(&header[..header.len().min(HTML_PREFIX_LIMIT)]) + { + self.mime_type = Some(mime.to_owned()); + return; + } + + let binary = sniff_text_or_binary(header) == "application/octet-stream"; + // A supplied plain-text type cannot be promoted to scriptable content. + // Its binary check must also precede image signatures such as GIF/BMP, + // whose shortest identifying prefix contains only printable bytes. + if self.sniff_scriptable || binary { + if let Some(mime) = sniff_image_mime_type(header) + .or_else(|| sniff_audio_video_mime_type(header)) + .or_else(|| sniff_archive_mime_type(header)) + { + self.mime_type = Some(mime.to_owned()); + return; + } + if header.starts_with(b"%PDF-") { + self.mime_type = Some("application/pdf".to_owned()); + return; + } + if header.starts_with(b"%!PS-Adobe-") { + self.mime_type = Some("application/postscript".to_owned()); + return; + } + if header.starts_with(b"MZ") { + self.mime_type = Some("application/octet-stream".to_owned()); + return; + } + } + let binary_limit = if self.sniff_scriptable { + HTML_PREFIX_LIMIT + } else { + MAGIC_PREFIX_LIMIT + }; + if header.len() == NAVIGATION_MIME_PREFIX_LIMIT || (binary && header.len() >= binary_limit) + { + self.mime_type = Some(sniff_text_or_binary(header).to_owned()); + } + } + + pub fn finish(&mut self) -> &str { + self.mime_type + .get_or_insert_with(|| sniff_text_or_binary(&self.prefix).to_owned()) + } +} + +fn sniff_navigation_scriptable(header: &[u8]) -> Option<&'static str> { + let header = header.trim_ascii_start(); + // Navigation sniffing accepts complete signature prefixes. This includes + // an HTML comment with no following space, unlike the general MIME parser. + const HTML_SIGNATURES: &[&[u8]] = &[ + b" Vec<(String, Vec)> { + vec![("Content-Type".to_owned(), mime.as_bytes().to_vec())] + } + + #[test] + fn navigation_signatures_survive_every_chunk_boundary() { + for (bytes, mime) in [ + (b"\x89PNG\r\n\x1a\n".as_slice(), "image/png"), + (b"GIF89a", "image/gif"), + (b"\xff\xd8\xff", "image/jpeg"), + (b"BM", "image/bmp"), + (b"RIFF1234WEBPVP", "image/webp"), + (b" later bytes cannot replace the decision"); + assert_eq!(sniffer.finish(), mime); + } + } + } + } + + #[test] + fn incomplete_signatures_do_not_commit_a_provisional_default() { + let mut sniffer = NavigationMimeSniffer::new("https", &[]); + for byte in b"\x89PNG\r\n\x1a" { + sniffer.push(&[*byte]); + assert_eq!(sniffer.mime_type(), None); + } + sniffer.push(b"\n"); + assert_eq!(sniffer.mime_type(), Some("image/png")); + let mut truncated = NavigationMimeSniffer::new("https", &[]); + truncated.push(b"\x89PNG\r\n\x1a"); + assert_eq!(truncated.finish(), "application/octet-stream"); + } + + #[test] + fn plain_text_can_become_an_image_but_never_scriptable_html() { + for mime in [ + "text/plain", + "text/plain; charset=gbk", + "TEXT/PLAIN;charset=UTF-8", + ] { + let mut image = NavigationMimeSniffer::new("http", &headers(mime)); + assert_eq!(image.mime_type(), None); + image.push(b"\x89PNG\r\n\x1a\n"); + assert_eq!(image.mime_type(), Some("image/png")); + let mut text = NavigationMimeSniffer::new("http", &headers(mime)); + text.push(b""); + assert_eq!(text.mime_type(), None); + assert_eq!(text.finish(), "text/plain"); + let mut binary = NavigationMimeSniffer::new("http", &headers(mime)); + binary.push(b"text\0"); + assert_eq!(binary.finish(), "application/octet-stream"); + let mut no_sniff = headers(mime); + no_sniff.push(("X-Content-Type-Options".to_owned(), b"nosniff".to_vec())); + let mut literal = NavigationMimeSniffer::new("http", &no_sniff); + literal.push(b"text\0"); + assert_eq!(literal.finish(), "text/plain"); + } + let mut gif = NavigationMimeSniffer::new("http", &headers("text/plain")); + gif.push(b"GIF89a"); + assert_eq!(gif.mime_type(), None); + gif.push(b"\x01\x00"); + assert_eq!(gif.mime_type(), Some("image/gif")); + } + + #[test] + fn authoritative_headers_do_not_wait_for_response_bytes() { + for supplied in [None, Some("not a MIME"), Some("text/plain")] { + let headers = supplied.map(headers).unwrap_or_default(); + let mut sniffer = NavigationMimeSniffer::new("blob", &headers); + assert_eq!(sniffer.mime_type(), Some("text/plain")); + sniffer.push(b"\x89PNG\r\n\x1a\n"); + assert_eq!(sniffer.finish(), "text/plain"); + } + for mime in [ + "text/html", + "application/xml", + "image/svg+xml", + "image/png", + "application/octet-stream", + "application/json", + ] { + let mut sniffer = NavigationMimeSniffer::new("https", &headers(mime)); + assert_eq!(sniffer.mime_type(), Some(mime)); + sniffer.push(b"GIF89a"); + assert_eq!(sniffer.finish(), mime); + } + for supplied in [ + None, + Some("not a MIME"), + Some("text/plain; charset=gbk"), + Some("unknown/unknown"), + ] { + let mut headers = supplied.map(headers).unwrap_or_default(); + headers.push(("X-Content-Type-Options".to_owned(), b"nosniff".to_vec())); + let expected = if supplied == Some("unknown/unknown") { + "unknown/unknown" + } else { + "text/plain" + }; + let mut sniffer = NavigationMimeSniffer::new("https", &headers); + assert_eq!(sniffer.mime_type(), Some(expected)); + sniffer.push(b"\x89PNG\r\n\x1a\n"); + assert_eq!(sniffer.finish(), expected); + } + } + + #[test] + fn provisional_text_is_bounded_and_cannot_execute_later_markup() { + let mut sniffer = NavigationMimeSniffer::new("https", &[]); + sniffer.push(&vec![b'a'; NAVIGATION_MIME_PREFIX_LIMIT - 1]); + assert_eq!(sniffer.mime_type(), None); + sniffer.push(b"a"); + assert_eq!(sniffer.mime_type(), Some("text/plain")); + assert_eq!(sniffer.prefix.len(), NAVIGATION_MIME_PREFIX_LIMIT); + let mut short = NavigationMimeSniffer::new("https", &[]); + short.push(b"short text"); + assert_eq!(short.mime_type(), None); + assert_eq!(short.finish(), "text/plain"); + assert_eq!( + NavigationMimeSniffer::new("https", &[]).finish(), + "text/plain" + ); + } +} diff --git a/moli-web-mime/src/response_policy.rs b/moli-web-mime/src/response_policy.rs index 9f4ee2a0de1..a19ee74260a 100644 --- a/moli-web-mime/src/response_policy.rs +++ b/moli-web-mime/src/response_policy.rs @@ -24,11 +24,12 @@ pub fn determine_nosniff(headers: &[(String, Vec)]) -> bool { }) } -/// Resolves a navigation's MIME type when its headers are sufficient. +/// Supplies navigation MIME metadata before a body decision is available. /// -/// A supplied type is authoritative. Without one, `nosniff` makes the response -/// a plain-text document; otherwise the caller still needs to inspect its body. -/// This navigation default does not apply to image or other subresources. +/// Without a supplied type, `nosniff` makes the response a plain-text document. +/// Use [`crate::NavigationMimeSniffer`] to select a parser: supplied unknown and +/// plain-text types can still require a body prefix. This navigation default +/// does not apply to image or other subresources. pub fn navigation_response_mime_type_from_headers(headers: &[(String, Vec)]) -> Option { extract_response_mime_essence(headers) .or_else(|| determine_nosniff(headers).then(|| "text/plain".to_owned()))