From cf79aa002dcd78adfd91e937e0878dcea4fa5b1d Mon Sep 17 00:00:00 2001 From: ldm0 Date: Wed, 4 Nov 2026 23:32:27 +0800 Subject: [PATCH] refactor(navigation): share document request execution Reuse NavigationInterception for main, child-frame and auxiliary document requests, replacing the protocol-only PreparedNavigationRequest wrapper. Reevaluate main-document Fetch interception per redirect and share per-session hop identities. Preserve Service Worker clients through response handling, publish source retirement before waiting for headers, and keep transport failures on the existing error-document path. --- moli-core/src/network/mod.rs | 10 +- moli-core/src/runtime/mod.rs | 18 +- moli-core/src/runtime/navigation_engine.rs | 44 +-- moli-protocol/src/conn.rs | 4 +- .../src/conn/browser_context/fetch_owner.rs | 54 +-- .../browser_context/target_session_owner.rs | 7 +- moli-protocol/src/conn/dispatch_tests/bidi.rs | 5 + moli-protocol/src/conn/fetch_support.rs | 69 +++- .../src/conn/page_state/fetch_state.rs | 1 + moli-protocol/src/conn/runtime_load.rs | 80 ++--- .../conn/runtime_load/captured_response.rs | 77 +++- moli-protocol/src/conn/state/fetch.rs | 39 +- .../conn/state/fetch/tests/supersession.rs | 4 + .../src/conn/state/navigation_outcome.rs | 1 + moli-protocol/src/conn/state/runtime_slot.rs | 20 +- moli-protocol/src/conn/state/tests.rs | 5 + moli-protocol/src/conn/state_access.rs | 14 + moli-protocol/src/conn/tests/mod.rs | 1 + .../src/conn/tests/navigation_error.rs | 11 +- .../src/conn/tests/resource_runtime.rs | 1 + .../src/domains/activity/main_document.rs | 1 + moli-protocol/src/domains/fetch.rs | 5 +- moli-protocol/src/domains/fetch/auth.rs | 8 + moli-protocol/src/domains/fetch/commands.rs | 18 +- .../src/domains/fetch/document_navigation.rs | 28 +- .../src/domains/fetch/document_request.rs | 308 ++++++++++++++-- moli-protocol/src/domains/fetch/helpers.rs | 7 +- moli-protocol/src/domains/fetch/navigation.rs | 63 ++-- .../domains/fetch/navigation_interception.rs | 107 ++++++ .../src/domains/fetch/subresource.rs | 40 +-- .../src/domains/fetch/tests/basics.rs | 18 + .../src/domains/fetch/tests/header_bytes.rs | 28 ++ .../src/domains/fetch/tests/runtime_fetch.rs | 3 + .../tests/runtime_fetch/unified_navigation.rs | 332 ++++++++++++++++++ moli-protocol/src/domains/network/agent.rs | 6 +- .../network/main_document_progress/mod.rs | 110 +++--- .../network/main_document_progress/tests.rs | 2 + .../src/domains/network/tests/cache.rs | 1 + .../domains/network/tests/response_body.rs | 2 + moli-protocol/src/domains/page/navigation.rs | 23 +- .../runtime/tests/evaluate/navigation.rs | 1 + .../context_host/child_documents/loads.rs | 128 +++---- .../src/native_bridge/context_host/popups.rs | 16 +- .../src/network/navigation/interception.rs | 161 +++++++++ .../network/navigation/interception/tests.rs | 104 ++++++ .../src/network/navigation/loader.rs | 40 ++- .../src/network/navigation/mod.rs | 2 + 47 files changed, 1609 insertions(+), 418 deletions(-) create mode 100644 moli-protocol/src/domains/fetch/navigation_interception.rs create mode 100644 moli-protocol/src/domains/fetch/tests/runtime_fetch/unified_navigation.rs create mode 100644 moli-renderer-v8/src/network/navigation/interception.rs create mode 100644 moli-renderer-v8/src/network/navigation/interception/tests.rs diff --git a/moli-core/src/network/mod.rs b/moli-core/src/network/mod.rs index 852ab9df9d5..044497d5c53 100644 --- a/moli-core/src/network/mod.rs +++ b/moli-core/src/network/mod.rs @@ -1,12 +1,12 @@ pub use moli_renderer_v8::network::{ - BrowserResourceRuntime, BrowserResourceRuntimeOwner, ResourceRequestClient, - SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation, WebStorageMutationRecord, - WebStorageMutationSubscription, WebStorageString, + BrowserResourceRuntime, BrowserResourceRuntimeOwner, RendererResourceTaskRunner, + ResourceRequestClient, SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation, + WebStorageMutationRecord, WebStorageMutationSubscription, WebStorageString, context::{DocumentResourceLoader, DocumentResourceLoaderDiagnostics}, deep_clone_shared_web_storage_store, navigation::{ - DocumentFetchContextSeed, NavigationResourceLoader, NavigationResourceLoaderDiagnostics, - NavigationResourceLoaderState, + DocumentFetchContextSeed, NavigationInterception, NavigationResourceLoader, + NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState, }, new_shared_web_storage_store, web_storage_partitioned_area_key, }; diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs index 7daa7137d91..0d0b14d2b01 100644 --- a/moli-core/src/runtime/mod.rs +++ b/moli-core/src/runtime/mod.rs @@ -1458,11 +1458,23 @@ fn external_raw_document_body_from_streaming_response_with_observer_and_progress body_stream } -fn streaming_raw_response_from_navigation_response( - response: NavigationResponse, +pub fn streaming_raw_response_from_navigation_response( + mut response: NavigationResponse, ) -> Result { let head = response.head(); - let body = response.clone_body_bytes(); + let body = response.take_body_bytes(); + streaming_raw_response_from_raw_response(moli_fetch::RawResponse::from_head_and_body( + head, body, + )) +} + +pub fn streaming_raw_response_from_raw_response( + response: moli_fetch::RawResponse, +) -> Result { + let (head, body) = response.into_body(); + let body = body + .try_into_materialized_bytes() + .map_err(|_| anyhow!("navigation response body must be materialized"))?; let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel(); if !body.is_empty() { body_tx diff --git a/moli-core/src/runtime/navigation_engine.rs b/moli-core/src/runtime/navigation_engine.rs index 84c34036e87..65c56596c75 100644 --- a/moli-core/src/runtime/navigation_engine.rs +++ b/moli-core/src/runtime/navigation_engine.rs @@ -74,28 +74,6 @@ pub enum CommittedDocumentResourceSource { Synthetic, } -fn streaming_raw_response_from_navigation_response( - response: NavigationResponse, -) -> Result { - let head = response.head(); - let body = response.clone_body_bytes(); - let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel(); - if !body.is_empty() { - body_tx - .send(body) - .map_err(|_| anyhow!("failed to enqueue service worker main resource body"))?; - } - drop(body_tx); - let (completion_tx, completion_rx) = tokio::sync::oneshot::channel(); - let _ = completion_tx.send(Ok(())); - Ok(StreamingRawResponse::new_with_head( - head, - body_rx, - FetchCancelHandle::new(), - completion_rx, - )) -} - #[derive(Clone)] pub struct NavigationResourceStorageHandles { cookie_store: SharedBrowserCookieStore, @@ -1207,10 +1185,13 @@ impl NavigationEngine { request.url.clone(), FetchCancelHandle::new(), )?; - navigation_loader - .fetch_with_network_metadata(request) - .await - .map(|result| result.map_response(NavigationResponse::from)) + moli_renderer_v8::network::navigation::NavigationInterception::new( + navigation_loader, + request, + ) + .fetch_text() + .await + .map(|result| result.map_response(NavigationResponse::from)) } pub async fn fetch_navigation_response_with_storage_async( @@ -1301,15 +1282,18 @@ impl NavigationEngine { let final_url = response.final_url.clone(); return Ok(NavigationStreamingRawResponse { fetch_result: NetworkFetchResult::without_request_observation( - streaming_raw_response_from_navigation_response(response)?, + super::streaming_raw_response_from_navigation_response(response)?, ), reserved_service_worker_client: reserved_client, document_fetch_context_seed: navigation_loader.commit(final_url)?, }); } - let fetch_result = navigation_loader - .fetch_raw_stream_with_network_metadata(request) - .await?; + let fetch_result = moli_renderer_v8::network::navigation::NavigationInterception::new( + navigation_loader.clone(), + request, + ) + .fetch_raw_stream() + .await?; let final_url = fetch_result.response().final_url.clone(); Ok(NavigationStreamingRawResponse { fetch_result, diff --git a/moli-protocol/src/conn.rs b/moli-protocol/src/conn.rs index d232ab48053..496e31ddf73 100644 --- a/moli-protocol/src/conn.rs +++ b/moli-protocol/src/conn.rs @@ -683,7 +683,7 @@ pub(crate) use runtime_load::{ BackgroundNavigationLoadJob, CapturedPageReplacement, CommittedNavigationPage, CompletedInitialDocumentPageBuild, FailedInitialDocumentPageBuild, InitialDocumentPageInstallResult, InitialDocumentPageOwner, PendingInitialDocumentPageBuild, - PreparedCapturedNavigationResponse, PreparedNavigationLoadError, PreparedNavigationRequest, + PreparedCapturedNavigationResponse, PreparedNavigationLoadError, PreparedStreamingNavigationResponse, ResponseCommitReady, StablePageNavigationCommitTarget, }; use scheduler_hooks::CdpSchedulerHooks; @@ -4271,3 +4271,5 @@ pub struct PermissionOverride { #[cfg(test)] mod tests; + +pub(crate) use state::TargetFetchSubresourceInterceptionSnapshot; diff --git a/moli-protocol/src/conn/browser_context/fetch_owner.rs b/moli-protocol/src/conn/browser_context/fetch_owner.rs index 5c0f3dff71c..8ec8d9a6942 100644 --- a/moli-protocol/src/conn/browser_context/fetch_owner.rs +++ b/moli-protocol/src/conn/browser_context/fetch_owner.rs @@ -576,25 +576,6 @@ impl CdpConnection { }) } - pub(crate) fn register_pending_fetch_response_navigation_for_owner( - &mut self, - owner: &CommandOwnerScope, - request_id: String, - document_navigation_token: Option, - navigation: crate::conn::NavigationDispatchState, - body: crate::conn::DocumentBodySource, - ) -> bool { - self.target_session_owner_mut_for_owner(owner) - .is_some_and(|mut owner| { - owner.register_pending_fetch_response_navigation( - request_id, - document_navigation_token, - navigation, - body, - ) - }) - } - pub(crate) fn take_fetch_response_transfer_for_cancellation_for_owner( &mut self, owner: &CommandOwnerScope, @@ -1430,34 +1411,6 @@ impl TargetSessionOwnerMut<'_> { true } - fn register_pending_fetch_response_navigation( - &mut self, - request_id: String, - document_navigation_token: Option, - navigation: crate::conn::NavigationDispatchState, - body: crate::conn::DocumentBodySource, - ) -> bool { - if let Some(token) = document_navigation_token.as_ref() - && let crate::conn::DocumentBodySource::StreamingRaw { response, .. } = &body - { - // Navigation cancellation also covers a response already handed - // to a body reader or a continuation outside the Fetch owner. - self.runtime_slot_mut() - .page_slot_mut() - .add_document_navigation_cancellation(token, response.cancellation_handle()); - } - let Some(mut owner) = self.pending_fetch_owner_mut() else { - return false; - }; - owner.register_pending_fetch_response_navigation( - request_id, - document_navigation_token, - navigation, - body, - ); - true - } - fn take_fetch_response_transfer_for_cancellation( &mut self, request_id: &str, @@ -1479,6 +1432,13 @@ impl TargetSessionOwnerMut<'_> { request_id: String, transfer: PausedDocumentTransfer, ) -> bool { + if let Some(token) = transfer.navigation_token() + && let Some(cancellation) = transfer.cancellation_handle() + { + self.runtime_slot_mut() + .page_slot_mut() + .add_document_navigation_cancellation(token, cancellation); + } let Some(mut owner) = self.pending_fetch_owner_mut() else { return false; }; diff --git a/moli-protocol/src/conn/browser_context/target_session_owner.rs b/moli-protocol/src/conn/browser_context/target_session_owner.rs index d7c7388c34e..1979c8dc581 100644 --- a/moli-protocol/src/conn/browser_context/target_session_owner.rs +++ b/moli-protocol/src/conn/browser_context/target_session_owner.rs @@ -1105,7 +1105,12 @@ impl<'a> TargetSessionOwnerMut<'a> { .map(|_| FetchRequestStage::Response) }) .or_else(|| { - document_fetch_response_stage_candidate.then_some(FetchRequestStage::Response) + (document_fetch_response_stage_candidate + || (!is_data_url + && fetch_snapshot.has_request_stage_candidate( + DevToolsNetworkResourceType::Document, + ))) + .then_some(FetchRequestStage::Response) }); let document_fetch_event_session_id = document_request_pause .as_ref() diff --git a/moli-protocol/src/conn/dispatch_tests/bidi.rs b/moli-protocol/src/conn/dispatch_tests/bidi.rs index 0b9d32fa20e..cb4fe360dce 100644 --- a/moli-protocol/src/conn/dispatch_tests/bidi.rs +++ b/moli-protocol/src/conn/dispatch_tests/bidi.rs @@ -197,6 +197,10 @@ async fn bidi_fetch_control_resolves_background_request_owner() { conn.register_pending_fetch_navigation_request_for_owner( &crate::conn::CommandOwnerScope::for_session("SID-background"), PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: "FETCH-background".to_owned(), interception_session_id: Some("bidi-session-1".to_owned()), @@ -216,6 +220,7 @@ async fn bidi_fetch_control_resolves_background_request_owner() { request_id: Some("NETWORK-background".to_owned()), loader_id: "LOADER-background".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: url::Url::parse("https://example.test/background").unwrap(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/conn/fetch_support.rs b/moli-protocol/src/conn/fetch_support.rs index 2e5f9a90b0c..cbbba1acfad 100644 --- a/moli-protocol/src/conn/fetch_support.rs +++ b/moli-protocol/src/conn/fetch_support.rs @@ -222,8 +222,16 @@ impl PendingSubresourceFetchOwnerKind { } } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct PendingFetchNavigation { + pub(crate) reserved_service_worker_client: + Option, + pub(crate) interception: Option<( + moli_core::network::NavigationInterception, + moli_fetch::NetworkObservationJournal, + )>, + pub(crate) remaining_request_stages: Vec, + pub(crate) redirected_request_id: Option, /// Authentication restart budget for this logical navigation, independent of pause IDs. pub(crate) auth_retry_budget: moli_fetch::AuthRetryBudget, pub fetch_request_id: String, @@ -300,6 +308,7 @@ impl PendingFetchAuthNavigation { /// Protocol keeps navigation ownership even while a task exclusively reads the body. #[derive(Debug)] pub struct PausedDocumentTransfer { + reserved_service_worker_client: Option, fetch_request_id: String, document_navigation_token: Option, navigation: NavigationDispatchState, @@ -510,6 +519,7 @@ impl PausedDocumentTransfers { self.transfers.insert(request_id, transfer); } + #[cfg(test)] pub(crate) fn register_pending_navigation( &mut self, request_id: String, @@ -737,6 +747,8 @@ impl PendingDocumentBodyRead { } pub(crate) struct PendingStreamingDocumentResponseNavigation { + pub(crate) reserved_service_worker_client: + Option, pub(crate) document_navigation_token: DocumentNavigationToken, pub(crate) navigation: NavigationDispatchState, pub(crate) response: StreamingRawResponse, @@ -745,6 +757,32 @@ pub(crate) struct PendingStreamingDocumentResponseNavigation { } impl PausedDocumentTransfer { + pub(crate) fn with_service_worker_client( + mut self, + client: Option, + ) -> Self { + self.reserved_service_worker_client = client; + self + } + + pub(crate) fn cancellation_handle(&self) -> Option { + match &self.state { + PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) => { + match body.as_ref() { + DocumentBodySource::StreamingRaw { response, .. } => { + Some(response.cancellation_handle()) + } + _ => None, + } + } + PausedDocumentTransferState::Available(PausedDocumentBody::ActiveBodyStream { + stream, + .. + }) => Some(stream.response.cancellation_handle()), + PausedDocumentTransferState::Reading(reading) => reading.cancellation.clone(), + } + } + pub(crate) fn navigation_token(&self) -> Option<&DocumentNavigationToken> { self.document_navigation_token.as_ref() } @@ -757,7 +795,7 @@ impl PausedDocumentTransfer { // Otherwise the same owned preparation accepts captured or streaming bodies. #[allow(clippy::result_large_err)] pub(crate) fn prepare_response( - self, + mut self, conn: &mut CdpConnection, response_code: Option, response_headers: Vec<(String, Vec)>, @@ -813,7 +851,12 @@ impl PausedDocumentTransfer { body_progress_source, response_code, response_headers, - ), + ) + .map(|response| { + response.with_service_worker_client( + self.reserved_service_worker_client.take(), + ) + }), )), ); } @@ -844,11 +887,17 @@ impl PausedDocumentTransfer { progress, ) }; - Ok(crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new(response))) + Ok( + crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new( + response.map(|response| { + response.with_service_worker_client(self.reserved_service_worker_client.take()) + }), + )), + ) } pub(crate) fn prepare_synthetic_response( - &self, + mut self, conn: &mut CdpConnection, response_code: u16, response_headers: Vec<(String, Vec)>, @@ -883,6 +932,9 @@ impl PausedDocumentTransfer { NetworkObservationJournal::default(), body_progress_source, ) + .map(|response| { + response.with_service_worker_client(self.reserved_service_worker_client.take()) + }) } pub(crate) fn into_navigation( @@ -899,6 +951,7 @@ impl PausedDocumentTransfer { body: DocumentBodySource, ) -> Self { Self { + reserved_service_worker_client: None, fetch_request_id, document_navigation_token, navigation, @@ -936,6 +989,7 @@ impl PausedDocumentTransfer { handle: String, ) -> Result> { let Self { + reserved_service_worker_client, fetch_request_id, document_navigation_token, navigation, @@ -944,6 +998,7 @@ impl PausedDocumentTransfer { let PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) = state else { return Err(Box::new(Self { + reserved_service_worker_client, fetch_request_id, document_navigation_token, navigation, @@ -985,6 +1040,7 @@ impl PausedDocumentTransfer { handle, buffered_body, transfer: Self { + reserved_service_worker_client, fetch_request_id, document_navigation_token, navigation, @@ -997,6 +1053,7 @@ impl PausedDocumentTransfer { self, ) -> Result> { let Self { + reserved_service_worker_client, fetch_request_id, document_navigation_token, navigation, @@ -1009,6 +1066,7 @@ impl PausedDocumentTransfer { ) if matches!(body.as_ref(), DocumentBodySource::StreamingRaw { .. }) => (token, body), (document_navigation_token, state) => { return Err(Box::new(Self { + reserved_service_worker_client, fetch_request_id, document_navigation_token, navigation, @@ -1026,6 +1084,7 @@ impl PausedDocumentTransfer { unreachable!("the response was checked before transferring it") }; Ok(PendingStreamingDocumentResponseNavigation { + reserved_service_worker_client, document_navigation_token, navigation, response, diff --git a/moli-protocol/src/conn/page_state/fetch_state.rs b/moli-protocol/src/conn/page_state/fetch_state.rs index a03b3ff1144..f17afe7a775 100644 --- a/moli-protocol/src/conn/page_state/fetch_state.rs +++ b/moli-protocol/src/conn/page_state/fetch_state.rs @@ -380,6 +380,7 @@ mod tests { request_id: Some("REQ-1".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: url.clone(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index e65910b2eb2..16d71e8adb6 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -903,52 +903,6 @@ struct BackgroundNavigationClaim { cancellation: FetchCancelHandle, } -/// Transport inputs captured while the navigation owner is available. Waiting -/// for response headers owns neither a connection nor a renderer Page. -pub(crate) struct PreparedNavigationRequest { - client: moli_core::network::ResourceRequestClient, - request: Request, - cancellation: FetchCancelHandle, -} - -struct CancelNavigationRequestOnDrop(Option); - -impl Drop for CancelNavigationRequestOnDrop { - fn drop(&mut self) { - if let Some(cancellation) = self.0.take() { - cancellation.cancel(); - } - } -} - -impl PreparedNavigationRequest { - pub(crate) async fn wait(self) -> Result> { - let mut cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone())); - let url = self.request.url.clone(); - let response = self - .client - .fetch_raw_stream_with_cancel_and_network_metadata(self.request, self.cancellation) - .await - .with_context(|| format!("failed to fetch page `{url}`")); - if response.is_ok() { - // The streaming response now owns transport cancellation. - cancellation.0.take(); - } - response - } - - pub(crate) async fn wait_buffered(self) -> Result> { - let _cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone())); - let url = self.request.url.clone(); - // libcurl owns Digest's intermediate challenges. The exact navigation - // cancellation reaches that transport as well as this waiting future. - self.client - .fetch_raw_with_cancel_and_network_metadata(self.request, self.cancellation) - .await - .with_context(|| format!("failed to fetch page `{url}`")) - } -} - pub(crate) struct BackgroundNavigationLoadJob { engine: NavigationEngine, page_reservation: RendererDocumentPreparationTarget, @@ -969,6 +923,7 @@ pub(crate) struct BackgroundNavigationLoadJob { } pub(crate) struct PreparedStreamingNavigationResponse { + reserved_service_worker_client: Option, engine: NavigationEngine, page_reservation: RendererDocumentPreparationTarget, load_inputs: TargetNavigationLoadInputs, @@ -1284,6 +1239,14 @@ impl BackgroundNavigationLoadJob { } impl PreparedStreamingNavigationResponse { + pub(crate) fn with_service_worker_client( + mut self, + client: Option, + ) -> Self { + self.reserved_service_worker_client = client; + self + } + pub(crate) async fn run( mut self, body_completion_sink: Option, @@ -1307,7 +1270,7 @@ impl PreparedStreamingNavigationResponse { self.response_headers_override, self.body_progress_source, body_completion_sink, - None, + self.reserved_service_worker_client, CommittedDocumentResourceSource::Synthetic, self.reply_boundary, ) @@ -2717,6 +2680,7 @@ impl CdpConnection { let (response, network_observation_journal) = response.into_parts_with_observation_journal(); Ok(PreparedStreamingNavigationResponse { + reserved_service_worker_client: None, engine, page_reservation, load_inputs, @@ -2756,6 +2720,7 @@ impl CdpConnection { let page_reservation = self.reserve_renderer_page_for_owner(&navigation.owner, &load_inputs, &engine); Ok(PreparedStreamingNavigationResponse { + reserved_service_worker_client: None, engine, page_reservation, load_inputs, @@ -3225,7 +3190,7 @@ impl CdpConnection { token: &DocumentNavigationToken, navigation: &NavigationDispatchState, auth: Option, - ) -> Result { + ) -> Result { let load_inputs = self.navigation_load_inputs_for_navigation(Some(token), navigation)?; let cancellation = self .runtime_session_owner_slot_for_owner(&navigation.owner) @@ -3243,11 +3208,15 @@ impl CdpConnection { let client = self .ensure_resource_request_client_for_navigation_load_inputs(&load_inputs)? .frozen_request_client(); - Ok(PreparedNavigationRequest { + let loader = moli_core::network::NavigationResourceLoader::new_with_cancel_handle( client, - request, + request.url.clone(), + moli_core::network::RendererResourceTaskRunner::from_current_tokio()?, cancellation, - }) + ); + Ok(moli_core::network::NavigationInterception::new( + loader, request, + )) } async fn fetch_navigation_streaming_raw_response_with_load_inputs_async( @@ -3271,8 +3240,13 @@ impl CdpConnection { let loader = self .ensure_resource_request_client_for_navigation_load_inputs(load_inputs)? .clone(); - loader - .fetch_raw_stream_with_cancel_and_network_metadata(request, FetchCancelHandle::new()) + let navigation = moli_core::network::NavigationResourceLoader::new( + loader, + request.url.clone(), + moli_core::network::RendererResourceTaskRunner::from_current_tokio()?, + ); + moli_core::network::NavigationInterception::new(navigation, request) + .fetch_raw_stream() .await .with_context(|| format!("failed to fetch page `{raw_url}`")) } diff --git a/moli-protocol/src/conn/runtime_load/captured_response.rs b/moli-protocol/src/conn/runtime_load/captured_response.rs index ed3722fbb15..5534b9e0e1e 100644 --- a/moli-protocol/src/conn/runtime_load/captured_response.rs +++ b/moli-protocol/src/conn/runtime_load/captured_response.rs @@ -5,6 +5,7 @@ use super::*; /// A captured response's parser inputs and exact renderer reservation. Taking /// this value ends protocol state access; preparation owns all of its waits. pub(crate) struct PreparedCapturedNavigationResponse { + reserved_service_worker_client: Option, engine: NavigationEngine, page_reservation: Option, load_inputs: TargetNavigationLoadInputs, @@ -45,6 +46,14 @@ impl PreparedNavigationLoadError { } impl PreparedCapturedNavigationResponse { + pub(crate) fn with_service_worker_client( + mut self, + client: Option, + ) -> Self { + self.reserved_service_worker_client = client; + self + } + pub(crate) async fn run(mut self) -> Result { if super::super::downloads::response_headers_indicate_download(&self.head.headers) { complete_auxiliary_download_response(&self.load_inputs, self.head.clone()); @@ -71,6 +80,34 @@ impl PreparedCapturedNavigationResponse { &self.body_progress_source, )); } + if let Some(client) = self.reserved_service_worker_client.take() { + let body = self + .body + .process_in_worker(|body| body.materialize_bytes()) + .await?; + let response = moli_core::runtime::streaming_raw_response_from_raw_response( + RawResponse::from_head_and_body(self.head, body), + )?; + return build_navigation_from_streaming_raw_response_with_engine_async( + &mut self.engine, + self.page_reservation + .expect("a Document response must reserve its renderer Page"), + &self.load_inputs, + self.requested_url, + self.request_method, + self.request_headers, + response, + self.network_observation_journal, + None, + Vec::new(), + self.body_progress_source, + None, + Some(client), + CommittedDocumentResourceSource::Synthetic, + RendererReplyBoundary::Stage, + ) + .await; + } prepare_navigation_from_captured_raw_response_with_engine_async( &mut self.engine, self.page_reservation @@ -99,9 +136,34 @@ impl CdpConnection { navigation: &NavigationDispatchState, error: anyhow::Error, ) -> Result { - let Some(network_error) = error.downcast_ref::() else { - return Err(error); - }; + let (unreachable_url, request_method, request_headers, error_text) = + if let Some(network_error) = error.downcast_ref::() { + ( + network_error.unreachable_url.clone(), + network_error.request_method.clone(), + network_error.request_headers.clone(), + network_error.kind.error_text().to_owned(), + ) + } else if !moli_fetch::is_fetch_cancelled(&error) + && let Some(failure) = error.downcast_ref::() + { + match failure.request_context() { + Some(request) => ( + request.current_url().clone(), + request.request_method().to_owned(), + request.request_headers().clone(), + failure.network_error_text().to_owned(), + ), + None => ( + navigation.requested_url.clone(), + navigation.request_method.clone(), + navigation.request_headers.clone(), + failure.network_error_text().to_owned(), + ), + } + } else { + return Err(error); + }; let load_inputs = self.navigation_load_inputs_for_navigation(token, navigation)?; let engine = self.navigation_engine_handle_for_load_inputs(&load_inputs); let page_reservation = @@ -110,10 +172,10 @@ impl CdpConnection { engine, page_reservation, load_inputs, - unreachable_url: network_error.unreachable_url.clone(), - request_method: network_error.request_method.clone(), - request_headers: network_error.request_headers.clone(), - error_text: network_error.kind.error_text().to_owned(), + unreachable_url, + request_method, + request_headers, + error_text, }) } @@ -201,6 +263,7 @@ impl CdpConnection { && !response_status_has_no_content(head.status)) .then(|| self.reserve_renderer_page_for_owner(owner, &load_inputs, &engine)); PreparedCapturedNavigationResponse { + reserved_service_worker_client: None, engine, page_reservation, load_inputs, diff --git a/moli-protocol/src/conn/state/fetch.rs b/moli-protocol/src/conn/state/fetch.rs index a5fa3a3d903..c16c30cf767 100644 --- a/moli-protocol/src/conn/state/fetch.rs +++ b/moli-protocol/src/conn/state/fetch.rs @@ -4,14 +4,13 @@ use serde_json::{Value, json}; use url::Url; use super::super::fetch_support::{ - ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, DocumentBodySource, - InFlightSubresourceFetchRequest, PausedDocumentTransfer, PausedDocumentTransfers, - PendingDocumentBodyRead, PendingFetchAuthNavigation, PendingFetchNavigation, - PendingFetchResponseBodyStreamRead, PendingFetchResponseBodyStreamReadStart, - PendingSubresourceFetchAuthRequest, PendingSubresourceFetchOwnerKind, - PendingSubresourceFetchRequest, PendingSubresourceFetchResponseRequest, - ResponseStageUrlMatchPolicy, fetch_subresource_interception_config_for_patterns, - matching_fetch_pattern, + ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, InFlightSubresourceFetchRequest, + PausedDocumentTransfer, PausedDocumentTransfers, PendingDocumentBodyRead, + PendingFetchAuthNavigation, PendingFetchNavigation, PendingFetchResponseBodyStreamRead, + PendingFetchResponseBodyStreamReadStart, PendingSubresourceFetchAuthRequest, + PendingSubresourceFetchOwnerKind, PendingSubresourceFetchRequest, + PendingSubresourceFetchResponseRequest, ResponseStageUrlMatchPolicy, + fetch_subresource_interception_config_for_patterns, matching_fetch_pattern, }; pub(super) use super::super::fetch_support::{ FetchInterceptionPattern, FetchRequestStage, FetchResourceTypeFilter, @@ -20,6 +19,8 @@ use super::DocumentNavigationToken; use super::navigation_outcome::NavigationDispatchState; use super::runtime_slot::TargetRuntimeSlot; use crate::automation::{DevToolsNetworkInterceptId, DevToolsNetworkResourceType}; +#[cfg(test)] +use crate::conn::DocumentBodySource; use moli_fetch::url_pattern_matches; #[derive(Debug, Default)] @@ -293,6 +294,12 @@ impl TargetFetchState { } fn prune_request_stage_chains_for_session(&mut self, session_id: Option<&str>) { + for pending in self.pending_fetch_navigations.values_mut() { + pending.remaining_request_stages.retain(|stage| { + !stage.owner_kind.drains_on_fetch_disable() + || stage.session_id.as_deref() != session_id + }); + } for pending in self.pending_subresource_fetches.values_mut() { if let Some(chain) = pending.request_stage_chain.as_mut() { chain.remaining_sessions.retain(|stage| { @@ -561,6 +568,7 @@ impl TargetFetchState { .insert(request_id, pending); } + #[cfg(test)] pub(crate) fn register_pending_fetch_response_navigation( &mut self, request_id: String, @@ -1726,6 +1734,19 @@ impl TargetFetchSubresourceInterceptionSnapshot { matching_auth_required_network_intercepts(&self.network_intercepts, url) } + pub(crate) fn has_request_stage_candidate( + &self, + resource_type: DevToolsNetworkResourceType, + ) -> bool { + self.enabled + && self.patterns.iter().any(|pattern| { + pattern.request_stage == FetchRequestStage::Request + && pattern + .resource_type_filter + .is_none_or(|filter| filter.matches_resource_type(resource_type)) + }) + } + pub(crate) fn has_response_stage_candidate( &self, resource_type: DevToolsNetworkResourceType, @@ -1978,6 +1999,7 @@ impl TargetFetchOwner { .register_pending_fetch_auth_navigation(request_id, pending); } + #[cfg(test)] pub(crate) fn register_pending_fetch_response_navigation( &mut self, request_id: String, @@ -2548,6 +2570,7 @@ mod tests { request_id: Some(format!("NETWORK-{request_id}")), loader_id: "LOADER-1".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: test_url("auth-page"), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/conn/state/fetch/tests/supersession.rs b/moli-protocol/src/conn/state/fetch/tests/supersession.rs index e1ef063051f..9b3588514bf 100644 --- a/moli-protocol/src/conn/state/fetch/tests/supersession.rs +++ b/moli-protocol/src/conn/state/fetch/tests/supersession.rs @@ -11,6 +11,10 @@ fn navigation(token: &DocumentNavigationToken, id: &str) -> NavigationDispatchSt fn request(token: &DocumentNavigationToken, id: &str) -> PendingFetchNavigation { PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: id.to_owned(), interception_session_id: Some("SID-A".to_owned()), diff --git a/moli-protocol/src/conn/state/navigation_outcome.rs b/moli-protocol/src/conn/state/navigation_outcome.rs index f6ca0c85c76..e695c86c01f 100644 --- a/moli-protocol/src/conn/state/navigation_outcome.rs +++ b/moli-protocol/src/conn/state/navigation_outcome.rs @@ -374,6 +374,7 @@ pub struct NavigationDispatchState { pub request_id: Option, pub loader_id: String, pub request_announced: bool, + pub(crate) announced_redirects: usize, pub requested_url: Url, pub request_method: String, /// Text projection used by CDP request events and Fetch interception. diff --git a/moli-protocol/src/conn/state/runtime_slot.rs b/moli-protocol/src/conn/state/runtime_slot.rs index 99dd276acd4..48ea07cf7d9 100644 --- a/moli-protocol/src/conn/state/runtime_slot.rs +++ b/moli-protocol/src/conn/state/runtime_slot.rs @@ -441,6 +441,10 @@ impl TargetRuntimeSlot { target_id: String, loader_id: String, ) -> DocumentNavigationToken { + if let Some(previous) = self.current_document_loader_id().map(str::to_owned) { + self.network_agent + .clear_fetch_pause_announced_request_id(&previous); + } self.devtools_renderer_channel.reopen_after_target_crash(); let token = self .page_slot @@ -929,14 +933,14 @@ impl TargetRuntimeSlot { .record_fetch_pause_announced_request_id(request_id); } - pub(crate) fn replace_child_document_fetch_request_id( + pub(crate) fn replace_navigation_fetch_request_id( &mut self, network_id: String, session_id: Option, fetch_id: String, ) -> Option { self.network_agent - .replace_child_document_fetch_request_id(network_id, session_id, fetch_id) + .replace_navigation_fetch_request_id(network_id, session_id, fetch_id) } pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool { @@ -975,14 +979,22 @@ impl TargetRuntimeSlot { &mut self, token: &DocumentNavigationToken, ) -> bool { - self.page_slot - .commit_pending_document_navigation_if_matches(token) + let committed = self + .page_slot + .commit_pending_document_navigation_if_matches(token); + if committed { + self.network_agent + .clear_fetch_pause_announced_request_id(&token.loader_id); + } + committed } pub(crate) fn clear_pending_document_navigation_if_loader_matches( &mut self, loader_id: &str, ) -> bool { + self.network_agent + .clear_fetch_pause_announced_request_id(loader_id); self.page_slot .clear_pending_document_navigation_if_loader_matches(loader_id) } diff --git a/moli-protocol/src/conn/state/tests.rs b/moli-protocol/src/conn/state/tests.rs index 05036f08670..8b84abfed30 100644 --- a/moli-protocol/src/conn/state/tests.rs +++ b/moli-protocol/src/conn/state/tests.rs @@ -41,6 +41,7 @@ fn test_navigation_dispatch_state(fetch_request_id: &str) -> NavigationDispatchS request_id: Some(format!("NETWORK-{fetch_request_id}")), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("https://example.test/").unwrap(), request_method: "GET".to_owned(), request_body: None, @@ -812,6 +813,10 @@ fn target_fetch_owner_reports_matching_auth_required_network_intercepts() { fn pending_navigation_rejects_generic_request_action_without_consuming_id() { let mut state = TargetFetchState::default(); state.register_pending_fetch_navigation_request(PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: "FETCH-NAV".to_owned(), interception_session_id: Some("SID-fetch".to_owned()), diff --git a/moli-protocol/src/conn/state_access.rs b/moli-protocol/src/conn/state_access.rs index a733987dcb6..6b8dbb4088a 100644 --- a/moli-protocol/src/conn/state_access.rs +++ b/moli-protocol/src/conn/state_access.rs @@ -322,6 +322,20 @@ impl ProtocolWorkflowState<'_> { .prepare_navigation_load_error(Some(&token), &navigation, error) } + pub(crate) fn prepare_fetch_navigation_redirect( + self, + pending: super::PendingFetchNavigation, + executor: moli_core::network::NavigationInterception, + prior: moli_fetch::NetworkObservationJournal, + ) -> crate::domains::fetch::RedirectNavigation { + crate::domains::fetch::prepare_navigation_redirect( + self.connection, + pending, + executor, + prior, + ) + } + pub(crate) fn accept_fetch_navigation_response( self, pending: super::PendingFetchNavigation, diff --git a/moli-protocol/src/conn/tests/mod.rs b/moli-protocol/src/conn/tests/mod.rs index d10a64b83b2..dd8b36b7f9c 100644 --- a/moli-protocol/src/conn/tests/mod.rs +++ b/moli-protocol/src/conn/tests/mod.rs @@ -1904,6 +1904,7 @@ fn materialized_navigation_test_state( request_id: Some(loader_id.to_owned()), loader_id: loader_id.to_owned(), request_announced: true, + announced_redirects: 0, requested_url: Url::parse(requested_url).unwrap(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/conn/tests/navigation_error.rs b/moli-protocol/src/conn/tests/navigation_error.rs index 200713a88ec..065504c7052 100644 --- a/moli-protocol/src/conn/tests/navigation_error.rs +++ b/moli-protocol/src/conn/tests/navigation_error.rs @@ -234,7 +234,7 @@ async fn prepared_navigation_request_freezes_policy_and_cancels_before_headers_o ctx.process_async(json!({"id": 11, "sessionId": "SID-1", "method": "Network.setExtraHTTPHeaders", "params": {"headers": {"X-Snapshot": "after"}}})).await; ctx.expect_result(11, json!({}), Some("SID-1")); let result = tokio::time::timeout(std::time::Duration::from_secs(30), async { - let (result, ()) = tokio::join!(job.wait(), async { + let (result, ()) = tokio::join!(job.fetch_raw_stream(), async { requested_rx.await.unwrap(); ctx.with_fixture(|conn| { conn.start_document_navigation_for_owner(&navigation.owner, "replacement".into()) @@ -269,7 +269,7 @@ async fn dropping_prepared_navigation_header_wait_cancels_its_transport() { .unwrap() }); tokio::time::timeout(std::time::Duration::from_secs(30), async { - let mut wait = Box::pin(job.wait()); + let mut wait = Box::pin(job.fetch_raw_stream()); tokio::select! { result = &mut wait => panic!("headers must stay held: {result:?}"), requested = requested_rx => requested.unwrap(), @@ -580,6 +580,7 @@ pub(super) fn navigation_fixture() -> ( request_id: Some("LID-test".to_owned()), loader_id: "LID-test".to_owned(), request_announced: true, + announced_redirects: 0, requested_url: Url::parse("https://original.example/document").unwrap(), request_method: "GET".to_owned(), request_body: None, @@ -1014,14 +1015,14 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context( let buffered_error = async { let request = ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?; - request.wait_buffered().await + request.fetch_buffered().await } .await .expect_err("offline response fetch must fail"); let streaming_error = async { let request = ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?; - request.wait().await + request.fetch_raw_stream().await } .await .expect_err("offline streaming response fetch must fail"); @@ -1038,7 +1039,7 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context( }), ) })?; - request.wait_buffered().await + request.fetch_buffered().await } .await .expect_err("offline authenticated response fetch must fail"); diff --git a/moli-protocol/src/conn/tests/resource_runtime.rs b/moli-protocol/src/conn/tests/resource_runtime.rs index 950816f5337..f90b19ca4d2 100644 --- a/moli-protocol/src/conn/tests/resource_runtime.rs +++ b/moli-protocol/src/conn/tests/resource_runtime.rs @@ -119,6 +119,7 @@ async fn buffered_navigation_for_inactive_session_retains_its_target_engine() { request_id: Some("LOADER-target".to_owned()), loader_id: "LOADER-target".to_owned(), request_announced: true, + announced_redirects: 0, requested_url: requested_url.clone(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/domains/activity/main_document.rs b/moli-protocol/src/domains/activity/main_document.rs index f61f25777f5..1a0a41de3b0 100644 --- a/moli-protocol/src/domains/activity/main_document.rs +++ b/moli-protocol/src/domains/activity/main_document.rs @@ -780,6 +780,7 @@ mod tests { request_id: Some("REQ-1".to_owned()), loader_id: "LID-1".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("https://example.test/start").unwrap(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/domains/fetch.rs b/moli-protocol/src/domains/fetch.rs index 67a5ef80374..e541c656362 100644 --- a/moli-protocol/src/domains/fetch.rs +++ b/moli-protocol/src/domains/fetch.rs @@ -10,6 +10,8 @@ mod document_navigation; mod document_request; mod helpers; mod navigation; +mod navigation_interception; +pub(crate) use navigation_interception::prepare_navigation_request_pause; mod params; mod patterns; mod state; @@ -43,7 +45,8 @@ pub(crate) use crate::conn::PendingFetchAuthNavigation; #[cfg(test)] pub(crate) use crate::conn::PendingFetchNavigation; pub(crate) use document_request::{ - FetchNavigationResponse, PreparedFetchNavigationResponse, accept_navigation_response, + FetchNavigationResponse, PreparedFetchNavigationResponse, RedirectNavigation, + accept_navigation_response, prepare_navigation_redirect, }; #[cfg(test)] pub(crate) use helpers::encode_basic_auth; diff --git a/moli-protocol/src/domains/fetch/auth.rs b/moli-protocol/src/domains/fetch/auth.rs index a611ac631d4..7049a2c638b 100644 --- a/moli-protocol/src/domains/fetch/auth.rs +++ b/moli-protocol/src/domains/fetch/auth.rs @@ -51,6 +51,10 @@ pub(super) fn navigation_for_auth_retry( pending: crate::conn::PendingFetchAuthNavigation, ) -> PendingFetchNavigation { PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: pending.auth_retry_budget, fetch_request_id: pending.response_stage_request_id, interception_session_id: pending.interception_session_id, @@ -75,6 +79,10 @@ pub(super) fn canceled_auth_response( }; ( PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: pending.auth_retry_budget, fetch_request_id: pending.response_stage_request_id, interception_session_id: pending.interception_session_id, diff --git a/moli-protocol/src/domains/fetch/commands.rs b/moli-protocol/src/domains/fetch/commands.rs index c3b819db7da..a712dbd0107 100644 --- a/moli-protocol/src/domains/fetch/commands.rs +++ b/moli-protocol/src/domains/fetch/commands.rs @@ -403,6 +403,21 @@ fn start_devtools_continue_intercepted_request_command( pending.navigation.request_load_policy, None, ); + if !pending.remaining_request_stages.is_empty() { + let stage = pending.remaining_request_stages.remove(0); + pending.fetch_request_id = stage.request_id; + pending.interception_session_id = stage.session_id; + pending.redirected_request_id = stage.redirected_request_id; + let event = super::request_paused_background_event( + pending.interception_session_id.as_deref(), + &pending, + ); + let navigation_owner = pending.navigation.owner.clone(); + conn.register_pending_fetch_navigation_request_for_owner(&navigation_owner, pending); + let mut plan = CommandOutputPlan::success(); + plan.push_background_event(event); + return FetchCommandTaskStep::Complete(plan); + } let termination = super::navigation::NavigationTermination::new( conn, owner, @@ -1560,6 +1575,7 @@ pub(super) fn continue_streaming_document_response_in_background( response_headers: Vec<(String, Vec)>, ) { let PendingStreamingDocumentResponseNavigation { + reserved_service_worker_client, document_navigation_token, navigation, response, @@ -1575,7 +1591,7 @@ pub(super) fn continue_streaming_document_response_in_background( response_headers, body_progress_source, ) { - Ok(job) => job, + Ok(job) => job.with_service_worker_client(reserved_service_worker_client), Err(BackgroundNavigationClaimError::AlreadyClaimed) => return, Err(BackgroundNavigationClaimError::Stale) => { let _ = sender.send(page::BackgroundNavigationCompletion::new( diff --git a/moli-protocol/src/domains/fetch/document_navigation.rs b/moli-protocol/src/domains/fetch/document_navigation.rs index aaecfa8461e..9f571a745c1 100644 --- a/moli-protocol/src/domains/fetch/document_navigation.rs +++ b/moli-protocol/src/domains/fetch/document_navigation.rs @@ -498,17 +498,23 @@ pub(super) fn start_fulfill( response_code, response_headers, decoded_body, - } => conn.prepare_synthetic_navigation_response( - Some(&token), - &navigation, - navigation.requested_url.clone(), - response_code, - response_headers, - crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body(decoded_body), - pending.request_cookie_report, - Default::default(), - network::MainDocumentBodyProgressSource::default(), - ), + } => { + let client = pending.reserved_service_worker_client; + conn.prepare_synthetic_navigation_response( + Some(&token), + &navigation, + navigation.requested_url.clone(), + response_code, + response_headers, + crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body( + decoded_body, + ), + pending.request_cookie_report, + Default::default(), + network::MainDocumentBodyProgressSource::default(), + ) + .map(|response| response.with_service_worker_client(client)) + } DocumentFulfillment::ResponseTransfer { transfer, response_code, diff --git a/moli-protocol/src/domains/fetch/document_request.rs b/moli-protocol/src/domains/fetch/document_request.rs index 02bf2bc5863..c7b70fc5062 100644 --- a/moli-protocol/src/domains/fetch/document_request.rs +++ b/moli-protocol/src/domains/fetch/document_request.rs @@ -1,13 +1,14 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use crate::conn::{ CdpConnection, NavigationLoadOutcome, PendingFetchNavigation, PendingStreamingDocumentResponseNavigation, PreparedCapturedNavigationResponse, - PreparedNavigationRequest, ProtocolWorkflowDriver, + ProtocolWorkflowDriver, }; use crate::domains::{ command_output::CommandOutputBuffer, command_output::CommandOutputPlan, network, }; +use moli_core::network::NavigationInterception; use moli_core::page::SubresourceAuthCredentials; use moli_fetch::{ NetworkFetchResult, NetworkObservationJournal, RawResponse, StreamingRawResponse, @@ -23,7 +24,8 @@ use super::{ /// awaits; only interpreting the accepted response needs another state turn. pub(crate) struct PreparedInterceptedNavigationRequest { pending: PendingFetchNavigation, - request: Option>, + request: Option>, + renderer_runtime: Option, buffered: bool, handle_auth: bool, prior_journal: Option, @@ -62,6 +64,11 @@ impl PreparedInterceptedNavigationRequest { &pending.navigation.requested_url, ); let buffered = !navigation::navigation_response_stage_auth_can_stream(auth.as_ref()); + let mut prior_journal = prior_journal; + let renderer_runtime = conn + .target_owner_identity_for_owner(&pending.navigation.owner) + .and_then(|(id, _)| conn.browser_context_by_id(&id)) + .map(|context| context.renderer_runtime()); let request = if auth.is_some() && !pending.auth_retry_budget.try_restart() { None } else if handle_auth && buffered && pending.intercept_response { @@ -73,7 +80,24 @@ impl PreparedInterceptedNavigationRequest { .into())) } else { network::record_main_document_request_body(conn, &pending.navigation); - Some( + let prepared = if let Some((mut interception, journal)) = pending.interception.take() { + prior_journal = Some(navigation::merge_network_observations( + prior_journal, + journal, + )); + let request = interception.request_mut(); + request.url = pending.navigation.requested_url.clone(); + request.method = pending.navigation.request_method.clone(); + request.request_headers = pending.navigation.request_headers.clone(); + request.body = pending.navigation.clone_request_body_bytes(); + *request = request + .clone() + .with_redirect_headers(pending.navigation.redirect_headers.clone()); + if let Some(auth) = auth { + request.set_auth(Some(auth.into())); + } + Ok(interception) + } else { conn.prepare_navigation_request( pending .document_navigation_token @@ -81,12 +105,26 @@ impl PreparedInterceptedNavigationRequest { .expect("request token"), &pending.navigation, auth, - ), - ) + ) + }; + Some(prepared.map(|mut prepared| { + let intercept = conn + .target_fetch_subresource_interception_snapshot_for_owner( + &pending.navigation.owner, + ) + .is_some_and(|snapshot| { + snapshot.has_request_stage_candidate( + crate::automation::DevToolsNetworkResourceType::Document, + ) + }); + prepared.set_interception(intercept); + prepared + })) }; Self { pending, request, + renderer_runtime, buffered, handle_auth, prior_journal, @@ -95,43 +133,134 @@ impl PreparedInterceptedNavigationRequest { } pub(crate) async fn prepare( - self, + mut self, driver: &mut ProtocolWorkflowDriver, ) -> PreparedFetchNavigationResponse { - let Some(request) = self.request else { + let Some(request) = self.request.take() else { return PreparedFetchNavigationResponse::Result(Ok( NavigationLoadOutcome::network_failure( moli_fetch::AuthRetryBudget::ERROR_TEXT.to_owned(), ), )); }; - let response = match request { - Ok(request) if self.buffered => request - .wait_buffered() - .await - .map(FetchNavigationResponse::Captured), - Ok(request) => match request.wait().await { - Ok(response) - if moli_web_mime::response_headers_indicate_attachment_download( - &response.response().headers, - ) || (self.handle_auth - && matches!(response.response().status, 401 | 407) - && extract_auth_challenge(&response.response().headers).is_some()) => + let mut request = match request { + Ok(request) => request, + Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)), + }; + let response = loop { + let service_worker_response = if let Some(runtime) = &self.renderer_runtime { + match runtime + .fetch_service_worker_main_resource_for_navigation( + request.request(), + request.resource_loader(), + ) + .await { - let (response, journal) = response.into_parts_with_observation_journal(); - response - .into_materialized_raw_response() - .await - .map(|response| { - FetchNavigationResponse::Captured( - NetworkFetchResult::with_observation_journal(response, journal), + Ok(fetch) => { + self.pending.reserved_service_worker_client = fetch.reserved_client; + fetch.response + } + Err(error) => break Err(error), + } + } else { + None + }; + let response = if self.buffered && service_worker_response.is_none() { + request + .fetch_buffered() + .await + .with_context(|| format!("failed to fetch page `{}`", request.request().url)) + .map(FetchNavigationResponse::Captured) + } else { + let response = if let Some(response) = service_worker_response { + request + .resource_loader() + .note_service_worker_response_ready() + .and_then(|()| { + moli_core::runtime::streaming_raw_response_from_navigation_response( + response, ) }) - .map_err(|error| error.context("failed to read page body from stream")) + .map(NetworkFetchResult::without_request_observation) + } else { + request.fetch_raw_stream().await + }; + match response + .with_context(|| format!("failed to fetch page `{}`", request.request().url)) + { + Ok(response) + if moli_web_mime::response_headers_indicate_attachment_download( + &response.response().headers, + ) || (self.handle_auth + && matches!(response.response().status, 401 | 407) + && extract_auth_challenge(&response.response().headers).is_some()) => + { + let (response, journal) = response.into_parts_with_observation_journal(); + response + .into_materialized_raw_response() + .await + .map(|response| { + FetchNavigationResponse::Captured( + NetworkFetchResult::with_observation_journal(response, journal), + ) + }) + .map_err(|error| error.context("failed to read page body from stream")) + } + response => response.map(FetchNavigationResponse::Streaming), } - response => response.map(FetchNavigationResponse::Streaming), - }, - Err(error) => Err(error), + }; + let head = match &response { + Ok(FetchNavigationResponse::Streaming(response)) => { + Some(response.response().head()) + } + Ok(FetchNavigationResponse::Captured(response)) => Some(response.response().head()), + Err(_) => None, + }; + match head.as_ref().map(|head| request.follow_redirect(head)) { + Some(Ok(Some(_))) => {} + Some(Err(error)) => break Err(error), + _ => break response, + } + let journal = match response.expect("redirect has a response") { + FetchNavigationResponse::Streaming(response) => { + response.into_parts_with_observation_journal().1 + } + FetchNavigationResponse::Captured(response) => { + response.into_parts_with_observation_journal().1 + } + }; + self.prior_journal = Some(navigation::merge_network_observations( + self.prior_journal.take(), + journal, + )); + let hop = request.request(); + self.pending.navigation.requested_url = hop.url.clone(); + self.pending.navigation.request_method = hop.method.clone(); + self.pending.navigation.request_headers = hop.request_headers.clone(); + self.pending.navigation.request_body_bytes = hop.body.clone(); + self.pending.navigation.request_body = hop + .body + .as_ref() + .map(|body| String::from_utf8_lossy(body).into_owned()); + self.pending.navigation.redirect_chain = hop.redirect_chain().to_vec(); + self.pending.navigation.redirect_headers = None; + let pending = self.pending; + let executor = request.clone(); + let prior = self.prior_journal.clone().unwrap_or_default(); + let redirected = driver.with_state(move |state| { + state.prepare_fetch_navigation_redirect(pending, executor, prior) + }); + match redirected { + Ok(RedirectNavigation::Continue(pending)) => self.pending = *pending, + Ok(RedirectNavigation::Paused(plan)) => { + self.transferred.mark_transferred(); + return PreparedFetchNavigationResponse::Transferred(*plan); + } + Ok(RedirectNavigation::Superseded) => { + return PreparedFetchNavigationResponse::Superseded; + } + Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)), + } }; let result = driver.with_state(move |state| { let result = state.accept_fetch_navigation_response( @@ -219,7 +348,12 @@ pub(crate) fn accept_navigation_response( ), journal, network::MainDocumentBodyProgressSource::default(), - ), + ) + .map(|response| { + response.with_service_worker_client( + pending.reserved_service_worker_client.take(), + ) + }), )); } } @@ -248,7 +382,12 @@ pub(crate) fn accept_navigation_response( network::MainDocumentBodyProgressSource::default(), None, Vec::new(), - ), + ) + .map(|response| { + response.with_service_worker_client( + pending.reserved_service_worker_client.take(), + ) + }), )); }; let (response, network_observation_journal) = @@ -257,6 +396,9 @@ pub(crate) fn accept_navigation_response( conn, sender, PendingStreamingDocumentResponseNavigation { + reserved_service_worker_client: pending + .reserved_service_worker_client + .take(), document_navigation_token: pending .document_navigation_token .expect("request token"), @@ -273,3 +415,103 @@ pub(crate) fn accept_navigation_response( } PreparedFetchNavigationResponse::Transferred(output.into_plan()) } + +pub(crate) enum RedirectNavigation { + Continue(Box), + Paused(Box), + Superseded, +} + +pub(crate) fn prepare_navigation_redirect( + conn: &mut CdpConnection, + mut pending: PendingFetchNavigation, + executor: moli_core::network::NavigationInterception, + prior: NetworkObservationJournal, +) -> RedirectNavigation { + if !pending + .document_navigation_token + .as_ref() + .is_some_and(|token| { + conn.accepts_pending_document_navigation_for_owner(&pending.navigation.owner, token) + }) + { + return RedirectNavigation::Superseded; + } + let Some((id, _)) = + conn.allocate_pending_subresource_fetch_request_ids_for_owner(&pending.navigation.owner) + else { + return RedirectNavigation::Superseded; + }; + pending.fetch_request_id = id; + // The same selector and per-handler hop IDs serve root and child Documents. + let paused = super::prepare_navigation_request_pause(conn, &mut pending); + if !paused { + return RedirectNavigation::Continue(Box::new(pending)); + } + pending.interception = Some((executor, prior)); + let owner = pending.navigation.owner.clone(); + let event = super::request_paused_background_event( + pending.interception_session_id.as_deref(), + &pending, + ); + let mut plan = crate::domains::command_output::CommandOutputPlan::default(); + append_redirect_request_events(conn, &mut plan, &mut pending); + if conn + .register_pending_fetch_navigation_request_for_owner(&owner, pending) + .is_none() + { + return RedirectNavigation::Superseded; + } + plan.push_background_event(event); + RedirectNavigation::Paused(Box::new(plan)) +} + +fn append_redirect_request_events( + conn: &CdpConnection, + plan: &mut crate::domains::command_output::CommandOutputPlan, + pending: &mut PendingFetchNavigation, +) { + let nav = &pending.navigation; + let Some(network_id) = &nav.request_id else { + return; + }; + let info = moli_core::page::PendingSubresourceFetchInfo { + internal_id: 0, + network_request_handle: None, + frame_id: Some(nav.frame_id.clone()), + document_url: nav.requested_url.clone(), + url: nav.requested_url.clone(), + websocket_socket_id: None, + method: nav.request_method.clone(), + request_headers: nav.request_headers.clone(), + request_body: nav.request_body.clone(), + request_body_bytes: nav.clone_request_body_bytes(), + resource_type: moli_core::page::SubresourceResourceType::Document, + request_cookie_report: pending.request_cookie_report.clone(), + }; + for redirect in nav.redirect_chain.iter().skip(nav.announced_redirects) { + let mut info = info.clone(); + info.url = redirect.to_url.clone(); + let output = network::TargetSubresourceFetchPauseNetworkOutput::from_pending_fetch_info( + network_id.clone(), + nav.frame_id.clone(), + nav.loader_id.clone(), + crate::conn::monotonic_timestamp_seconds(), + nav.requested_url.clone(), + &info, + ) + .with_child_document(Some(moli_core::page::ChildDocumentFetchRequest { + loader_id: nav.loader_id.clone(), + redirect: Some(redirect.clone().into()), + })); + for session in conn.network_event_session_ids_for_owner(&nav.owner) { + plan.extend_background_events( + network::fetch_subresource_initial_request_network_events( + session.as_deref(), + &output, + ), + ); + } + } + pending.navigation.announced_redirects = pending.navigation.redirect_chain.len(); +} diff --git a/moli-protocol/src/domains/fetch/helpers.rs b/moli-protocol/src/domains/fetch/helpers.rs index 1dd6d7343c3..1c34687841e 100644 --- a/moli-protocol/src/domains/fetch/helpers.rs +++ b/moli-protocol/src/domains/fetch/helpers.rs @@ -17,7 +17,7 @@ use moli_core::page::{ SubresourceAuthChallenge, SubresourceAuthCredentials, extract_subresource_auth_challenge, subresource_auth_credentials_for_challenge, }; -use serde_json::Value; +use serde_json::{Value, json}; use url::Url; use super::params::HeaderEntry; @@ -518,7 +518,10 @@ pub(crate) fn request_paused_background_event( .map(DevToolsRequestId::from), auth_challenge: None, }; - let payload = network::fetch_request_paused_params(&network_event); + let mut payload = network::fetch_request_paused_params(&network_event); + if let Some(previous) = &pending.redirected_request_id { + payload["redirectedRequestId"] = json!(previous); + } let message = build_event("Fetch.requestPaused", payload, session_id); BackgroundProtocolEvent::immediate_automation_event( message, diff --git a/moli-protocol/src/domains/fetch/navigation.rs b/moli-protocol/src/domains/fetch/navigation.rs index d6d314b4734..0d21ec06bc1 100644 --- a/moli-protocol/src/domains/fetch/navigation.rs +++ b/moli-protocol/src/domains/fetch/navigation.rs @@ -143,7 +143,7 @@ pub(super) fn navigation_response_stage_auth_can_stream( pub(super) fn pause_streaming_response_stage_navigation_into_buffer( conn: &mut CdpConnection, out: &mut CommandOutputBuffer, - pending: PendingFetchNavigation, + mut pending: PendingFetchNavigation, response: NetworkFetchResult, ) { let response_head = response.response(); @@ -176,19 +176,23 @@ pub(super) fn pause_streaming_response_stage_navigation_into_buffer( // here would wait on the old document's debugger pause before we could even // publish Fetch.requestPaused. Continue/fulfill owns document replacement. let (response, network_observation_journal) = response.into_parts_with_observation_journal(); - conn.register_pending_fetch_response_navigation_for_owner( + conn.register_pending_fetch_response_transfer_for_owner( &pending.navigation.owner, pending.fetch_request_id.clone(), - pending.document_navigation_token.clone(), - pending.navigation.clone(), - DocumentBodySource::StreamingRaw { - requested_url: pending.navigation.requested_url.clone(), - request_method: pending.navigation.request_method.clone(), - request_headers: pending.navigation.request_headers.clone(), - response, - network_observation_journal, - body_progress_source, - }, + crate::conn::PausedDocumentTransfer::pending( + pending.fetch_request_id.clone(), + pending.document_navigation_token.clone(), + pending.navigation.clone(), + DocumentBodySource::StreamingRaw { + requested_url: pending.navigation.requested_url.clone(), + request_method: pending.navigation.request_method.clone(), + request_headers: pending.navigation.request_headers.clone(), + response, + network_observation_journal, + body_progress_source, + }, + ) + .with_service_worker_client(pending.reserved_service_worker_client.take()), ); out.extend_background_events_after_messages([navigation_response_stage_request_paused_event( pending.interception_session_id.as_deref(), @@ -244,7 +248,7 @@ pub(super) fn pause_data_url_response_stage_navigation_into_buffer( pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer( conn: &mut CdpConnection, out: &mut CommandOutputBuffer, - pending: PendingFetchNavigation, + mut pending: PendingFetchNavigation, response: NetworkFetchResult, ) { let response_status = response.response().status; @@ -252,18 +256,22 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer( let response_headers = response.response().headers.clone(); let request_cookie_report = response.response().request_cookie_report.clone(); let (response, network_observation_journal) = response.into_parts_with_observation_journal(); - conn.register_pending_fetch_response_navigation_for_owner( + conn.register_pending_fetch_response_transfer_for_owner( &pending.navigation.owner, pending.fetch_request_id.clone(), - pending.document_navigation_token.clone(), - pending.navigation.clone(), - DocumentBodySource::BufferedRaw { - requested_url: pending.navigation.requested_url.clone(), - request_method: pending.navigation.request_method.clone(), - request_headers: pending.navigation.request_headers.clone(), - response, - network_observation_journal, - }, + crate::conn::PausedDocumentTransfer::pending( + pending.fetch_request_id.clone(), + pending.document_navigation_token.clone(), + pending.navigation.clone(), + DocumentBodySource::BufferedRaw { + requested_url: pending.navigation.requested_url.clone(), + request_method: pending.navigation.request_method.clone(), + request_headers: pending.navigation.request_headers.clone(), + response, + network_observation_journal, + }, + ) + .with_service_worker_client(pending.reserved_service_worker_client.take()), ); out.extend_background_events_after_messages([navigation_response_stage_request_paused_event( pending.interception_session_id.as_deref(), @@ -276,6 +284,15 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer( )]); } +pub(super) fn merge_network_observations( + prior: Option, + current: NetworkObservationJournal, +) -> NetworkObservationJournal { + let mut journal = prior.unwrap_or_default(); + journal.append(current); + journal +} + pub(super) fn append_prior_network_observations( response: NetworkFetchResult, prior: Option, diff --git a/moli-protocol/src/domains/fetch/navigation_interception.rs b/moli-protocol/src/domains/fetch/navigation_interception.rs new file mode 100644 index 00000000000..f23bf8226e2 --- /dev/null +++ b/moli-protocol/src/domains/fetch/navigation_interception.rs @@ -0,0 +1,107 @@ +use std::collections::HashMap; + +use crate::automation::DevToolsNetworkResourceType; +use crate::conn::{ + CdpConnection, CommandOwnerScope, PendingFetchNavigation, PendingSubresourceFetchOwnerKind, + PendingSubresourceFetchRequestStage, TargetFetchSubresourceInterceptionSnapshot, +}; + +type NavigationHopIds = HashMap, (String, Option)>; + +/// Every Fetch handler advances its own hop identity, including unmatched +/// hops. The transport's navigation identity is independent of these IDs. +pub(super) fn navigation_hop_ids( + conn: &mut CdpConnection, + owner: &CommandOwnerScope, + snapshot: &TargetFetchSubresourceInterceptionSnapshot, + network_id: &str, + first_id: &str, + redirected: bool, +) -> Option { + let mut ids = HashMap::new(); + for (index, session_id) in snapshot + .fetch_session_ids(owner.session_id()) + .into_iter() + .enumerate() + { + let id = if index == 0 { + first_id.to_owned() + } else { + conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)? + .0 + }; + let previous = conn + .runtime_session_owner_slot_mut_for_owner(owner) + .and_then(|slot| { + slot.replace_navigation_fetch_request_id( + network_id.to_owned(), + session_id.clone(), + id.clone(), + ) + }); + ids.insert(session_id, (id, previous.filter(|_| redirected))); + } + Some(ids) +} + +pub(crate) fn prepare_navigation_request_pause( + conn: &mut CdpConnection, + pending: &mut PendingFetchNavigation, +) -> bool { + let owner = &pending.navigation.owner; + let Some(snapshot) = conn.target_fetch_subresource_interception_snapshot_for_owner(owner) + else { + return false; + }; + let redirected = !pending.navigation.redirect_chain.is_empty(); + let Some(ids) = navigation_hop_ids( + conn, + owner, + &snapshot, + &pending.navigation.loader_id, + &pending.fetch_request_id, + redirected, + ) else { + return false; + }; + let blocked = snapshot.matching_network_intercepts( + crate::conn::FetchRequestStage::Request, + DevToolsNetworkResourceType::Document, + &pending.navigation.requested_url, + ); + let mut stages = Vec::new(); + for stage in snapshot.matching_request_stage_pause_sessions( + owner.session_id(), + DevToolsNetworkResourceType::Document, + &pending.navigation.requested_url, + ) { + let (id, previous) = if stage.owner_kind == PendingSubresourceFetchOwnerKind::Fetch { + ids.get(&stage.session_id) + .cloned() + .unwrap_or_else(|| (pending.fetch_request_id.clone(), None)) + } else { + let Some((id, _)) = + conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner) + else { + return false; + }; + (id, None) + }; + stages.push(PendingSubresourceFetchRequestStage { + session_id: stage.session_id, + owner_kind: stage.owner_kind, + request_id: id, + redirected_request_id: previous, + blocked_intercepts: blocked.clone(), + }); + } + if stages.is_empty() { + return false; + } + let stage = stages.remove(0); + pending.fetch_request_id = stage.request_id; + pending.interception_session_id = stage.session_id; + pending.redirected_request_id = stage.redirected_request_id; + pending.remaining_request_stages = stages; + true +} diff --git a/moli-protocol/src/domains/fetch/subresource.rs b/moli-protocol/src/domains/fetch/subresource.rs index f231be513a4..a0220e6b0cd 100644 --- a/moli-protocol/src/domains/fetch/subresource.rs +++ b/moli-protocol/src/domains/fetch/subresource.rs @@ -193,35 +193,17 @@ fn prepare_subresource_fetch_pause_sources( // including hops that do not match that handler's URL patterns. let mut child_fetch_ids = HashMap::new(); if let Some(document) = child_document.as_ref() { - for (index, fetch_session_id) in fetch_snapshot - .fetch_session_ids(session_id) - .into_iter() - .enumerate() - { - let fetch_id = if index == 0 { - request_id.clone() - } else { - let Some((id, _)) = - conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner) - else { - return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic); - }; - id - }; - let previous = conn - .runtime_session_owner_slot_mut_for_owner(owner) - .and_then(|slot| { - slot.replace_child_document_fetch_request_id( - network_request_id.clone(), - fetch_session_id.clone(), - fetch_id.clone(), - ) - }); - child_fetch_ids.insert( - fetch_session_id, - (fetch_id, previous.filter(|_| document.redirect.is_some())), - ); - } + let Some(ids) = super::navigation_interception::navigation_hop_ids( + conn, + owner, + &fetch_snapshot, + &network_request_id, + &request_id, + document.redirect.is_some(), + ) else { + return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic); + }; + child_fetch_ids = ids; } if request_stage_pause_sessions.is_empty() { if child_document.is_some() { diff --git a/moli-protocol/src/domains/fetch/tests/basics.rs b/moli-protocol/src/domains/fetch/tests/basics.rs index ba13e12fbda..51731e0ae96 100644 --- a/moli-protocol/src/domains/fetch/tests/basics.rs +++ b/moli-protocol/src/domains/fetch/tests/basics.rs @@ -1212,6 +1212,10 @@ async fn disable_clears_fetch_state() { bc.active_page_target_mut() .fetch_owner .register_pending_fetch_navigation_request(PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: "INT-1".to_owned(), interception_session_id: Some("SID-1".to_owned()), @@ -1235,6 +1239,7 @@ async fn disable_clears_fetch_state() { request_id: Some("REQ-1".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/auth").unwrap(), request_method: "POST".to_owned(), request_body: Some("payload".to_owned()), @@ -1282,6 +1287,7 @@ async fn disable_clears_fetch_state() { request_id: Some("REQ-1".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/auth").unwrap(), request_method: "POST".to_owned(), request_body: Some("payload".to_owned()), @@ -1530,6 +1536,7 @@ async fn continue_with_auth_rejects_invalid_response_without_consuming_pending_a request_id: Some("REQ-8".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/auth").unwrap(), request_method: "GET".to_owned(), request_body: None, @@ -1614,6 +1621,7 @@ async fn continue_with_auth_unsupported_challenge_preserves_pending_auth_navigat request_id: Some("REQ-9".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/auth").unwrap(), request_method: "GET".to_owned(), request_body: None, @@ -1774,6 +1782,10 @@ fn extract_auth_challenge_preserves_quoted_commas_in_realm() { #[test] fn emit_auth_required_preserves_request_headers_and_post_data_shape() { let pending = PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: "INT-11".to_owned(), interception_session_id: Some("SID-1".to_owned()), @@ -1795,6 +1807,7 @@ fn emit_auth_required_preserves_request_headers_and_post_data_shape() { request_id: Some("REQ-11".to_owned()), loader_id: "LID-0000000001".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/auth").unwrap(), request_method: "POST".to_owned(), request_body: Some("payload".to_owned()), @@ -1867,6 +1880,7 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig request_id: Some("REQ-retired".to_owned()), loader_id: "LID-retired".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("https://retired.example/document").unwrap(), request_method: "GET".to_owned(), request_body: None, @@ -1913,6 +1927,10 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig conn.register_pending_fetch_navigation_request_for_owner( &owner, PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id: "FETCH-retired".to_owned(), interception_session_id: Some("SID-handler".to_owned()), diff --git a/moli-protocol/src/domains/fetch/tests/header_bytes.rs b/moli-protocol/src/domains/fetch/tests/header_bytes.rs index ac0151aa3d2..d56fc1c0be2 100644 --- a/moli-protocol/src/domains/fetch/tests/header_bytes.rs +++ b/moli-protocol/src/domains/fetch/tests/header_bytes.rs @@ -366,6 +366,34 @@ async fn navigation_header_override_expires_on_redirect_before_auth_retry() { ) .await; ctx.expect_result(70302, json!({}), Some("SID-1")); + crate::testing::wait_until_scheduler_message(&mut ctx, "redirect request pause", |message| { + message["method"] == "Fetch.requestPaused" + && message["sessionId"] == "SID-1" + && message["params"]["request"]["url"] == format!("http://{addr}/protected") + }) + .await; + let redirected = ctx.take_first_matching("redirect request pause", |message| { + message["method"] == "Fetch.requestPaused" + && message["params"]["request"]["url"] == format!("http://{addr}/protected") + }); + assert_eq!( + redirected["params"]["request"]["url"], + format!("http://{addr}/protected") + ); + assert_eq!( + redirected["params"]["redirectedRequestId"], + paused["params"]["requestId"] + ); + assert_ne!( + redirected["params"]["requestId"], + paused["params"]["requestId"] + ); + ctx.process_async( + json!({"id":70304,"sessionId":"SID-1","method":"Fetch.continueRequest", + "params":{"requestId":redirected["params"]["requestId"]}}), + ) + .await; + ctx.expect_result(70304, json!({}), Some("SID-1")); wait_until_message(&mut ctx, "SID-1", "redirected navigation auth", |message| { message["method"] == "Fetch.authRequired" }) diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch.rs index 213b48b7109..eb2a3583423 100644 --- a/moli-protocol/src/domains/fetch/tests/runtime_fetch.rs +++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch.rs @@ -176,3 +176,6 @@ mod runtime_subresources; mod session_interception; #[path = "runtime_fetch/worker_requests.rs"] mod worker_requests; + +#[path = "runtime_fetch/unified_navigation.rs"] +mod unified_navigation; diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch/unified_navigation.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch/unified_navigation.rs new file mode 100644 index 00000000000..0c8f0602ee4 --- /dev/null +++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch/unified_navigation.rs @@ -0,0 +1,332 @@ +use super::*; + +#[derive(Clone, Copy, Debug)] +enum NavigationEntry { + Main, + Frame, + Popup, +} + +async fn navigation_redirect_contract(entry: NavigationEntry, fail: bool) { + let hits = Arc::new(AtomicUsize::new(0)); + let observed_hits = hits.clone(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new() + .route( + "/page", + get(|| async { + ( + [(CONTENT_TYPE.as_str(), "text/html")], + "opener", + ) + }), + ) + .route( + "/redirect", + get(|| async { (StatusCode::FOUND, [("location", "/destination")]) }), + ) + .route( + "/destination", + get(move || { + let hits = observed_hits.clone(); + async move { + hits.fetch_add(1, Ordering::SeqCst); + ( + [(CONTENT_TYPE.as_str(), "text/html")], + "destinationshared navigation", + ) + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let mut ctx = TestContext::new(); + with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await; + enable_runtime_async(&mut ctx, "SID-1", 41_000).await; + let redirect = format!("http://{addr}/redirect"); + let destination = format!("http://{addr}/destination"); + let session = if matches!(entry, NavigationEntry::Popup) { + ctx.enable_background_navigation_scheduler_for_test(); + ctx.process_async(json!({"id":41_001,"method":"Target.setAutoAttach", + "params":{"autoAttach":true,"waitForDebuggerOnStart":true,"flatten":true}})) + .await; + ctx.expect_result(41_001, json!({}), None); + let (_, session) = + open_auto_attached_popup_from_session(&mut ctx, 41_002, "SID-1", &redirect).await; + ctx.process_async(json!({"id":41_003,"method":"Network.enable","sessionId":session})) + .await; + ctx.expect_result(41_003, json!({}), Some(&session)); + ctx.process_async(json!({"id":41_004,"method":"Page.enable","sessionId":session})) + .await; + ctx.expect_result(41_004, json!({}), Some(&session)); + session + } else { + "SID-1".to_owned() + }; + ctx.sent.clear(); + ctx.process_async( + json!({"id":41_005,"method":"Fetch.enable","sessionId":session, + "params":{"patterns":[{"urlPattern":"*/destination","resourceType":"Document"}]}}), + ) + .await; + ctx.expect_result(41_005, json!({}), Some(&session)); + match entry { + NavigationEntry::Main => ctx.process_async(json!({"id":41_006,"method":"Page.navigate","sessionId":session,"params":{"url":redirect}})).await, + NavigationEntry::Frame => ctx.process_async(json!({"id":41_006,"method":"Runtime.evaluate","sessionId":session,"params":{"expression":"document.querySelector('iframe').src = '/redirect'"}})).await, + NavigationEntry::Popup => ctx.process_async(json!({"id":41_006,"method":"Runtime.runIfWaitingForDebugger","sessionId":session})).await, + } + crate::testing::wait_until_scheduler_message( + &mut ctx, + "destination redirect must pause for every navigation entry", + |event| { + event["method"] == "Fetch.requestPaused" + && event["sessionId"] == session + && event["params"]["request"]["url"] == destination + && event["params"]["resourceType"] == "Document" + }, + ) + .await; + let paused = ctx + .sent + .iter() + .find(|event| { + event["method"] == "Fetch.requestPaused" + && event["sessionId"] == session + && event["params"]["request"]["url"] == destination + }) + .unwrap() + .clone(); + assert_eq!( + hits.load(Ordering::SeqCst), + 0, + "{entry:?} sent a redirect destination before interception" + ); + assert!( + paused["params"]["redirectedRequestId"].is_string(), + "{entry:?} must retain the hidden first-hop Fetch ID: {paused}" + ); + let network_id = paused["params"]["networkId"].as_str().unwrap().to_owned(); + assert!( + ctx.sent + .iter() + .any(|event| event["method"] == "Network.requestWillBeSent" + && event["params"]["requestId"] == network_id + && event["params"]["request"]["url"] == destination + && event["params"]["redirectResponse"]["status"] == 302), + "{entry:?}: redirect Network event must precede pause: {:?}", + ctx.sent + ); + let (method, params) = if fail { + ( + "Fetch.failRequest", + json!({"requestId":paused["params"]["requestId"],"errorReason":"BlockedByClient"}), + ) + } else { + ( + "Fetch.continueRequest", + json!({"requestId":paused["params"]["requestId"]}), + ) + }; + ctx.process_async(json!({"id":41_007,"method":method,"sessionId":session,"params":params})) + .await; + ctx.expect_result(41_007, json!({}), Some(&session)); + wait_until_message( + &mut ctx, + session.as_str(), + "one navigation Network terminal", + |event| { + matches!( + event["method"].as_str(), + Some("Network.loadingFinished" | "Network.loadingFailed") + ) && event["params"]["requestId"] == network_id + }, + ) + .await; + let terminals = ctx + .sent + .iter() + .filter(|event| { + matches!( + event["method"].as_str(), + Some("Network.loadingFinished" | "Network.loadingFailed") + ) && event["params"]["requestId"] == network_id + }) + .collect::>(); + assert_eq!(terminals.len(), 1, "{entry:?}: exactly one terminal"); + assert_eq!( + terminals[0]["method"], + if fail { + "Network.loadingFailed" + } else { + "Network.loadingFinished" + } + ); + assert_eq!(hits.load(Ordering::SeqCst), usize::from(!fail)); + let redirects = ctx + .sent + .iter() + .filter(|event| { + event["method"] == "Network.requestWillBeSent" + && event["params"]["requestId"] == network_id + && event["params"]["request"]["url"] == destination + }) + .count(); + assert_eq!( + redirects, 1, + "{entry:?}: redirect must not be announced again at completion" + ); + server.abort(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn main_navigation_uses_shared_redirect_interception() { + for fail in [true, false] { + navigation_redirect_contract(NavigationEntry::Main, fail).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn iframe_navigation_uses_shared_redirect_interception() { + for fail in [true, false] { + navigation_redirect_contract(NavigationEntry::Frame, fail).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn popup_navigation_uses_shared_redirect_interception() { + tokio::task::LocalSet::new() + .run_until(async { + for fail in [true, false] { + navigation_redirect_contract(NavigationEntry::Popup, fail).await; + } + }) + .await; +} + +async fn service_worker_navigation_contract(stage: Option<&str>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve( + listener, + Router::new() + .route( + "/page", + get(|| async { + ( + [(CONTENT_TYPE.as_str(), "text/html")], + "register worker", + ) + }), + ) + .route( + "/worker.js", + get(|| async { + ( + [(CONTENT_TYPE.as_str(), "text/javascript")], + r#" +self.addEventListener('install', event => event.waitUntil(self.skipWaiting())); +self.addEventListener('activate', event => event.waitUntil(clients.claim())); +self.addEventListener('fetch', event => { + if (new URL(event.request.url).pathname === '/controlled') { + event.respondWith(new Response('service worker navigation', { + headers: {'Content-Type': 'text/html'} + })); + } +}); +"#, + ) + }), + ) + .route( + "/controlled", + get(|| async { + ( + [(CONTENT_TYPE.as_str(), "text/html")], + "network navigation", + ) + }), + ), + ) + .await + .unwrap(); + }); + let mut ctx = TestContext::new(); + with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await; + enable_runtime_async(&mut ctx, "SID-1", 42_000).await; + ctx.process_async(json!({"id":42_001,"sessionId":"SID-1","method":"Runtime.evaluate", + "params":{"expression":"(async () => { await navigator.serviceWorker.register('/worker.js'); await navigator.serviceWorker.ready; return true; })()", "awaitPromise":true,"returnByValue":true}})).await; + crate::testing::wait_until_scheduler_message( + &mut ctx, + "service worker registration", + |event| event["id"] == 42_001, + ) + .await; + let registered = take_response_by_id(&mut ctx, 42_001); + assert_eq!(registered["result"]["result"]["value"], true); + let pattern = match stage { + Some(stage) => { + json!({"urlPattern":"*/controlled","resourceType":"Document","requestStage":stage}) + } + None => json!({"urlPattern":"*/never-matches","resourceType":"Document"}), + }; + ctx.process_async(json!({"id":42_002,"sessionId":"SID-1","method":"Fetch.enable","params":{"patterns":[pattern]}})).await; + ctx.expect_result(42_002, json!({}), Some("SID-1")); + ctx.sent.clear(); + ctx.process_async( + json!({"id":42_003,"sessionId":"SID-1","method":"Page.navigate", + "params":{"url":format!("http://{addr}/controlled")}}), + ) + .await; + if let Some(stage) = stage { + crate::testing::wait_until_scheduler_message( + &mut ctx, + "controlled navigation pause", + |event| event["method"] == "Fetch.requestPaused" && event["sessionId"] == "SID-1", + ) + .await; + let pause = ctx.take_first_matching("controlled navigation pause", |event| { + event["method"] == "Fetch.requestPaused" + }); + if stage == "Response" { + ctx.process_async(json!({"id":42_005,"sessionId":"SID-1","method":"Fetch.getResponseBody","params":{"requestId":pause["params"]["requestId"]}})).await; + crate::testing::wait_until_scheduler_message( + &mut ctx, + "captured controlled response body", + |event| event["id"] == 42_005, + ) + .await; + let body = take_response_by_id(&mut ctx, 42_005); + assert!(body.get("result").is_some(), "{body:?}"); + } + ctx.process_async(json!({"id":42_006,"sessionId":"SID-1","method":if stage == "Response" {"Fetch.continueResponse"} else {"Fetch.continueRequest"},"params":{"requestId":pause["params"]["requestId"]}})).await; + ctx.expect_result(42_006, json!({}), Some("SID-1")); + } + crate::testing::wait_until_frame_stopped_loading(&mut ctx, "TID-1").await; + ctx.process_async( + json!({"id":42_004,"sessionId":"SID-1","method":"Runtime.evaluate", + "params":{"expression":"[document.body.textContent, navigator.serviceWorker.controller !== null]","returnByValue":true}}), + ) + .await; + let result = take_response_by_id(&mut ctx, 42_004); + assert_eq!( + result["result"]["result"]["value"], + json!(["service worker navigation", true]) + ); + server.abort(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn unmatched_navigation_interception_preserves_service_worker_response() { + service_worker_navigation_contract(None).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn request_interception_preserves_service_worker_navigation() { + service_worker_navigation_contract(Some("Request")).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn response_interception_preserves_service_worker_client_after_body_capture() { + service_worker_navigation_contract(Some("Response")).await; +} diff --git a/moli-protocol/src/domains/network/agent.rs b/moli-protocol/src/domains/network/agent.rs index e099bbfb0df..78ff7501a72 100644 --- a/moli-protocol/src/domains/network/agent.rs +++ b/moli-protocol/src/domains/network/agent.rs @@ -510,7 +510,7 @@ impl TargetNetworkAgentState { .record_fetch_pause_announced_request_id(request_id); } - pub(crate) fn replace_child_document_fetch_request_id( + pub(crate) fn replace_navigation_fetch_request_id( &mut self, network_id: String, session_id: Option, @@ -518,7 +518,7 @@ impl TargetNetworkAgentState { ) -> Option { self.artifacts .subresource_network_artifacts - .replace_child_document_fetch_request_id(network_id, session_id, fetch_id) + .replace_navigation_fetch_request_id(network_id, session_id, fetch_id) } pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool { @@ -1935,7 +1935,7 @@ impl SubresourceNetworkArtifacts { .or_default(); } - fn replace_child_document_fetch_request_id( + fn replace_navigation_fetch_request_id( &mut self, network_id: String, session_id: Option, diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rs index cd770f62e83..6535849986c 100644 --- a/moli-protocol/src/domains/network/main_document_progress/mod.rs +++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs @@ -603,6 +603,7 @@ struct MainDocumentLiveNetworkProgressSource { timestamp: f64, initial_request_headers: Vec<(String, String)>, initial_request_cookie_report: Option, + announced_redirects: usize, } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -1003,6 +1004,7 @@ impl MainDocumentLiveNetworkProgressSource { loader_id: state.loader_id.clone(), frame_id: state.frame_id.clone(), timestamp: state.timestamp, + announced_redirects: state.announced_redirects, initial_request_headers: state.request_headers.to_byte_strings(), initial_request_cookie_report: initial_request_cookie_report.cloned(), }) @@ -1138,30 +1140,32 @@ impl MainDocumentLiveNetworkProgressSource { }, ); } - events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent { - target: target.clone(), - url: redirect.to_url.clone(), - method: final_request_method.to_owned(), - request_body: request_body.map(str::to_owned), - request_headers: final_request_headers.to_vec(), - request_initiator_type: SubresourceRequestInitiatorType::Other, - redirect_response: Box::new(Some(MainDocumentRedirectResponse { - url: redirect.from_url.clone(), - status: redirect.status, - status_text: critical_client_hint_internal_redirect_status_text( - &redirect.from_url, - &redirect.to_url, - redirect.status, - redirect.network_extra_info_available, - observed_response_available, - ), - headers: redirect.headers.clone(), - from_cache: redirect.from_cache, - negotiated_http_version: redirect.negotiated_http_version, - })), - redirect_has_extra_info: redirect.network_extra_info_available, - cookie_access_report: redirect.request_cookie_report.clone(), - }); + if index >= self.announced_redirects { + events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent { + target: target.clone(), + url: redirect.to_url.clone(), + method: final_request_method.to_owned(), + request_body: request_body.map(str::to_owned), + request_headers: final_request_headers.to_vec(), + request_initiator_type: SubresourceRequestInitiatorType::Other, + redirect_response: Box::new(Some(MainDocumentRedirectResponse { + url: redirect.from_url.clone(), + status: redirect.status, + status_text: critical_client_hint_internal_redirect_status_text( + &redirect.from_url, + &redirect.to_url, + redirect.status, + redirect.network_extra_info_available, + observed_response_available, + ), + headers: redirect.headers.clone(), + from_cache: redirect.from_cache, + negotiated_http_version: redirect.negotiated_http_version, + })), + redirect_has_extra_info: redirect.network_extra_info_available, + cookie_access_report: redirect.request_cookie_report.clone(), + }); + } if observed_response_available && redirect.network_extra_info_available { let (status, headers) = observed_response_metadata( network_observation_journal, @@ -1667,6 +1671,7 @@ struct CompletedMainDocumentProgressContext { session_ids: Vec>, request_id: Option, request_announced: bool, + announced_redirects: usize, requested_url: Url, request_method: String, request_body: Option, @@ -1682,7 +1687,7 @@ impl CompletedMainDocumentProgressContext { state: &NavigationDispatchState, request_id: Option, ) -> Self { - Self::new( + let mut context = Self::new( main_document_network_event_session_ids(conn, state.session_id.as_deref()), request_id, state.request_announced, @@ -1693,7 +1698,9 @@ impl CompletedMainDocumentProgressContext { state.loader_id.clone(), state.frame_id.clone(), state.timestamp, - ) + ); + context.announced_redirects = state.announced_redirects; + context } fn new( @@ -1712,6 +1719,7 @@ impl CompletedMainDocumentProgressContext { session_ids, request_id, request_announced, + announced_redirects: 0, requested_url, request_method, request_body, @@ -1808,30 +1816,32 @@ impl CompletedMainDocumentProgressContext { }, ); } - progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent { - target: target.clone(), - url: redirect.to_url.clone(), - method: events.request_method.clone(), - request_body: self.request_body.clone(), - request_headers: events.request_headers.clone(), - request_initiator_type: SubresourceRequestInitiatorType::Other, - redirect_response: Box::new(Some(MainDocumentRedirectResponse { - url: redirect.from_url.clone(), - status: redirect.status, - status_text: critical_client_hint_internal_redirect_status_text( - &redirect.from_url, - &redirect.to_url, - redirect.status, - redirect.network_extra_info_available, - observed_response_available, - ), - headers: redirect.headers.clone(), - from_cache: redirect.from_cache, - negotiated_http_version: redirect.negotiated_http_version, - })), - redirect_has_extra_info: redirect.network_extra_info_available, - cookie_access_report: redirect.request_cookie_report.clone(), - }); + if index >= self.announced_redirects { + progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent { + target: target.clone(), + url: redirect.to_url.clone(), + method: events.request_method.clone(), + request_body: self.request_body.clone(), + request_headers: events.request_headers.clone(), + request_initiator_type: SubresourceRequestInitiatorType::Other, + redirect_response: Box::new(Some(MainDocumentRedirectResponse { + url: redirect.from_url.clone(), + status: redirect.status, + status_text: critical_client_hint_internal_redirect_status_text( + &redirect.from_url, + &redirect.to_url, + redirect.status, + redirect.network_extra_info_available, + observed_response_available, + ), + headers: redirect.headers.clone(), + from_cache: redirect.from_cache, + negotiated_http_version: redirect.negotiated_http_version, + })), + redirect_has_extra_info: redirect.network_extra_info_available, + cookie_access_report: redirect.request_cookie_report.clone(), + }); + } if observed_response_available && redirect.network_extra_info_available { let (status, headers) = observed_response_metadata( &events.network_observation_journal, diff --git a/moli-protocol/src/domains/network/main_document_progress/tests.rs b/moli-protocol/src/domains/network/main_document_progress/tests.rs index d0a1676f27b..23111b66189 100644 --- a/moli-protocol/src/domains/network/main_document_progress/tests.rs +++ b/moli-protocol/src/domains/network/main_document_progress/tests.rs @@ -120,6 +120,7 @@ fn live_progress_source() -> ( let (sender, receiver) = unbounded_channel(); ( MainDocumentLiveNetworkProgressSource { + announced_redirects: 0, sender: Some(sender), progress_queue: super::gate::MainDocumentProgressQueueHandle::from_source( MainDocumentProgressSource::streaming(), @@ -1604,6 +1605,7 @@ fn no_content_navigation_retains_response_metadata_with_live_progress() { request_id: Some("REQ-1".to_owned()), loader_id: "LOADER-1".to_owned(), request_announced: false, + announced_redirects: 0, requested_url: Url::parse("http://example.test/start").unwrap(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/domains/network/tests/cache.rs b/moli-protocol/src/domains/network/tests/cache.rs index e3b1b794ba1..fd9581beefe 100644 --- a/moli-protocol/src/domains/network/tests/cache.rs +++ b/moli-protocol/src/domains/network/tests/cache.rs @@ -666,6 +666,7 @@ async fn clear_browser_cache_keeps_pending_response_navigation_transfer() { request_id: Some("REQ-1".to_owned()), loader_id: LOADER_ID.to_owned(), request_announced: true, + announced_redirects: 0, requested_url: url.clone(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-protocol/src/domains/network/tests/response_body.rs b/moli-protocol/src/domains/network/tests/response_body.rs index 535e6573d58..09de64f3e86 100644 --- a/moli-protocol/src/domains/network/tests/response_body.rs +++ b/moli-protocol/src/domains/network/tests/response_body.rs @@ -524,6 +524,7 @@ async fn get_request_post_data_returns_main_document_navigation_post_body() { request_id: Some("REQ-main-post".to_owned()), loader_id: LOADER_ID.to_owned(), request_announced: false, + announced_redirects: 0, requested_url, request_method: "POST".to_owned(), request_body: Some("username=alice&pw=s3cret".to_owned()), @@ -620,6 +621,7 @@ async fn get_request_post_data_uses_text_projection_while_bidi_collector_keeps_t request_id: Some("REQ-multipart".to_owned()), loader_id: LOADER_ID.to_owned(), request_announced: false, + announced_redirects: 0, requested_url, request_method: "POST".to_owned(), request_body: Some(multipart_text.to_owned()), diff --git a/moli-protocol/src/domains/page/navigation.rs b/moli-protocol/src/domains/page/navigation.rs index 731c0bf6762..7e8d2d02d33 100644 --- a/moli-protocol/src/domains/page/navigation.rs +++ b/moli-protocol/src/domains/page/navigation.rs @@ -2654,6 +2654,7 @@ fn start_navigate_to_url_command_with_background_policy_and_request( request_id: None, loader_id: document_loader_id.clone(), request_announced: false, + announced_redirects: 0, requested_url: requested_url.clone(), request_method: request_method.to_owned(), request_body: request_body @@ -2699,6 +2700,10 @@ fn start_navigate_to_url_command_with_background_policy_and_request( )); }; pending_fetch_navigation = Some(PendingFetchNavigation { + reserved_service_worker_client: None, + interception: None, + remaining_request_stages: Vec::new(), + redirected_request_id: None, auth_retry_budget: Default::default(), fetch_request_id, interception_session_id: preflight @@ -2791,7 +2796,8 @@ fn start_navigate_to_url_command_with_background_policy_and_request( } if let Some(mut pending) = pending_fetch_navigation { - if fetch_request_stage == FetchRequestStage::Request { + let request_pause = fetch::prepare_navigation_request_pause(conn, &mut pending); + if fetch_request_stage == FetchRequestStage::Request && request_pause { pending.request_cookie_report = navigation_cookie_access_report_for_owner( conn, &pending.navigation.owner, @@ -2810,14 +2816,13 @@ fn start_navigate_to_url_command_with_background_policy_and_request( ); } pending.navigation.request_announced = pending.navigation.request_id.is_some(); - let _ = conn.register_pending_fetch_navigation_request_for_owner( - &pending.navigation.owner, - pending.clone(), - ); out.push(fetch::request_paused_background_event( pending.interception_session_id.as_deref(), &pending, )); + let pending_owner = pending.navigation.owner.clone(); + let _ = + conn.register_pending_fetch_navigation_request_for_owner(&pending_owner, pending); let mut output = CommandOutputBuffer::default(); output.extend_background_events_after_messages(out); return NavigateCommandStart::CompleteImmediate(output.into_plan()); @@ -2840,6 +2845,14 @@ fn start_navigate_to_url_command_with_background_policy_and_request( ); } pending.navigation.request_announced = pending.navigation.request_id.is_some(); + if let Some(sender) = conn.background_event_sender() { + // Retire the previous source and announce the request before + // waiting for headers. A held transport must not retain either + // the old Worker targets or this navigation's initial events. + for event in out.drain(..) { + let _ = sender.send(event); + } + } return NavigateCommandStart::PendingContinueWithoutRequestPause(Box::new( PendingContinueNavigationWithoutRequestPauseCommand { prefix_events: out, diff --git a/moli-protocol/src/domains/runtime/tests/evaluate/navigation.rs b/moli-protocol/src/domains/runtime/tests/evaluate/navigation.rs index d175bbeacad..a1aaa388036 100644 --- a/moli-protocol/src/domains/runtime/tests/evaluate/navigation.rs +++ b/moli-protocol/src/domains/runtime/tests/evaluate/navigation.rs @@ -681,6 +681,7 @@ async fn page_navigate_network_failure_commits_error_document() { request_id: Some(stale_request_id.to_owned()), loader_id: old_document_token.loader_id.clone(), request_announced: true, + announced_redirects: 0, requested_url: url::Url::parse("https://stale.example.test/old-body").unwrap(), request_method: "GET".to_owned(), request_body: None, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index 3fd41aa560e..57a04a7c07f 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -207,19 +207,24 @@ impl JsContextHost { let fetch = ChildDocumentFetch { target, network_attribution, - navigation_url: target_url, network_info: None, - intercept_redirects: false, redirect: None, - request: child_document_load_request(bootstrap).map(|request| { - configure_child_document_navigation_request( - request, - &initiator_url, - &browser_context, - ) - .with_page_network_policy() - .with_network_partition_key(network_partition_key) - }), + request: child_document_load_request(bootstrap) + .map(|request| { + configure_child_document_navigation_request( + request, + &initiator_url, + &browser_context, + ) + .with_page_network_policy() + .with_network_partition_key(network_partition_key) + }) + .map(|request| { + crate::network::navigation::NavigationInterception::new( + resource_loader.clone(), + request, + ) + }), service_worker_client_id, browser_context_runtime, resource_loader, @@ -801,9 +806,11 @@ impl JsContextHost { initiating_loader: &crate::network::context::DocumentResourceLoader, mut fetch: ChildDocumentFetch, ) -> Option { - fetch.intercept_redirects = false; + if let Some(request) = fetch.request.as_mut() { + request.set_interception(false); + } let Some(request) = fetch.request.as_ref().filter(|request| { - matches!(request.url.scheme(), "http" | "https") + matches!(request.request().url.scheme(), "http" | "https") && self.should_intercept_subresource(SubresourceResourceType::Document) }) else { fetch.start(); @@ -827,16 +834,17 @@ impl JsContextHost { internal_id: 0, network_request_handle: None, frame_id: Some(fetch.network_attribution.frame_id().to_owned()), - document_url: request.url.clone(), - url: request.url.clone(), + document_url: request.request().url.clone(), + url: request.request().url.clone(), websocket_socket_id: None, - method: request.method.clone(), - request_headers: request.request_headers.clone(), + method: request.request().method.clone(), + request_headers: request.request().request_headers.clone(), request_body: request + .request() .body .as_ref() .map(|body| String::from_utf8_lossy(body).into_owned()), - request_body_bytes: request.body.clone(), + request_body_bytes: request.request().body.clone(), resource_type: SubresourceResourceType::Document, request_cookie_report: None, }; @@ -862,12 +870,13 @@ impl JsContextHost { headers: moli_fetch::RequestHeaders, body: Option, ) { - if let Some(mut request) = fetch.request.take() { + if let Some(interception) = fetch.request.as_mut() { + let request = interception.request_mut(); request.url = url; request.method = method; request.request_headers = headers; request.body = body; - request = request.with_redirect_headers(redirect_headers); + *request = request.clone().with_redirect_headers(redirect_headers); if let Some(info) = fetch.network_info.as_mut() { info.url = request.url.clone(); info.method = request.method.clone(); @@ -878,7 +887,6 @@ impl JsContextHost { .as_ref() .map(|body| String::from_utf8_lossy(body).into_owned()); } - fetch.request = Some(request); } // A superseded navigation has already settled without this request. if self @@ -942,10 +950,8 @@ const UNSUPPORTED_CHILD_DOCUMENT_BOOTSTRAP: &str = pub(crate) struct ChildDocumentFetch { target: ChildDocumentNavigationFetchTarget, network_attribution: ChildDocumentLoadNetworkAttribution, - request: Option, - navigation_url: url::Url, + request: Option, network_info: Option, - intercept_redirects: bool, redirect: Option, service_worker_client_id: Option, browser_context_runtime: crate::runtime::RendererBrowserContextRuntime, @@ -959,7 +965,7 @@ impl std::fmt::Debug for ChildDocumentFetch { formatter .debug_struct("ChildDocumentFetch") .field("target", &self.target) - .field("navigation_url", &self.navigation_url) + .field("request", &self.request) .finish_non_exhaustive() } } @@ -967,7 +973,9 @@ impl std::fmt::Debug for ChildDocumentFetch { impl ChildDocumentFetch { pub(crate) fn set_network_info(&mut self, info: PendingSubresourceFetchInfo) { self.network_info = Some(info); - self.intercept_redirects = true; + if let Some(request) = self.request.as_mut() { + request.set_interception(true); + } } pub(crate) fn take_request_metadata( @@ -997,7 +1005,7 @@ impl ChildDocumentFetch { .browser_context_runtime .fetch_service_worker_child_main_resource_for_reserved_client( client_id, - &request, + request.request(), self.resource_loader.request_client(), self.resource_loader.task_runner(), self.completion_tx.clone(), @@ -1011,14 +1019,8 @@ impl ChildDocumentFetch { let (head, body) = response.into_body(); return self.accept_response(request, head, body); } - let transport_request = if self.intercept_redirects { - request.clone().with_single_hop() - } else { - request.clone() - }; - let response = self - .resource_loader - .fetch_raw(transport_request) + let response = request + .fetch_raw() .await .map_err(|error| error.to_string())?; let (head, body) = response.into_body(); @@ -1027,58 +1029,23 @@ impl ChildDocumentFetch { fn accept_response( &mut self, - mut request: moli_fetch::Request, + mut request: crate::network::navigation::NavigationInterception, head: moli_fetch::ResponseHead, body: moli_fetch::ResponseBody, ) -> Result, String> { - if self.intercept_redirects - && let Some(next_url) = crate::network_host::next_redirect_url( - &head.final_url, - head.status, - &head.headers, - request.redirect_count(), - )? + if let Some(redirect) = request + .follow_redirect(&head) + .map_err(|error| error.to_string())? { - if !matches!(next_url.scheme(), "http" | "https") { - return Err(format!( - "child document redirect to unsupported URL {next_url}" - )); - } - let redirect = moli_fetch::RedirectInfo { - source: moli_fetch::RedirectSource::Network, - from_url: head.final_url.clone(), - to_url: next_url.clone(), - status: head.status, - headers: head.headers.clone(), - network_extra_info_available: false, - request_extra_info: None, - response_extra_info: None, - redirect_has_extra_info: false, - request_cookie_report: head.request_cookie_report, - cookie_set_reports: head.cookie_set_reports, - from_cache: head.from_cache, - negotiated_http_version: head.negotiated_http_version, - }; - request.update_referrer_for_redirect(&head.final_url, &head.headers); - request - .follow_redirect(redirect.clone()) - .map_err(|error| error.to_string())?; - self.resource_loader - .prepare_redirect() - .map_err(|error| error.to_string())?; - self.navigation_url = next_url; self.redirect = Some(redirect.into()); self.request = Some(request); return Ok(None); } - let navigation_url = if !self.intercept_redirects && head.redirected { - head.final_url.clone() - } else { - self.navigation_url.clone() - }; + let navigation_url = request.navigation_url(&head); + let request = request.request(); child_document_load_outcome_from_response( request.url.as_str().to_owned(), - request.method, + request.method.clone(), request.request_headers.to_byte_strings(), head, body, @@ -1300,11 +1267,12 @@ mod tests { None, "retired-loader".to_owned(), ), - navigation_url: url.clone(), network_info: None, - intercept_redirects: false, redirect: None, - request: Some(moli_fetch::Request::get_with_url(url)), + request: Some(crate::network::navigation::NavigationInterception::new( + resource_loader.clone(), + moli_fetch::Request::get_with_url(url), + )), service_worker_client_id: None, browser_context_runtime: host.browser_context_runtime(), resource_loader: resource_loader.clone(), diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rs index 4637eff619f..a6344b4dd48 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs @@ -2431,20 +2431,16 @@ impl JsContextHost { task_resource_loader.note_external_response_ready()?; response } - None => task_resource_loader - .fetch( + None => { + crate::network::navigation::NavigationInterception::new( + task_resource_loader.clone(), request .with_request_origin(request_origin) - .with_page_network_policy() .with_top_level_navigation_cookie_context(), ) - .await - .map(|response| { - moli_fetch::RawResponse::from_head_and_body( - response.head(), - response.body_bytes().to_vec(), - ) - })?, + .fetch_raw() + .await? + } }; let head = response.head(); if popup_document_response_should_ignore_navigation(response.status, &head.headers) diff --git a/moli-renderer-v8/src/network/navigation/interception.rs b/moli-renderer-v8/src/network/navigation/interception.rs new file mode 100644 index 00000000000..eb61f479c69 --- /dev/null +++ b/moli-renderer-v8/src/network/navigation/interception.rs @@ -0,0 +1,161 @@ +use anyhow::{Result, bail}; +use moli_fetch::{ + NetworkFetchResult, RawResponse, RedirectInfo, Request, Response, ResponseHead, + StreamingRawResponse, +}; +use url::Url; + +use super::NavigationResourceLoader; + +/// Request execution shared by top-level, child and auxiliary navigations. +/// The navigation owner decides when a hop may start and where its result can +/// commit. This object owns transport routing, request overrides and redirects; +/// it deliberately has no DevTools session or frame-lifecycle state. +#[derive(Clone, Debug)] +pub struct NavigationInterception { + loader: NavigationResourceLoader, + request: Request, + navigation_url: Url, + intercept_redirects: bool, +} + +struct CancelOnDrop(Option); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if let Some(loader) = self.0.take() { + loader.cancel(); + } + } +} + +impl NavigationInterception { + pub fn new(loader: NavigationResourceLoader, request: Request) -> Self { + Self { + navigation_url: request.url.clone(), + loader, + request, + intercept_redirects: false, + } + } + + pub fn resource_loader(&self) -> &NavigationResourceLoader { + &self.loader + } + + pub fn request(&self) -> &Request { + &self.request + } + + pub fn request_mut(&mut self) -> &mut Request { + &mut self.request + } + + pub fn set_interception(&mut self, enabled: bool) { + self.intercept_redirects = enabled; + } + + pub fn navigation_url(&self, response: &ResponseHead) -> Url { + if !self.intercept_redirects && response.redirected { + response.final_url.clone() + } else { + self.navigation_url.clone() + } + } + + fn transport_request(&self) -> Result { + moli_url_policy::route_navigation_url( + &self.request.url, + moli_url_policy::LocalFileNavigationAccess::Denied, + )?; + Ok(if self.intercept_redirects { + self.request.clone().with_single_hop() + } else { + self.request.clone() + }) + } + + async fn execute( + &self, + operation: impl std::future::Future>, + ) -> Result { + let mut guard = CancelOnDrop(Some(self.loader.clone())); + let result = operation.await; + guard.0.take(); + result + } + + pub async fn fetch_text(&self) -> Result> { + self.execute( + self.loader + .fetch_with_network_metadata(self.transport_request()?), + ) + .await + } + + pub async fn fetch_raw(&self) -> Result { + self.execute(self.loader.fetch_raw(self.transport_request()?)) + .await + } + + pub async fn fetch_raw_stream(&self) -> Result> { + self.execute( + self.loader + .fetch_raw_stream_with_network_metadata(self.transport_request()?), + ) + .await + } + + pub async fn fetch_buffered(&self) -> Result> { + self.execute( + self.loader + .fetch_raw_with_network_metadata(self.transport_request()?), + ) + .await + } + + /// Returns the next hop to its owner before any transport is started. + /// Synthetic and network responses use exactly the same redirect rules. + pub fn follow_redirect(&mut self, head: &ResponseHead) -> Result> { + if !self.intercept_redirects { + return Ok(None); + } + let Some(next_url) = crate::network_host::next_redirect_url( + &head.final_url, + head.status, + &head.headers, + self.request.redirect_count(), + ) + .map_err(anyhow::Error::msg)? + else { + return Ok(None); + }; + if !matches!(next_url.scheme(), "http" | "https") { + bail!("document redirect to unsupported URL {next_url}"); + } + let redirect = RedirectInfo { + source: moli_fetch::RedirectSource::Network, + from_url: head.final_url.clone(), + to_url: next_url.clone(), + status: head.status, + headers: head.headers.clone(), + network_extra_info_available: false, + request_extra_info: None, + response_extra_info: None, + redirect_has_extra_info: false, + request_cookie_report: head.request_cookie_report.clone(), + cookie_set_reports: head.cookie_set_reports.clone(), + from_cache: head.from_cache, + negotiated_http_version: head.negotiated_http_version, + }; + self.request + .update_referrer_for_redirect(&head.final_url, &head.headers); + self.request.follow_redirect(redirect.clone())?; + self.loader.prepare_redirect()?; + self.navigation_url = next_url; + Ok(Some(redirect)) + } +} + +#[cfg(test)] +mod tests; diff --git a/moli-renderer-v8/src/network/navigation/interception/tests.rs b/moli-renderer-v8/src/network/navigation/interception/tests.rs new file mode 100644 index 00000000000..6f0c826d4c4 --- /dev/null +++ b/moli-renderer-v8/src/network/navigation/interception/tests.rs @@ -0,0 +1,104 @@ +use super::*; +use crate::network::{RendererResourceTaskRunner, ResourceRequestClient}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +async fn request_head(socket: &mut tokio::net::TcpStream) -> String { + let mut head = Vec::new(); + let mut bytes = [0u8; 4096]; + while !head.ends_with(b"\r\n\r\n") { + let length = socket.read(&mut bytes).await.unwrap(); + assert!(length > 0, "connection closed before request headers"); + head.extend_from_slice(&bytes[..length]); + } + String::from_utf8(head).unwrap() +} + +#[tokio::test] +async fn redirect_body_cancellation_does_not_cancel_the_next_navigation_hop() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut first, _) = listener.accept().await.unwrap(); + assert!(request_head(&mut first).await.starts_with("GET /redirect ")); + first + .write_all(b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 1000000\r\n\r\n") + .await + .unwrap(); + let (mut second, _) = listener.accept().await.unwrap(); + assert!(request_head(&mut second).await.starts_with("GET /final ")); + second + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + }); + let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap(); + let request = Request::get(&format!("http://{addr}/redirect")).unwrap(); + let origin = moli_url::WebOrigin::from_url(&request.url); + let request = request.with_request_origin(origin); + let loader = NavigationResourceLoader::new( + owner.handle(), + request.url.clone(), + RendererResourceTaskRunner::from_current_tokio().unwrap(), + ); + let mut navigation = NavigationInterception::new(loader.clone(), request); + navigation.set_interception(true); + let response = navigation.fetch_raw_stream().await.unwrap(); + let redirect = navigation + .follow_redirect(&response.response().head()) + .unwrap() + .unwrap(); + assert_eq!(redirect.to_url.path(), "/final"); + drop(response); + let response = tokio::time::timeout(std::time::Duration::from_secs(10), navigation.fetch_raw()) + .await + .unwrap() + .unwrap(); + let (_, body) = response.into_body(); + assert_eq!(body.try_into_materialized_bytes().unwrap(), b"ok"); + assert_eq!( + loader.state(), + super::super::NavigationResourceLoaderState::ResponseReady + ); + server.await.unwrap(); +} + +#[tokio::test] +async fn dropping_a_request_future_cancels_transport_while_its_navigation_owner_survives() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (received, request_received) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = [0u8; 4096]; + assert!(socket.read(&mut bytes).await.unwrap() > 0); + received.send(()).unwrap(); + let read = + tokio::time::timeout(std::time::Duration::from_secs(10), socket.read(&mut bytes)) + .await + .unwrap(); + assert!( + matches!(read, Ok(0)) || read.is_err(), + "cancelled navigation must close the transport" + ); + }); + let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap(); + let request = Request::get(&format!("http://{addr}/pending")).unwrap(); + let origin = moli_url::WebOrigin::from_url(&request.url); + let request = request.with_request_origin(origin); + let loader = NavigationResourceLoader::new( + owner.handle(), + request.url.clone(), + RendererResourceTaskRunner::from_current_tokio().unwrap(), + ); + let navigation = NavigationInterception::new(loader.clone(), request); + let task = tokio::spawn(async move { navigation.fetch_raw_stream().await }); + request_received.await.unwrap(); + task.abort(); + assert!(task.await.is_err_and(|error| error.is_cancelled())); + assert_eq!( + loader.state(), + super::super::NavigationResourceLoaderState::Cancelled + ); + server.await.unwrap(); +} diff --git a/moli-renderer-v8/src/network/navigation/loader.rs b/moli-renderer-v8/src/network/navigation/loader.rs index 0b5500664f9..d0099531b76 100644 --- a/moli-renderer-v8/src/network/navigation/loader.rs +++ b/moli-renderer-v8/src/network/navigation/loader.rs @@ -277,6 +277,30 @@ impl NavigationResourceLoader { Ok(()) } + pub async fn fetch_raw_with_network_metadata( + &self, + request: Request, + ) -> Result> { + self.begin_fetch()?; + let result = self + .request_client + .fetch_raw_with_cancel_and_network_metadata( + request.with_page_network_policy(), + self.inner.cancel.clone(), + ) + .await; + match result { + Ok(response) => { + self.finish_response_ready()?; + Ok(response) + } + Err(error) => { + self.finish_failed(); + Err(error) + } + } + } + pub async fn fetch_raw_stream(&self, request: Request) -> Result { self.fetch_raw_stream_with_network_metadata(request) .await @@ -288,16 +312,30 @@ impl NavigationResourceLoader { request: Request, ) -> Result> { self.begin_fetch()?; + let cancellation = if request.transport_follows_redirects() { + self.inner.cancel.clone() + } else { + self.inner.cancel.child_for_subrequest() + }; match self .request_client .fetch_raw_stream_with_cancel_and_network_metadata( request.with_page_network_policy(), - self.inner.cancel.clone(), + cancellation.clone(), ) .await { Ok(response) => { self.finish_response_ready()?; + if !matches!(response.response().status, 301 | 302 | 303 | 307 | 308) + || !response + .response() + .headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("location")) + { + self.inner.cancel.adopt_response_progress(&cancellation); + } Ok(response) } Err(error) => { diff --git a/moli-renderer-v8/src/network/navigation/mod.rs b/moli-renderer-v8/src/network/navigation/mod.rs index 670c87009bd..b67dbe50ba5 100644 --- a/moli-renderer-v8/src/network/navigation/mod.rs +++ b/moli-renderer-v8/src/network/navigation/mod.rs @@ -1,7 +1,9 @@ mod context; +mod interception; mod loader; pub use context::DocumentFetchContextSeed; +pub use interception::NavigationInterception; pub use loader::{ NavigationResourceLoader, NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState, };