refactor(navigation): share document request execution

Reuse NavigationInterception for main, child-frame and auxiliary document requests, replacing the protocol-only PreparedNavigationRequest wrapper. Reevaluate main-document Fetch interception per redirect and share per-session hop identities.

Preserve Service Worker clients through response handling, publish source retirement before waiting for headers, and keep transport failures on the existing error-document path.
This commit is contained in:
ldm0
2026-11-05 04:20:11 +08:00
committed by Donough Liu
parent a21e82fe5b
commit cf79aa002d
47 changed files with 1609 additions and 418 deletions
+5 -5
View File
@@ -1,12 +1,12 @@
pub use moli_renderer_v8::network::{
BrowserResourceRuntime, BrowserResourceRuntimeOwner, ResourceRequestClient,
SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation, WebStorageMutationRecord,
WebStorageMutationSubscription, WebStorageString,
BrowserResourceRuntime, BrowserResourceRuntimeOwner, RendererResourceTaskRunner,
ResourceRequestClient, SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation,
WebStorageMutationRecord, WebStorageMutationSubscription, WebStorageString,
context::{DocumentResourceLoader, DocumentResourceLoaderDiagnostics},
deep_clone_shared_web_storage_store,
navigation::{
DocumentFetchContextSeed, NavigationResourceLoader, NavigationResourceLoaderDiagnostics,
NavigationResourceLoaderState,
DocumentFetchContextSeed, NavigationInterception, NavigationResourceLoader,
NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState,
},
new_shared_web_storage_store, web_storage_partitioned_area_key,
};
+15 -3
View File
@@ -1458,11 +1458,23 @@ fn external_raw_document_body_from_streaming_response_with_observer_and_progress
body_stream
}
fn streaming_raw_response_from_navigation_response(
response: NavigationResponse,
pub fn streaming_raw_response_from_navigation_response(
mut response: NavigationResponse,
) -> Result<StreamingRawResponse> {
let head = response.head();
let body = response.clone_body_bytes();
let body = response.take_body_bytes();
streaming_raw_response_from_raw_response(moli_fetch::RawResponse::from_head_and_body(
head, body,
))
}
pub fn streaming_raw_response_from_raw_response(
response: moli_fetch::RawResponse,
) -> Result<StreamingRawResponse> {
let (head, body) = response.into_body();
let body = body
.try_into_materialized_bytes()
.map_err(|_| anyhow!("navigation response body must be materialized"))?;
let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
if !body.is_empty() {
body_tx
+14 -30
View File
@@ -74,28 +74,6 @@ pub enum CommittedDocumentResourceSource {
Synthetic,
}
fn streaming_raw_response_from_navigation_response(
response: NavigationResponse,
) -> Result<StreamingRawResponse> {
let head = response.head();
let body = response.clone_body_bytes();
let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
if !body.is_empty() {
body_tx
.send(body)
.map_err(|_| anyhow!("failed to enqueue service worker main resource body"))?;
}
drop(body_tx);
let (completion_tx, completion_rx) = tokio::sync::oneshot::channel();
let _ = completion_tx.send(Ok(()));
Ok(StreamingRawResponse::new_with_head(
head,
body_rx,
FetchCancelHandle::new(),
completion_rx,
))
}
#[derive(Clone)]
pub struct NavigationResourceStorageHandles {
cookie_store: SharedBrowserCookieStore,
@@ -1207,10 +1185,13 @@ impl NavigationEngine {
request.url.clone(),
FetchCancelHandle::new(),
)?;
navigation_loader
.fetch_with_network_metadata(request)
.await
.map(|result| result.map_response(NavigationResponse::from))
moli_renderer_v8::network::navigation::NavigationInterception::new(
navigation_loader,
request,
)
.fetch_text()
.await
.map(|result| result.map_response(NavigationResponse::from))
}
pub async fn fetch_navigation_response_with_storage_async(
@@ -1301,15 +1282,18 @@ impl NavigationEngine {
let final_url = response.final_url.clone();
return Ok(NavigationStreamingRawResponse {
fetch_result: NetworkFetchResult::without_request_observation(
streaming_raw_response_from_navigation_response(response)?,
super::streaming_raw_response_from_navigation_response(response)?,
),
reserved_service_worker_client: reserved_client,
document_fetch_context_seed: navigation_loader.commit(final_url)?,
});
}
let fetch_result = navigation_loader
.fetch_raw_stream_with_network_metadata(request)
.await?;
let fetch_result = moli_renderer_v8::network::navigation::NavigationInterception::new(
navigation_loader.clone(),
request,
)
.fetch_raw_stream()
.await?;
let final_url = fetch_result.response().final_url.clone();
Ok(NavigationStreamingRawResponse {
fetch_result,
+3 -1
View File
@@ -683,7 +683,7 @@ pub(crate) use runtime_load::{
BackgroundNavigationLoadJob, CapturedPageReplacement, CommittedNavigationPage,
CompletedInitialDocumentPageBuild, FailedInitialDocumentPageBuild,
InitialDocumentPageInstallResult, InitialDocumentPageOwner, PendingInitialDocumentPageBuild,
PreparedCapturedNavigationResponse, PreparedNavigationLoadError, PreparedNavigationRequest,
PreparedCapturedNavigationResponse, PreparedNavigationLoadError,
PreparedStreamingNavigationResponse, ResponseCommitReady, StablePageNavigationCommitTarget,
};
use scheduler_hooks::CdpSchedulerHooks;
@@ -4271,3 +4271,5 @@ pub struct PermissionOverride {
#[cfg(test)]
mod tests;
pub(crate) use state::TargetFetchSubresourceInterceptionSnapshot;
@@ -576,25 +576,6 @@ impl CdpConnection {
})
}
pub(crate) fn register_pending_fetch_response_navigation_for_owner(
&mut self,
owner: &CommandOwnerScope,
request_id: String,
document_navigation_token: Option<crate::conn::DocumentNavigationToken>,
navigation: crate::conn::NavigationDispatchState,
body: crate::conn::DocumentBodySource,
) -> bool {
self.target_session_owner_mut_for_owner(owner)
.is_some_and(|mut owner| {
owner.register_pending_fetch_response_navigation(
request_id,
document_navigation_token,
navigation,
body,
)
})
}
pub(crate) fn take_fetch_response_transfer_for_cancellation_for_owner(
&mut self,
owner: &CommandOwnerScope,
@@ -1430,34 +1411,6 @@ impl TargetSessionOwnerMut<'_> {
true
}
fn register_pending_fetch_response_navigation(
&mut self,
request_id: String,
document_navigation_token: Option<crate::conn::DocumentNavigationToken>,
navigation: crate::conn::NavigationDispatchState,
body: crate::conn::DocumentBodySource,
) -> bool {
if let Some(token) = document_navigation_token.as_ref()
&& let crate::conn::DocumentBodySource::StreamingRaw { response, .. } = &body
{
// Navigation cancellation also covers a response already handed
// to a body reader or a continuation outside the Fetch owner.
self.runtime_slot_mut()
.page_slot_mut()
.add_document_navigation_cancellation(token, response.cancellation_handle());
}
let Some(mut owner) = self.pending_fetch_owner_mut() else {
return false;
};
owner.register_pending_fetch_response_navigation(
request_id,
document_navigation_token,
navigation,
body,
);
true
}
fn take_fetch_response_transfer_for_cancellation(
&mut self,
request_id: &str,
@@ -1479,6 +1432,13 @@ impl TargetSessionOwnerMut<'_> {
request_id: String,
transfer: PausedDocumentTransfer,
) -> bool {
if let Some(token) = transfer.navigation_token()
&& let Some(cancellation) = transfer.cancellation_handle()
{
self.runtime_slot_mut()
.page_slot_mut()
.add_document_navigation_cancellation(token, cancellation);
}
let Some(mut owner) = self.pending_fetch_owner_mut() else {
return false;
};
@@ -1105,7 +1105,12 @@ impl<'a> TargetSessionOwnerMut<'a> {
.map(|_| FetchRequestStage::Response)
})
.or_else(|| {
document_fetch_response_stage_candidate.then_some(FetchRequestStage::Response)
(document_fetch_response_stage_candidate
|| (!is_data_url
&& fetch_snapshot.has_request_stage_candidate(
DevToolsNetworkResourceType::Document,
)))
.then_some(FetchRequestStage::Response)
});
let document_fetch_event_session_id = document_request_pause
.as_ref()
@@ -197,6 +197,10 @@ async fn bidi_fetch_control_resolves_background_request_owner() {
conn.register_pending_fetch_navigation_request_for_owner(
&crate::conn::CommandOwnerScope::for_session("SID-background"),
PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: "FETCH-background".to_owned(),
interception_session_id: Some("bidi-session-1".to_owned()),
@@ -216,6 +220,7 @@ async fn bidi_fetch_control_resolves_background_request_owner() {
request_id: Some("NETWORK-background".to_owned()),
loader_id: "LOADER-background".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: url::Url::parse("https://example.test/background").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
+64 -5
View File
@@ -222,8 +222,16 @@ impl PendingSubresourceFetchOwnerKind {
}
}
#[derive(Debug, Clone)]
#[derive(Debug)]
pub struct PendingFetchNavigation {
pub(crate) reserved_service_worker_client:
Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
pub(crate) interception: Option<(
moli_core::network::NavigationInterception,
moli_fetch::NetworkObservationJournal,
)>,
pub(crate) remaining_request_stages: Vec<PendingSubresourceFetchRequestStage>,
pub(crate) redirected_request_id: Option<String>,
/// Authentication restart budget for this logical navigation, independent of pause IDs.
pub(crate) auth_retry_budget: moli_fetch::AuthRetryBudget,
pub fetch_request_id: String,
@@ -300,6 +308,7 @@ impl PendingFetchAuthNavigation {
/// Protocol keeps navigation ownership even while a task exclusively reads the body.
#[derive(Debug)]
pub struct PausedDocumentTransfer {
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
fetch_request_id: String,
document_navigation_token: Option<DocumentNavigationToken>,
navigation: NavigationDispatchState,
@@ -510,6 +519,7 @@ impl PausedDocumentTransfers {
self.transfers.insert(request_id, transfer);
}
#[cfg(test)]
pub(crate) fn register_pending_navigation(
&mut self,
request_id: String,
@@ -737,6 +747,8 @@ impl PendingDocumentBodyRead {
}
pub(crate) struct PendingStreamingDocumentResponseNavigation {
pub(crate) reserved_service_worker_client:
Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
pub(crate) document_navigation_token: DocumentNavigationToken,
pub(crate) navigation: NavigationDispatchState,
pub(crate) response: StreamingRawResponse,
@@ -745,6 +757,32 @@ pub(crate) struct PendingStreamingDocumentResponseNavigation {
}
impl PausedDocumentTransfer {
pub(crate) fn with_service_worker_client(
mut self,
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
) -> Self {
self.reserved_service_worker_client = client;
self
}
pub(crate) fn cancellation_handle(&self) -> Option<moli_fetch::FetchCancelHandle> {
match &self.state {
PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) => {
match body.as_ref() {
DocumentBodySource::StreamingRaw { response, .. } => {
Some(response.cancellation_handle())
}
_ => None,
}
}
PausedDocumentTransferState::Available(PausedDocumentBody::ActiveBodyStream {
stream,
..
}) => Some(stream.response.cancellation_handle()),
PausedDocumentTransferState::Reading(reading) => reading.cancellation.clone(),
}
}
pub(crate) fn navigation_token(&self) -> Option<&DocumentNavigationToken> {
self.document_navigation_token.as_ref()
}
@@ -757,7 +795,7 @@ impl PausedDocumentTransfer {
// Otherwise the same owned preparation accepts captured or streaming bodies.
#[allow(clippy::result_large_err)]
pub(crate) fn prepare_response(
self,
mut self,
conn: &mut CdpConnection,
response_code: Option<u16>,
response_headers: Vec<(String, Vec<u8>)>,
@@ -813,7 +851,12 @@ impl PausedDocumentTransfer {
body_progress_source,
response_code,
response_headers,
),
)
.map(|response| {
response.with_service_worker_client(
self.reserved_service_worker_client.take(),
)
}),
)),
);
}
@@ -844,11 +887,17 @@ impl PausedDocumentTransfer {
progress,
)
};
Ok(crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new(response)))
Ok(
crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new(
response.map(|response| {
response.with_service_worker_client(self.reserved_service_worker_client.take())
}),
)),
)
}
pub(crate) fn prepare_synthetic_response(
&self,
mut self,
conn: &mut CdpConnection,
response_code: u16,
response_headers: Vec<(String, Vec<u8>)>,
@@ -883,6 +932,9 @@ impl PausedDocumentTransfer {
NetworkObservationJournal::default(),
body_progress_source,
)
.map(|response| {
response.with_service_worker_client(self.reserved_service_worker_client.take())
})
}
pub(crate) fn into_navigation(
@@ -899,6 +951,7 @@ impl PausedDocumentTransfer {
body: DocumentBodySource,
) -> Self {
Self {
reserved_service_worker_client: None,
fetch_request_id,
document_navigation_token,
navigation,
@@ -936,6 +989,7 @@ impl PausedDocumentTransfer {
handle: String,
) -> Result<PendingFetchResponseOpenedBodyStream, Box<Self>> {
let Self {
reserved_service_worker_client,
fetch_request_id,
document_navigation_token,
navigation,
@@ -944,6 +998,7 @@ impl PausedDocumentTransfer {
let PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) = state
else {
return Err(Box::new(Self {
reserved_service_worker_client,
fetch_request_id,
document_navigation_token,
navigation,
@@ -985,6 +1040,7 @@ impl PausedDocumentTransfer {
handle,
buffered_body,
transfer: Self {
reserved_service_worker_client,
fetch_request_id,
document_navigation_token,
navigation,
@@ -997,6 +1053,7 @@ impl PausedDocumentTransfer {
self,
) -> Result<PendingStreamingDocumentResponseNavigation, Box<Self>> {
let Self {
reserved_service_worker_client,
fetch_request_id,
document_navigation_token,
navigation,
@@ -1009,6 +1066,7 @@ impl PausedDocumentTransfer {
) if matches!(body.as_ref(), DocumentBodySource::StreamingRaw { .. }) => (token, body),
(document_navigation_token, state) => {
return Err(Box::new(Self {
reserved_service_worker_client,
fetch_request_id,
document_navigation_token,
navigation,
@@ -1026,6 +1084,7 @@ impl PausedDocumentTransfer {
unreachable!("the response was checked before transferring it")
};
Ok(PendingStreamingDocumentResponseNavigation {
reserved_service_worker_client,
document_navigation_token,
navigation,
response,
@@ -380,6 +380,7 @@ mod tests {
request_id: Some("REQ-1".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: url.clone(),
request_method: "GET".to_owned(),
request_body: None,
+27 -53
View File
@@ -903,52 +903,6 @@ struct BackgroundNavigationClaim {
cancellation: FetchCancelHandle,
}
/// Transport inputs captured while the navigation owner is available. Waiting
/// for response headers owns neither a connection nor a renderer Page.
pub(crate) struct PreparedNavigationRequest {
client: moli_core::network::ResourceRequestClient,
request: Request,
cancellation: FetchCancelHandle,
}
struct CancelNavigationRequestOnDrop(Option<FetchCancelHandle>);
impl Drop for CancelNavigationRequestOnDrop {
fn drop(&mut self) {
if let Some(cancellation) = self.0.take() {
cancellation.cancel();
}
}
}
impl PreparedNavigationRequest {
pub(crate) async fn wait(self) -> Result<NetworkFetchResult<StreamingRawResponse>> {
let mut cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone()));
let url = self.request.url.clone();
let response = self
.client
.fetch_raw_stream_with_cancel_and_network_metadata(self.request, self.cancellation)
.await
.with_context(|| format!("failed to fetch page `{url}`"));
if response.is_ok() {
// The streaming response now owns transport cancellation.
cancellation.0.take();
}
response
}
pub(crate) async fn wait_buffered(self) -> Result<NetworkFetchResult<RawResponse>> {
let _cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone()));
let url = self.request.url.clone();
// libcurl owns Digest's intermediate challenges. The exact navigation
// cancellation reaches that transport as well as this waiting future.
self.client
.fetch_raw_with_cancel_and_network_metadata(self.request, self.cancellation)
.await
.with_context(|| format!("failed to fetch page `{url}`"))
}
}
pub(crate) struct BackgroundNavigationLoadJob {
engine: NavigationEngine,
page_reservation: RendererDocumentPreparationTarget,
@@ -969,6 +923,7 @@ pub(crate) struct BackgroundNavigationLoadJob {
}
pub(crate) struct PreparedStreamingNavigationResponse {
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
engine: NavigationEngine,
page_reservation: RendererDocumentPreparationTarget,
load_inputs: TargetNavigationLoadInputs,
@@ -1284,6 +1239,14 @@ impl BackgroundNavigationLoadJob {
}
impl PreparedStreamingNavigationResponse {
pub(crate) fn with_service_worker_client(
mut self,
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
) -> Self {
self.reserved_service_worker_client = client;
self
}
pub(crate) async fn run(
mut self,
body_completion_sink: Option<BackgroundNavigationBodyCompletionSink>,
@@ -1307,7 +1270,7 @@ impl PreparedStreamingNavigationResponse {
self.response_headers_override,
self.body_progress_source,
body_completion_sink,
None,
self.reserved_service_worker_client,
CommittedDocumentResourceSource::Synthetic,
self.reply_boundary,
)
@@ -2717,6 +2680,7 @@ impl CdpConnection {
let (response, network_observation_journal) =
response.into_parts_with_observation_journal();
Ok(PreparedStreamingNavigationResponse {
reserved_service_worker_client: None,
engine,
page_reservation,
load_inputs,
@@ -2756,6 +2720,7 @@ impl CdpConnection {
let page_reservation =
self.reserve_renderer_page_for_owner(&navigation.owner, &load_inputs, &engine);
Ok(PreparedStreamingNavigationResponse {
reserved_service_worker_client: None,
engine,
page_reservation,
load_inputs,
@@ -3225,7 +3190,7 @@ impl CdpConnection {
token: &DocumentNavigationToken,
navigation: &NavigationDispatchState,
auth: Option<SubresourceAuthCredentials>,
) -> Result<PreparedNavigationRequest> {
) -> Result<moli_core::network::NavigationInterception> {
let load_inputs = self.navigation_load_inputs_for_navigation(Some(token), navigation)?;
let cancellation = self
.runtime_session_owner_slot_for_owner(&navigation.owner)
@@ -3243,11 +3208,15 @@ impl CdpConnection {
let client = self
.ensure_resource_request_client_for_navigation_load_inputs(&load_inputs)?
.frozen_request_client();
Ok(PreparedNavigationRequest {
let loader = moli_core::network::NavigationResourceLoader::new_with_cancel_handle(
client,
request,
request.url.clone(),
moli_core::network::RendererResourceTaskRunner::from_current_tokio()?,
cancellation,
})
);
Ok(moli_core::network::NavigationInterception::new(
loader, request,
))
}
async fn fetch_navigation_streaming_raw_response_with_load_inputs_async(
@@ -3271,8 +3240,13 @@ impl CdpConnection {
let loader = self
.ensure_resource_request_client_for_navigation_load_inputs(load_inputs)?
.clone();
loader
.fetch_raw_stream_with_cancel_and_network_metadata(request, FetchCancelHandle::new())
let navigation = moli_core::network::NavigationResourceLoader::new(
loader,
request.url.clone(),
moli_core::network::RendererResourceTaskRunner::from_current_tokio()?,
);
moli_core::network::NavigationInterception::new(navigation, request)
.fetch_raw_stream()
.await
.with_context(|| format!("failed to fetch page `{raw_url}`"))
}
@@ -5,6 +5,7 @@ use super::*;
/// A captured response's parser inputs and exact renderer reservation. Taking
/// this value ends protocol state access; preparation owns all of its waits.
pub(crate) struct PreparedCapturedNavigationResponse {
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
engine: NavigationEngine,
page_reservation: Option<RendererDocumentPreparationTarget>,
load_inputs: TargetNavigationLoadInputs,
@@ -45,6 +46,14 @@ impl PreparedNavigationLoadError {
}
impl PreparedCapturedNavigationResponse {
pub(crate) fn with_service_worker_client(
mut self,
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
) -> Self {
self.reserved_service_worker_client = client;
self
}
pub(crate) async fn run(mut self) -> Result<NavigationLoadOutcome> {
if super::super::downloads::response_headers_indicate_download(&self.head.headers) {
complete_auxiliary_download_response(&self.load_inputs, self.head.clone());
@@ -71,6 +80,34 @@ impl PreparedCapturedNavigationResponse {
&self.body_progress_source,
));
}
if let Some(client) = self.reserved_service_worker_client.take() {
let body = self
.body
.process_in_worker(|body| body.materialize_bytes())
.await?;
let response = moli_core::runtime::streaming_raw_response_from_raw_response(
RawResponse::from_head_and_body(self.head, body),
)?;
return build_navigation_from_streaming_raw_response_with_engine_async(
&mut self.engine,
self.page_reservation
.expect("a Document response must reserve its renderer Page"),
&self.load_inputs,
self.requested_url,
self.request_method,
self.request_headers,
response,
self.network_observation_journal,
None,
Vec::new(),
self.body_progress_source,
None,
Some(client),
CommittedDocumentResourceSource::Synthetic,
RendererReplyBoundary::Stage,
)
.await;
}
prepare_navigation_from_captured_raw_response_with_engine_async(
&mut self.engine,
self.page_reservation
@@ -99,9 +136,34 @@ impl CdpConnection {
navigation: &NavigationDispatchState,
error: anyhow::Error,
) -> Result<PreparedNavigationLoadError> {
let Some(network_error) = error.downcast_ref::<NavigationNetworkError>() else {
return Err(error);
};
let (unreachable_url, request_method, request_headers, error_text) =
if let Some(network_error) = error.downcast_ref::<NavigationNetworkError>() {
(
network_error.unreachable_url.clone(),
network_error.request_method.clone(),
network_error.request_headers.clone(),
network_error.kind.error_text().to_owned(),
)
} else if !moli_fetch::is_fetch_cancelled(&error)
&& let Some(failure) = error.downcast_ref::<NetworkFetchFailureContext>()
{
match failure.request_context() {
Some(request) => (
request.current_url().clone(),
request.request_method().to_owned(),
request.request_headers().clone(),
failure.network_error_text().to_owned(),
),
None => (
navigation.requested_url.clone(),
navigation.request_method.clone(),
navigation.request_headers.clone(),
failure.network_error_text().to_owned(),
),
}
} else {
return Err(error);
};
let load_inputs = self.navigation_load_inputs_for_navigation(token, navigation)?;
let engine = self.navigation_engine_handle_for_load_inputs(&load_inputs);
let page_reservation =
@@ -110,10 +172,10 @@ impl CdpConnection {
engine,
page_reservation,
load_inputs,
unreachable_url: network_error.unreachable_url.clone(),
request_method: network_error.request_method.clone(),
request_headers: network_error.request_headers.clone(),
error_text: network_error.kind.error_text().to_owned(),
unreachable_url,
request_method,
request_headers,
error_text,
})
}
@@ -201,6 +263,7 @@ impl CdpConnection {
&& !response_status_has_no_content(head.status))
.then(|| self.reserve_renderer_page_for_owner(owner, &load_inputs, &engine));
PreparedCapturedNavigationResponse {
reserved_service_worker_client: None,
engine,
page_reservation,
load_inputs,
+31 -8
View File
@@ -4,14 +4,13 @@ use serde_json::{Value, json};
use url::Url;
use super::super::fetch_support::{
ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, DocumentBodySource,
InFlightSubresourceFetchRequest, PausedDocumentTransfer, PausedDocumentTransfers,
PendingDocumentBodyRead, PendingFetchAuthNavigation, PendingFetchNavigation,
PendingFetchResponseBodyStreamRead, PendingFetchResponseBodyStreamReadStart,
PendingSubresourceFetchAuthRequest, PendingSubresourceFetchOwnerKind,
PendingSubresourceFetchRequest, PendingSubresourceFetchResponseRequest,
ResponseStageUrlMatchPolicy, fetch_subresource_interception_config_for_patterns,
matching_fetch_pattern,
ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, InFlightSubresourceFetchRequest,
PausedDocumentTransfer, PausedDocumentTransfers, PendingDocumentBodyRead,
PendingFetchAuthNavigation, PendingFetchNavigation, PendingFetchResponseBodyStreamRead,
PendingFetchResponseBodyStreamReadStart, PendingSubresourceFetchAuthRequest,
PendingSubresourceFetchOwnerKind, PendingSubresourceFetchRequest,
PendingSubresourceFetchResponseRequest, ResponseStageUrlMatchPolicy,
fetch_subresource_interception_config_for_patterns, matching_fetch_pattern,
};
pub(super) use super::super::fetch_support::{
FetchInterceptionPattern, FetchRequestStage, FetchResourceTypeFilter,
@@ -20,6 +19,8 @@ use super::DocumentNavigationToken;
use super::navigation_outcome::NavigationDispatchState;
use super::runtime_slot::TargetRuntimeSlot;
use crate::automation::{DevToolsNetworkInterceptId, DevToolsNetworkResourceType};
#[cfg(test)]
use crate::conn::DocumentBodySource;
use moli_fetch::url_pattern_matches;
#[derive(Debug, Default)]
@@ -293,6 +294,12 @@ impl TargetFetchState {
}
fn prune_request_stage_chains_for_session(&mut self, session_id: Option<&str>) {
for pending in self.pending_fetch_navigations.values_mut() {
pending.remaining_request_stages.retain(|stage| {
!stage.owner_kind.drains_on_fetch_disable()
|| stage.session_id.as_deref() != session_id
});
}
for pending in self.pending_subresource_fetches.values_mut() {
if let Some(chain) = pending.request_stage_chain.as_mut() {
chain.remaining_sessions.retain(|stage| {
@@ -561,6 +568,7 @@ impl TargetFetchState {
.insert(request_id, pending);
}
#[cfg(test)]
pub(crate) fn register_pending_fetch_response_navigation(
&mut self,
request_id: String,
@@ -1726,6 +1734,19 @@ impl TargetFetchSubresourceInterceptionSnapshot {
matching_auth_required_network_intercepts(&self.network_intercepts, url)
}
pub(crate) fn has_request_stage_candidate(
&self,
resource_type: DevToolsNetworkResourceType,
) -> bool {
self.enabled
&& self.patterns.iter().any(|pattern| {
pattern.request_stage == FetchRequestStage::Request
&& pattern
.resource_type_filter
.is_none_or(|filter| filter.matches_resource_type(resource_type))
})
}
pub(crate) fn has_response_stage_candidate(
&self,
resource_type: DevToolsNetworkResourceType,
@@ -1978,6 +1999,7 @@ impl TargetFetchOwner {
.register_pending_fetch_auth_navigation(request_id, pending);
}
#[cfg(test)]
pub(crate) fn register_pending_fetch_response_navigation(
&mut self,
request_id: String,
@@ -2548,6 +2570,7 @@ mod tests {
request_id: Some(format!("NETWORK-{request_id}")),
loader_id: "LOADER-1".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: test_url("auth-page"),
request_method: "GET".to_owned(),
request_body: None,
@@ -11,6 +11,10 @@ fn navigation(token: &DocumentNavigationToken, id: &str) -> NavigationDispatchSt
fn request(token: &DocumentNavigationToken, id: &str) -> PendingFetchNavigation {
PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: id.to_owned(),
interception_session_id: Some("SID-A".to_owned()),
@@ -374,6 +374,7 @@ pub struct NavigationDispatchState {
pub request_id: Option<String>,
pub loader_id: String,
pub request_announced: bool,
pub(crate) announced_redirects: usize,
pub requested_url: Url,
pub request_method: String,
/// Text projection used by CDP request events and Fetch interception.
+16 -4
View File
@@ -441,6 +441,10 @@ impl TargetRuntimeSlot {
target_id: String,
loader_id: String,
) -> DocumentNavigationToken {
if let Some(previous) = self.current_document_loader_id().map(str::to_owned) {
self.network_agent
.clear_fetch_pause_announced_request_id(&previous);
}
self.devtools_renderer_channel.reopen_after_target_crash();
let token = self
.page_slot
@@ -929,14 +933,14 @@ impl TargetRuntimeSlot {
.record_fetch_pause_announced_request_id(request_id);
}
pub(crate) fn replace_child_document_fetch_request_id(
pub(crate) fn replace_navigation_fetch_request_id(
&mut self,
network_id: String,
session_id: Option<String>,
fetch_id: String,
) -> Option<String> {
self.network_agent
.replace_child_document_fetch_request_id(network_id, session_id, fetch_id)
.replace_navigation_fetch_request_id(network_id, session_id, fetch_id)
}
pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool {
@@ -975,14 +979,22 @@ impl TargetRuntimeSlot {
&mut self,
token: &DocumentNavigationToken,
) -> bool {
self.page_slot
.commit_pending_document_navigation_if_matches(token)
let committed = self
.page_slot
.commit_pending_document_navigation_if_matches(token);
if committed {
self.network_agent
.clear_fetch_pause_announced_request_id(&token.loader_id);
}
committed
}
pub(crate) fn clear_pending_document_navigation_if_loader_matches(
&mut self,
loader_id: &str,
) -> bool {
self.network_agent
.clear_fetch_pause_announced_request_id(loader_id);
self.page_slot
.clear_pending_document_navigation_if_loader_matches(loader_id)
}
+5
View File
@@ -41,6 +41,7 @@ fn test_navigation_dispatch_state(fetch_request_id: &str) -> NavigationDispatchS
request_id: Some(format!("NETWORK-{fetch_request_id}")),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("https://example.test/").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -812,6 +813,10 @@ fn target_fetch_owner_reports_matching_auth_required_network_intercepts() {
fn pending_navigation_rejects_generic_request_action_without_consuming_id() {
let mut state = TargetFetchState::default();
state.register_pending_fetch_navigation_request(PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: "FETCH-NAV".to_owned(),
interception_session_id: Some("SID-fetch".to_owned()),
+14
View File
@@ -322,6 +322,20 @@ impl ProtocolWorkflowState<'_> {
.prepare_navigation_load_error(Some(&token), &navigation, error)
}
pub(crate) fn prepare_fetch_navigation_redirect(
self,
pending: super::PendingFetchNavigation,
executor: moli_core::network::NavigationInterception,
prior: moli_fetch::NetworkObservationJournal,
) -> crate::domains::fetch::RedirectNavigation {
crate::domains::fetch::prepare_navigation_redirect(
self.connection,
pending,
executor,
prior,
)
}
pub(crate) fn accept_fetch_navigation_response(
self,
pending: super::PendingFetchNavigation,
+1
View File
@@ -1904,6 +1904,7 @@ fn materialized_navigation_test_state(
request_id: Some(loader_id.to_owned()),
loader_id: loader_id.to_owned(),
request_announced: true,
announced_redirects: 0,
requested_url: Url::parse(requested_url).unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -234,7 +234,7 @@ async fn prepared_navigation_request_freezes_policy_and_cancels_before_headers_o
ctx.process_async(json!({"id": 11, "sessionId": "SID-1", "method": "Network.setExtraHTTPHeaders", "params": {"headers": {"X-Snapshot": "after"}}})).await;
ctx.expect_result(11, json!({}), Some("SID-1"));
let result = tokio::time::timeout(std::time::Duration::from_secs(30), async {
let (result, ()) = tokio::join!(job.wait(), async {
let (result, ()) = tokio::join!(job.fetch_raw_stream(), async {
requested_rx.await.unwrap();
ctx.with_fixture(|conn| {
conn.start_document_navigation_for_owner(&navigation.owner, "replacement".into())
@@ -269,7 +269,7 @@ async fn dropping_prepared_navigation_header_wait_cancels_its_transport() {
.unwrap()
});
tokio::time::timeout(std::time::Duration::from_secs(30), async {
let mut wait = Box::pin(job.wait());
let mut wait = Box::pin(job.fetch_raw_stream());
tokio::select! {
result = &mut wait => panic!("headers must stay held: {result:?}"),
requested = requested_rx => requested.unwrap(),
@@ -580,6 +580,7 @@ pub(super) fn navigation_fixture() -> (
request_id: Some("LID-test".to_owned()),
loader_id: "LID-test".to_owned(),
request_announced: true,
announced_redirects: 0,
requested_url: Url::parse("https://original.example/document").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -1014,14 +1015,14 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context(
let buffered_error = async {
let request =
ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?;
request.wait_buffered().await
request.fetch_buffered().await
}
.await
.expect_err("offline response fetch must fail");
let streaming_error = async {
let request =
ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?;
request.wait().await
request.fetch_raw_stream().await
}
.await
.expect_err("offline streaming response fetch must fail");
@@ -1038,7 +1039,7 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context(
}),
)
})?;
request.wait_buffered().await
request.fetch_buffered().await
}
.await
.expect_err("offline authenticated response fetch must fail");
@@ -119,6 +119,7 @@ async fn buffered_navigation_for_inactive_session_retains_its_target_engine() {
request_id: Some("LOADER-target".to_owned()),
loader_id: "LOADER-target".to_owned(),
request_announced: true,
announced_redirects: 0,
requested_url: requested_url.clone(),
request_method: "GET".to_owned(),
request_body: None,
@@ -780,6 +780,7 @@ mod tests {
request_id: Some("REQ-1".to_owned()),
loader_id: "LID-1".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("https://example.test/start").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
+4 -1
View File
@@ -10,6 +10,8 @@ mod document_navigation;
mod document_request;
mod helpers;
mod navigation;
mod navigation_interception;
pub(crate) use navigation_interception::prepare_navigation_request_pause;
mod params;
mod patterns;
mod state;
@@ -43,7 +45,8 @@ pub(crate) use crate::conn::PendingFetchAuthNavigation;
#[cfg(test)]
pub(crate) use crate::conn::PendingFetchNavigation;
pub(crate) use document_request::{
FetchNavigationResponse, PreparedFetchNavigationResponse, accept_navigation_response,
FetchNavigationResponse, PreparedFetchNavigationResponse, RedirectNavigation,
accept_navigation_response, prepare_navigation_redirect,
};
#[cfg(test)]
pub(crate) use helpers::encode_basic_auth;
+8
View File
@@ -51,6 +51,10 @@ pub(super) fn navigation_for_auth_retry(
pending: crate::conn::PendingFetchAuthNavigation,
) -> PendingFetchNavigation {
PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: pending.auth_retry_budget,
fetch_request_id: pending.response_stage_request_id,
interception_session_id: pending.interception_session_id,
@@ -75,6 +79,10 @@ pub(super) fn canceled_auth_response(
};
(
PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: pending.auth_retry_budget,
fetch_request_id: pending.response_stage_request_id,
interception_session_id: pending.interception_session_id,
+17 -1
View File
@@ -403,6 +403,21 @@ fn start_devtools_continue_intercepted_request_command(
pending.navigation.request_load_policy,
None,
);
if !pending.remaining_request_stages.is_empty() {
let stage = pending.remaining_request_stages.remove(0);
pending.fetch_request_id = stage.request_id;
pending.interception_session_id = stage.session_id;
pending.redirected_request_id = stage.redirected_request_id;
let event = super::request_paused_background_event(
pending.interception_session_id.as_deref(),
&pending,
);
let navigation_owner = pending.navigation.owner.clone();
conn.register_pending_fetch_navigation_request_for_owner(&navigation_owner, pending);
let mut plan = CommandOutputPlan::success();
plan.push_background_event(event);
return FetchCommandTaskStep::Complete(plan);
}
let termination = super::navigation::NavigationTermination::new(
conn,
owner,
@@ -1560,6 +1575,7 @@ pub(super) fn continue_streaming_document_response_in_background(
response_headers: Vec<(String, Vec<u8>)>,
) {
let PendingStreamingDocumentResponseNavigation {
reserved_service_worker_client,
document_navigation_token,
navigation,
response,
@@ -1575,7 +1591,7 @@ pub(super) fn continue_streaming_document_response_in_background(
response_headers,
body_progress_source,
) {
Ok(job) => job,
Ok(job) => job.with_service_worker_client(reserved_service_worker_client),
Err(BackgroundNavigationClaimError::AlreadyClaimed) => return,
Err(BackgroundNavigationClaimError::Stale) => {
let _ = sender.send(page::BackgroundNavigationCompletion::new(
@@ -498,17 +498,23 @@ pub(super) fn start_fulfill(
response_code,
response_headers,
decoded_body,
} => conn.prepare_synthetic_navigation_response(
Some(&token),
&navigation,
navigation.requested_url.clone(),
response_code,
response_headers,
crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body(decoded_body),
pending.request_cookie_report,
Default::default(),
network::MainDocumentBodyProgressSource::default(),
),
} => {
let client = pending.reserved_service_worker_client;
conn.prepare_synthetic_navigation_response(
Some(&token),
&navigation,
navigation.requested_url.clone(),
response_code,
response_headers,
crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body(
decoded_body,
),
pending.request_cookie_report,
Default::default(),
network::MainDocumentBodyProgressSource::default(),
)
.map(|response| response.with_service_worker_client(client))
}
DocumentFulfillment::ResponseTransfer {
transfer,
response_code,
@@ -1,13 +1,14 @@
use anyhow::Result;
use anyhow::{Context, Result};
use crate::conn::{
CdpConnection, NavigationLoadOutcome, PendingFetchNavigation,
PendingStreamingDocumentResponseNavigation, PreparedCapturedNavigationResponse,
PreparedNavigationRequest, ProtocolWorkflowDriver,
ProtocolWorkflowDriver,
};
use crate::domains::{
command_output::CommandOutputBuffer, command_output::CommandOutputPlan, network,
};
use moli_core::network::NavigationInterception;
use moli_core::page::SubresourceAuthCredentials;
use moli_fetch::{
NetworkFetchResult, NetworkObservationJournal, RawResponse, StreamingRawResponse,
@@ -23,7 +24,8 @@ use super::{
/// awaits; only interpreting the accepted response needs another state turn.
pub(crate) struct PreparedInterceptedNavigationRequest {
pending: PendingFetchNavigation,
request: Option<Result<PreparedNavigationRequest>>,
request: Option<Result<NavigationInterception>>,
renderer_runtime: Option<moli_core::runtime::RendererBrowserContextRuntime>,
buffered: bool,
handle_auth: bool,
prior_journal: Option<NetworkObservationJournal>,
@@ -62,6 +64,11 @@ impl PreparedInterceptedNavigationRequest {
&pending.navigation.requested_url,
);
let buffered = !navigation::navigation_response_stage_auth_can_stream(auth.as_ref());
let mut prior_journal = prior_journal;
let renderer_runtime = conn
.target_owner_identity_for_owner(&pending.navigation.owner)
.and_then(|(id, _)| conn.browser_context_by_id(&id))
.map(|context| context.renderer_runtime());
let request = if auth.is_some() && !pending.auth_retry_budget.try_restart() {
None
} else if handle_auth && buffered && pending.intercept_response {
@@ -73,7 +80,24 @@ impl PreparedInterceptedNavigationRequest {
.into()))
} else {
network::record_main_document_request_body(conn, &pending.navigation);
Some(
let prepared = if let Some((mut interception, journal)) = pending.interception.take() {
prior_journal = Some(navigation::merge_network_observations(
prior_journal,
journal,
));
let request = interception.request_mut();
request.url = pending.navigation.requested_url.clone();
request.method = pending.navigation.request_method.clone();
request.request_headers = pending.navigation.request_headers.clone();
request.body = pending.navigation.clone_request_body_bytes();
*request = request
.clone()
.with_redirect_headers(pending.navigation.redirect_headers.clone());
if let Some(auth) = auth {
request.set_auth(Some(auth.into()));
}
Ok(interception)
} else {
conn.prepare_navigation_request(
pending
.document_navigation_token
@@ -81,12 +105,26 @@ impl PreparedInterceptedNavigationRequest {
.expect("request token"),
&pending.navigation,
auth,
),
)
)
};
Some(prepared.map(|mut prepared| {
let intercept = conn
.target_fetch_subresource_interception_snapshot_for_owner(
&pending.navigation.owner,
)
.is_some_and(|snapshot| {
snapshot.has_request_stage_candidate(
crate::automation::DevToolsNetworkResourceType::Document,
)
});
prepared.set_interception(intercept);
prepared
}))
};
Self {
pending,
request,
renderer_runtime,
buffered,
handle_auth,
prior_journal,
@@ -95,43 +133,134 @@ impl PreparedInterceptedNavigationRequest {
}
pub(crate) async fn prepare(
self,
mut self,
driver: &mut ProtocolWorkflowDriver,
) -> PreparedFetchNavigationResponse {
let Some(request) = self.request else {
let Some(request) = self.request.take() else {
return PreparedFetchNavigationResponse::Result(Ok(
NavigationLoadOutcome::network_failure(
moli_fetch::AuthRetryBudget::ERROR_TEXT.to_owned(),
),
));
};
let response = match request {
Ok(request) if self.buffered => request
.wait_buffered()
.await
.map(FetchNavigationResponse::Captured),
Ok(request) => match request.wait().await {
Ok(response)
if moli_web_mime::response_headers_indicate_attachment_download(
&response.response().headers,
) || (self.handle_auth
&& matches!(response.response().status, 401 | 407)
&& extract_auth_challenge(&response.response().headers).is_some()) =>
let mut request = match request {
Ok(request) => request,
Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)),
};
let response = loop {
let service_worker_response = if let Some(runtime) = &self.renderer_runtime {
match runtime
.fetch_service_worker_main_resource_for_navigation(
request.request(),
request.resource_loader(),
)
.await
{
let (response, journal) = response.into_parts_with_observation_journal();
response
.into_materialized_raw_response()
.await
.map(|response| {
FetchNavigationResponse::Captured(
NetworkFetchResult::with_observation_journal(response, journal),
Ok(fetch) => {
self.pending.reserved_service_worker_client = fetch.reserved_client;
fetch.response
}
Err(error) => break Err(error),
}
} else {
None
};
let response = if self.buffered && service_worker_response.is_none() {
request
.fetch_buffered()
.await
.with_context(|| format!("failed to fetch page `{}`", request.request().url))
.map(FetchNavigationResponse::Captured)
} else {
let response = if let Some(response) = service_worker_response {
request
.resource_loader()
.note_service_worker_response_ready()
.and_then(|()| {
moli_core::runtime::streaming_raw_response_from_navigation_response(
response,
)
})
.map_err(|error| error.context("failed to read page body from stream"))
.map(NetworkFetchResult::without_request_observation)
} else {
request.fetch_raw_stream().await
};
match response
.with_context(|| format!("failed to fetch page `{}`", request.request().url))
{
Ok(response)
if moli_web_mime::response_headers_indicate_attachment_download(
&response.response().headers,
) || (self.handle_auth
&& matches!(response.response().status, 401 | 407)
&& extract_auth_challenge(&response.response().headers).is_some()) =>
{
let (response, journal) = response.into_parts_with_observation_journal();
response
.into_materialized_raw_response()
.await
.map(|response| {
FetchNavigationResponse::Captured(
NetworkFetchResult::with_observation_journal(response, journal),
)
})
.map_err(|error| error.context("failed to read page body from stream"))
}
response => response.map(FetchNavigationResponse::Streaming),
}
response => response.map(FetchNavigationResponse::Streaming),
},
Err(error) => Err(error),
};
let head = match &response {
Ok(FetchNavigationResponse::Streaming(response)) => {
Some(response.response().head())
}
Ok(FetchNavigationResponse::Captured(response)) => Some(response.response().head()),
Err(_) => None,
};
match head.as_ref().map(|head| request.follow_redirect(head)) {
Some(Ok(Some(_))) => {}
Some(Err(error)) => break Err(error),
_ => break response,
}
let journal = match response.expect("redirect has a response") {
FetchNavigationResponse::Streaming(response) => {
response.into_parts_with_observation_journal().1
}
FetchNavigationResponse::Captured(response) => {
response.into_parts_with_observation_journal().1
}
};
self.prior_journal = Some(navigation::merge_network_observations(
self.prior_journal.take(),
journal,
));
let hop = request.request();
self.pending.navigation.requested_url = hop.url.clone();
self.pending.navigation.request_method = hop.method.clone();
self.pending.navigation.request_headers = hop.request_headers.clone();
self.pending.navigation.request_body_bytes = hop.body.clone();
self.pending.navigation.request_body = hop
.body
.as_ref()
.map(|body| String::from_utf8_lossy(body).into_owned());
self.pending.navigation.redirect_chain = hop.redirect_chain().to_vec();
self.pending.navigation.redirect_headers = None;
let pending = self.pending;
let executor = request.clone();
let prior = self.prior_journal.clone().unwrap_or_default();
let redirected = driver.with_state(move |state| {
state.prepare_fetch_navigation_redirect(pending, executor, prior)
});
match redirected {
Ok(RedirectNavigation::Continue(pending)) => self.pending = *pending,
Ok(RedirectNavigation::Paused(plan)) => {
self.transferred.mark_transferred();
return PreparedFetchNavigationResponse::Transferred(*plan);
}
Ok(RedirectNavigation::Superseded) => {
return PreparedFetchNavigationResponse::Superseded;
}
Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)),
}
};
let result = driver.with_state(move |state| {
let result = state.accept_fetch_navigation_response(
@@ -219,7 +348,12 @@ pub(crate) fn accept_navigation_response(
),
journal,
network::MainDocumentBodyProgressSource::default(),
),
)
.map(|response| {
response.with_service_worker_client(
pending.reserved_service_worker_client.take(),
)
}),
));
}
}
@@ -248,7 +382,12 @@ pub(crate) fn accept_navigation_response(
network::MainDocumentBodyProgressSource::default(),
None,
Vec::new(),
),
)
.map(|response| {
response.with_service_worker_client(
pending.reserved_service_worker_client.take(),
)
}),
));
};
let (response, network_observation_journal) =
@@ -257,6 +396,9 @@ pub(crate) fn accept_navigation_response(
conn,
sender,
PendingStreamingDocumentResponseNavigation {
reserved_service_worker_client: pending
.reserved_service_worker_client
.take(),
document_navigation_token: pending
.document_navigation_token
.expect("request token"),
@@ -273,3 +415,103 @@ pub(crate) fn accept_navigation_response(
}
PreparedFetchNavigationResponse::Transferred(output.into_plan())
}
pub(crate) enum RedirectNavigation {
Continue(Box<PendingFetchNavigation>),
Paused(Box<crate::domains::command_output::CommandOutputPlan>),
Superseded,
}
pub(crate) fn prepare_navigation_redirect(
conn: &mut CdpConnection,
mut pending: PendingFetchNavigation,
executor: moli_core::network::NavigationInterception,
prior: NetworkObservationJournal,
) -> RedirectNavigation {
if !pending
.document_navigation_token
.as_ref()
.is_some_and(|token| {
conn.accepts_pending_document_navigation_for_owner(&pending.navigation.owner, token)
})
{
return RedirectNavigation::Superseded;
}
let Some((id, _)) =
conn.allocate_pending_subresource_fetch_request_ids_for_owner(&pending.navigation.owner)
else {
return RedirectNavigation::Superseded;
};
pending.fetch_request_id = id;
// The same selector and per-handler hop IDs serve root and child Documents.
let paused = super::prepare_navigation_request_pause(conn, &mut pending);
if !paused {
return RedirectNavigation::Continue(Box::new(pending));
}
pending.interception = Some((executor, prior));
let owner = pending.navigation.owner.clone();
let event = super::request_paused_background_event(
pending.interception_session_id.as_deref(),
&pending,
);
let mut plan = crate::domains::command_output::CommandOutputPlan::default();
append_redirect_request_events(conn, &mut plan, &mut pending);
if conn
.register_pending_fetch_navigation_request_for_owner(&owner, pending)
.is_none()
{
return RedirectNavigation::Superseded;
}
plan.push_background_event(event);
RedirectNavigation::Paused(Box::new(plan))
}
fn append_redirect_request_events(
conn: &CdpConnection,
plan: &mut crate::domains::command_output::CommandOutputPlan,
pending: &mut PendingFetchNavigation,
) {
let nav = &pending.navigation;
let Some(network_id) = &nav.request_id else {
return;
};
let info = moli_core::page::PendingSubresourceFetchInfo {
internal_id: 0,
network_request_handle: None,
frame_id: Some(nav.frame_id.clone()),
document_url: nav.requested_url.clone(),
url: nav.requested_url.clone(),
websocket_socket_id: None,
method: nav.request_method.clone(),
request_headers: nav.request_headers.clone(),
request_body: nav.request_body.clone(),
request_body_bytes: nav.clone_request_body_bytes(),
resource_type: moli_core::page::SubresourceResourceType::Document,
request_cookie_report: pending.request_cookie_report.clone(),
};
for redirect in nav.redirect_chain.iter().skip(nav.announced_redirects) {
let mut info = info.clone();
info.url = redirect.to_url.clone();
let output = network::TargetSubresourceFetchPauseNetworkOutput::from_pending_fetch_info(
network_id.clone(),
nav.frame_id.clone(),
nav.loader_id.clone(),
crate::conn::monotonic_timestamp_seconds(),
nav.requested_url.clone(),
&info,
)
.with_child_document(Some(moli_core::page::ChildDocumentFetchRequest {
loader_id: nav.loader_id.clone(),
redirect: Some(redirect.clone().into()),
}));
for session in conn.network_event_session_ids_for_owner(&nav.owner) {
plan.extend_background_events(
network::fetch_subresource_initial_request_network_events(
session.as_deref(),
&output,
),
);
}
}
pending.navigation.announced_redirects = pending.navigation.redirect_chain.len();
}
+5 -2
View File
@@ -17,7 +17,7 @@ use moli_core::page::{
SubresourceAuthChallenge, SubresourceAuthCredentials, extract_subresource_auth_challenge,
subresource_auth_credentials_for_challenge,
};
use serde_json::Value;
use serde_json::{Value, json};
use url::Url;
use super::params::HeaderEntry;
@@ -518,7 +518,10 @@ pub(crate) fn request_paused_background_event(
.map(DevToolsRequestId::from),
auth_challenge: None,
};
let payload = network::fetch_request_paused_params(&network_event);
let mut payload = network::fetch_request_paused_params(&network_event);
if let Some(previous) = &pending.redirected_request_id {
payload["redirectedRequestId"] = json!(previous);
}
let message = build_event("Fetch.requestPaused", payload, session_id);
BackgroundProtocolEvent::immediate_automation_event(
message,
+40 -23
View File
@@ -143,7 +143,7 @@ pub(super) fn navigation_response_stage_auth_can_stream(
pub(super) fn pause_streaming_response_stage_navigation_into_buffer(
conn: &mut CdpConnection,
out: &mut CommandOutputBuffer,
pending: PendingFetchNavigation,
mut pending: PendingFetchNavigation,
response: NetworkFetchResult<StreamingRawResponse>,
) {
let response_head = response.response();
@@ -176,19 +176,23 @@ pub(super) fn pause_streaming_response_stage_navigation_into_buffer(
// here would wait on the old document's debugger pause before we could even
// publish Fetch.requestPaused. Continue/fulfill owns document replacement.
let (response, network_observation_journal) = response.into_parts_with_observation_journal();
conn.register_pending_fetch_response_navigation_for_owner(
conn.register_pending_fetch_response_transfer_for_owner(
&pending.navigation.owner,
pending.fetch_request_id.clone(),
pending.document_navigation_token.clone(),
pending.navigation.clone(),
DocumentBodySource::StreamingRaw {
requested_url: pending.navigation.requested_url.clone(),
request_method: pending.navigation.request_method.clone(),
request_headers: pending.navigation.request_headers.clone(),
response,
network_observation_journal,
body_progress_source,
},
crate::conn::PausedDocumentTransfer::pending(
pending.fetch_request_id.clone(),
pending.document_navigation_token.clone(),
pending.navigation.clone(),
DocumentBodySource::StreamingRaw {
requested_url: pending.navigation.requested_url.clone(),
request_method: pending.navigation.request_method.clone(),
request_headers: pending.navigation.request_headers.clone(),
response,
network_observation_journal,
body_progress_source,
},
)
.with_service_worker_client(pending.reserved_service_worker_client.take()),
);
out.extend_background_events_after_messages([navigation_response_stage_request_paused_event(
pending.interception_session_id.as_deref(),
@@ -244,7 +248,7 @@ pub(super) fn pause_data_url_response_stage_navigation_into_buffer(
pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
conn: &mut CdpConnection,
out: &mut CommandOutputBuffer,
pending: PendingFetchNavigation,
mut pending: PendingFetchNavigation,
response: NetworkFetchResult<RawResponse>,
) {
let response_status = response.response().status;
@@ -252,18 +256,22 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
let response_headers = response.response().headers.clone();
let request_cookie_report = response.response().request_cookie_report.clone();
let (response, network_observation_journal) = response.into_parts_with_observation_journal();
conn.register_pending_fetch_response_navigation_for_owner(
conn.register_pending_fetch_response_transfer_for_owner(
&pending.navigation.owner,
pending.fetch_request_id.clone(),
pending.document_navigation_token.clone(),
pending.navigation.clone(),
DocumentBodySource::BufferedRaw {
requested_url: pending.navigation.requested_url.clone(),
request_method: pending.navigation.request_method.clone(),
request_headers: pending.navigation.request_headers.clone(),
response,
network_observation_journal,
},
crate::conn::PausedDocumentTransfer::pending(
pending.fetch_request_id.clone(),
pending.document_navigation_token.clone(),
pending.navigation.clone(),
DocumentBodySource::BufferedRaw {
requested_url: pending.navigation.requested_url.clone(),
request_method: pending.navigation.request_method.clone(),
request_headers: pending.navigation.request_headers.clone(),
response,
network_observation_journal,
},
)
.with_service_worker_client(pending.reserved_service_worker_client.take()),
);
out.extend_background_events_after_messages([navigation_response_stage_request_paused_event(
pending.interception_session_id.as_deref(),
@@ -276,6 +284,15 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
)]);
}
pub(super) fn merge_network_observations(
prior: Option<NetworkObservationJournal>,
current: NetworkObservationJournal,
) -> NetworkObservationJournal {
let mut journal = prior.unwrap_or_default();
journal.append(current);
journal
}
pub(super) fn append_prior_network_observations<R>(
response: NetworkFetchResult<R>,
prior: Option<NetworkObservationJournal>,
@@ -0,0 +1,107 @@
use std::collections::HashMap;
use crate::automation::DevToolsNetworkResourceType;
use crate::conn::{
CdpConnection, CommandOwnerScope, PendingFetchNavigation, PendingSubresourceFetchOwnerKind,
PendingSubresourceFetchRequestStage, TargetFetchSubresourceInterceptionSnapshot,
};
type NavigationHopIds = HashMap<Option<String>, (String, Option<String>)>;
/// Every Fetch handler advances its own hop identity, including unmatched
/// hops. The transport's navigation identity is independent of these IDs.
pub(super) fn navigation_hop_ids(
conn: &mut CdpConnection,
owner: &CommandOwnerScope,
snapshot: &TargetFetchSubresourceInterceptionSnapshot,
network_id: &str,
first_id: &str,
redirected: bool,
) -> Option<NavigationHopIds> {
let mut ids = HashMap::new();
for (index, session_id) in snapshot
.fetch_session_ids(owner.session_id())
.into_iter()
.enumerate()
{
let id = if index == 0 {
first_id.to_owned()
} else {
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)?
.0
};
let previous = conn
.runtime_session_owner_slot_mut_for_owner(owner)
.and_then(|slot| {
slot.replace_navigation_fetch_request_id(
network_id.to_owned(),
session_id.clone(),
id.clone(),
)
});
ids.insert(session_id, (id, previous.filter(|_| redirected)));
}
Some(ids)
}
pub(crate) fn prepare_navigation_request_pause(
conn: &mut CdpConnection,
pending: &mut PendingFetchNavigation,
) -> bool {
let owner = &pending.navigation.owner;
let Some(snapshot) = conn.target_fetch_subresource_interception_snapshot_for_owner(owner)
else {
return false;
};
let redirected = !pending.navigation.redirect_chain.is_empty();
let Some(ids) = navigation_hop_ids(
conn,
owner,
&snapshot,
&pending.navigation.loader_id,
&pending.fetch_request_id,
redirected,
) else {
return false;
};
let blocked = snapshot.matching_network_intercepts(
crate::conn::FetchRequestStage::Request,
DevToolsNetworkResourceType::Document,
&pending.navigation.requested_url,
);
let mut stages = Vec::new();
for stage in snapshot.matching_request_stage_pause_sessions(
owner.session_id(),
DevToolsNetworkResourceType::Document,
&pending.navigation.requested_url,
) {
let (id, previous) = if stage.owner_kind == PendingSubresourceFetchOwnerKind::Fetch {
ids.get(&stage.session_id)
.cloned()
.unwrap_or_else(|| (pending.fetch_request_id.clone(), None))
} else {
let Some((id, _)) =
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)
else {
return false;
};
(id, None)
};
stages.push(PendingSubresourceFetchRequestStage {
session_id: stage.session_id,
owner_kind: stage.owner_kind,
request_id: id,
redirected_request_id: previous,
blocked_intercepts: blocked.clone(),
});
}
if stages.is_empty() {
return false;
}
let stage = stages.remove(0);
pending.fetch_request_id = stage.request_id;
pending.interception_session_id = stage.session_id;
pending.redirected_request_id = stage.redirected_request_id;
pending.remaining_request_stages = stages;
true
}
+11 -29
View File
@@ -193,35 +193,17 @@ fn prepare_subresource_fetch_pause_sources(
// including hops that do not match that handler's URL patterns.
let mut child_fetch_ids = HashMap::new();
if let Some(document) = child_document.as_ref() {
for (index, fetch_session_id) in fetch_snapshot
.fetch_session_ids(session_id)
.into_iter()
.enumerate()
{
let fetch_id = if index == 0 {
request_id.clone()
} else {
let Some((id, _)) =
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)
else {
return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic);
};
id
};
let previous = conn
.runtime_session_owner_slot_mut_for_owner(owner)
.and_then(|slot| {
slot.replace_child_document_fetch_request_id(
network_request_id.clone(),
fetch_session_id.clone(),
fetch_id.clone(),
)
});
child_fetch_ids.insert(
fetch_session_id,
(fetch_id, previous.filter(|_| document.redirect.is_some())),
);
}
let Some(ids) = super::navigation_interception::navigation_hop_ids(
conn,
owner,
&fetch_snapshot,
&network_request_id,
&request_id,
document.redirect.is_some(),
) else {
return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic);
};
child_fetch_ids = ids;
}
if request_stage_pause_sessions.is_empty() {
if child_document.is_some() {
@@ -1212,6 +1212,10 @@ async fn disable_clears_fetch_state() {
bc.active_page_target_mut()
.fetch_owner
.register_pending_fetch_navigation_request(PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: "INT-1".to_owned(),
interception_session_id: Some("SID-1".to_owned()),
@@ -1235,6 +1239,7 @@ async fn disable_clears_fetch_state() {
request_id: Some("REQ-1".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/auth").unwrap(),
request_method: "POST".to_owned(),
request_body: Some("payload".to_owned()),
@@ -1282,6 +1287,7 @@ async fn disable_clears_fetch_state() {
request_id: Some("REQ-1".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/auth").unwrap(),
request_method: "POST".to_owned(),
request_body: Some("payload".to_owned()),
@@ -1530,6 +1536,7 @@ async fn continue_with_auth_rejects_invalid_response_without_consuming_pending_a
request_id: Some("REQ-8".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/auth").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -1614,6 +1621,7 @@ async fn continue_with_auth_unsupported_challenge_preserves_pending_auth_navigat
request_id: Some("REQ-9".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/auth").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -1774,6 +1782,10 @@ fn extract_auth_challenge_preserves_quoted_commas_in_realm() {
#[test]
fn emit_auth_required_preserves_request_headers_and_post_data_shape() {
let pending = PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: "INT-11".to_owned(),
interception_session_id: Some("SID-1".to_owned()),
@@ -1795,6 +1807,7 @@ fn emit_auth_required_preserves_request_headers_and_post_data_shape() {
request_id: Some("REQ-11".to_owned()),
loader_id: "LID-0000000001".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/auth").unwrap(),
request_method: "POST".to_owned(),
request_body: Some("payload".to_owned()),
@@ -1867,6 +1880,7 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig
request_id: Some("REQ-retired".to_owned()),
loader_id: "LID-retired".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("https://retired.example/document").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -1913,6 +1927,10 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig
conn.register_pending_fetch_navigation_request_for_owner(
&owner,
PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id: "FETCH-retired".to_owned(),
interception_session_id: Some("SID-handler".to_owned()),
@@ -366,6 +366,34 @@ async fn navigation_header_override_expires_on_redirect_before_auth_retry() {
)
.await;
ctx.expect_result(70302, json!({}), Some("SID-1"));
crate::testing::wait_until_scheduler_message(&mut ctx, "redirect request pause", |message| {
message["method"] == "Fetch.requestPaused"
&& message["sessionId"] == "SID-1"
&& message["params"]["request"]["url"] == format!("http://{addr}/protected")
})
.await;
let redirected = ctx.take_first_matching("redirect request pause", |message| {
message["method"] == "Fetch.requestPaused"
&& message["params"]["request"]["url"] == format!("http://{addr}/protected")
});
assert_eq!(
redirected["params"]["request"]["url"],
format!("http://{addr}/protected")
);
assert_eq!(
redirected["params"]["redirectedRequestId"],
paused["params"]["requestId"]
);
assert_ne!(
redirected["params"]["requestId"],
paused["params"]["requestId"]
);
ctx.process_async(
json!({"id":70304,"sessionId":"SID-1","method":"Fetch.continueRequest",
"params":{"requestId":redirected["params"]["requestId"]}}),
)
.await;
ctx.expect_result(70304, json!({}), Some("SID-1"));
wait_until_message(&mut ctx, "SID-1", "redirected navigation auth", |message| {
message["method"] == "Fetch.authRequired"
})
@@ -176,3 +176,6 @@ mod runtime_subresources;
mod session_interception;
#[path = "runtime_fetch/worker_requests.rs"]
mod worker_requests;
#[path = "runtime_fetch/unified_navigation.rs"]
mod unified_navigation;
@@ -0,0 +1,332 @@
use super::*;
#[derive(Clone, Copy, Debug)]
enum NavigationEntry {
Main,
Frame,
Popup,
}
async fn navigation_redirect_contract(entry: NavigationEntry, fail: bool) {
let hits = Arc::new(AtomicUsize::new(0));
let observed_hits = hits.clone();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let router = Router::new()
.route(
"/page",
get(|| async {
(
[(CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><body>opener<iframe></iframe></body>",
)
}),
)
.route(
"/redirect",
get(|| async { (StatusCode::FOUND, [("location", "/destination")]) }),
)
.route(
"/destination",
get(move || {
let hits = observed_hits.clone();
async move {
hits.fetch_add(1, Ordering::SeqCst);
(
[(CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><title>destination</title><body>shared navigation</body>",
)
}
}),
);
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
let mut ctx = TestContext::new();
with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await;
enable_runtime_async(&mut ctx, "SID-1", 41_000).await;
let redirect = format!("http://{addr}/redirect");
let destination = format!("http://{addr}/destination");
let session = if matches!(entry, NavigationEntry::Popup) {
ctx.enable_background_navigation_scheduler_for_test();
ctx.process_async(json!({"id":41_001,"method":"Target.setAutoAttach",
"params":{"autoAttach":true,"waitForDebuggerOnStart":true,"flatten":true}}))
.await;
ctx.expect_result(41_001, json!({}), None);
let (_, session) =
open_auto_attached_popup_from_session(&mut ctx, 41_002, "SID-1", &redirect).await;
ctx.process_async(json!({"id":41_003,"method":"Network.enable","sessionId":session}))
.await;
ctx.expect_result(41_003, json!({}), Some(&session));
ctx.process_async(json!({"id":41_004,"method":"Page.enable","sessionId":session}))
.await;
ctx.expect_result(41_004, json!({}), Some(&session));
session
} else {
"SID-1".to_owned()
};
ctx.sent.clear();
ctx.process_async(
json!({"id":41_005,"method":"Fetch.enable","sessionId":session,
"params":{"patterns":[{"urlPattern":"*/destination","resourceType":"Document"}]}}),
)
.await;
ctx.expect_result(41_005, json!({}), Some(&session));
match entry {
NavigationEntry::Main => ctx.process_async(json!({"id":41_006,"method":"Page.navigate","sessionId":session,"params":{"url":redirect}})).await,
NavigationEntry::Frame => ctx.process_async(json!({"id":41_006,"method":"Runtime.evaluate","sessionId":session,"params":{"expression":"document.querySelector('iframe').src = '/redirect'"}})).await,
NavigationEntry::Popup => ctx.process_async(json!({"id":41_006,"method":"Runtime.runIfWaitingForDebugger","sessionId":session})).await,
}
crate::testing::wait_until_scheduler_message(
&mut ctx,
"destination redirect must pause for every navigation entry",
|event| {
event["method"] == "Fetch.requestPaused"
&& event["sessionId"] == session
&& event["params"]["request"]["url"] == destination
&& event["params"]["resourceType"] == "Document"
},
)
.await;
let paused = ctx
.sent
.iter()
.find(|event| {
event["method"] == "Fetch.requestPaused"
&& event["sessionId"] == session
&& event["params"]["request"]["url"] == destination
})
.unwrap()
.clone();
assert_eq!(
hits.load(Ordering::SeqCst),
0,
"{entry:?} sent a redirect destination before interception"
);
assert!(
paused["params"]["redirectedRequestId"].is_string(),
"{entry:?} must retain the hidden first-hop Fetch ID: {paused}"
);
let network_id = paused["params"]["networkId"].as_str().unwrap().to_owned();
assert!(
ctx.sent
.iter()
.any(|event| event["method"] == "Network.requestWillBeSent"
&& event["params"]["requestId"] == network_id
&& event["params"]["request"]["url"] == destination
&& event["params"]["redirectResponse"]["status"] == 302),
"{entry:?}: redirect Network event must precede pause: {:?}",
ctx.sent
);
let (method, params) = if fail {
(
"Fetch.failRequest",
json!({"requestId":paused["params"]["requestId"],"errorReason":"BlockedByClient"}),
)
} else {
(
"Fetch.continueRequest",
json!({"requestId":paused["params"]["requestId"]}),
)
};
ctx.process_async(json!({"id":41_007,"method":method,"sessionId":session,"params":params}))
.await;
ctx.expect_result(41_007, json!({}), Some(&session));
wait_until_message(
&mut ctx,
session.as_str(),
"one navigation Network terminal",
|event| {
matches!(
event["method"].as_str(),
Some("Network.loadingFinished" | "Network.loadingFailed")
) && event["params"]["requestId"] == network_id
},
)
.await;
let terminals = ctx
.sent
.iter()
.filter(|event| {
matches!(
event["method"].as_str(),
Some("Network.loadingFinished" | "Network.loadingFailed")
) && event["params"]["requestId"] == network_id
})
.collect::<Vec<_>>();
assert_eq!(terminals.len(), 1, "{entry:?}: exactly one terminal");
assert_eq!(
terminals[0]["method"],
if fail {
"Network.loadingFailed"
} else {
"Network.loadingFinished"
}
);
assert_eq!(hits.load(Ordering::SeqCst), usize::from(!fail));
let redirects = ctx
.sent
.iter()
.filter(|event| {
event["method"] == "Network.requestWillBeSent"
&& event["params"]["requestId"] == network_id
&& event["params"]["request"]["url"] == destination
})
.count();
assert_eq!(
redirects, 1,
"{entry:?}: redirect must not be announced again at completion"
);
server.abort();
}
#[tokio::test(flavor = "multi_thread")]
async fn main_navigation_uses_shared_redirect_interception() {
for fail in [true, false] {
navigation_redirect_contract(NavigationEntry::Main, fail).await;
}
}
#[tokio::test(flavor = "multi_thread")]
async fn iframe_navigation_uses_shared_redirect_interception() {
for fail in [true, false] {
navigation_redirect_contract(NavigationEntry::Frame, fail).await;
}
}
#[tokio::test(flavor = "multi_thread")]
async fn popup_navigation_uses_shared_redirect_interception() {
tokio::task::LocalSet::new()
.run_until(async {
for fail in [true, false] {
navigation_redirect_contract(NavigationEntry::Popup, fail).await;
}
})
.await;
}
async fn service_worker_navigation_contract(stage: Option<&str>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(
listener,
Router::new()
.route(
"/page",
get(|| async {
(
[(CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><body>register worker</body>",
)
}),
)
.route(
"/worker.js",
get(|| async {
(
[(CONTENT_TYPE.as_str(), "text/javascript")],
r#"
self.addEventListener('install', event => event.waitUntil(self.skipWaiting()));
self.addEventListener('activate', event => event.waitUntil(clients.claim()));
self.addEventListener('fetch', event => {
if (new URL(event.request.url).pathname === '/controlled') {
event.respondWith(new Response('<!doctype html><body>service worker navigation</body>', {
headers: {'Content-Type': 'text/html'}
}));
}
});
"#,
)
}),
)
.route(
"/controlled",
get(|| async {
(
[(CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><body>network navigation</body>",
)
}),
),
)
.await
.unwrap();
});
let mut ctx = TestContext::new();
with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await;
enable_runtime_async(&mut ctx, "SID-1", 42_000).await;
ctx.process_async(json!({"id":42_001,"sessionId":"SID-1","method":"Runtime.evaluate",
"params":{"expression":"(async () => { await navigator.serviceWorker.register('/worker.js'); await navigator.serviceWorker.ready; return true; })()", "awaitPromise":true,"returnByValue":true}})).await;
crate::testing::wait_until_scheduler_message(
&mut ctx,
"service worker registration",
|event| event["id"] == 42_001,
)
.await;
let registered = take_response_by_id(&mut ctx, 42_001);
assert_eq!(registered["result"]["result"]["value"], true);
let pattern = match stage {
Some(stage) => {
json!({"urlPattern":"*/controlled","resourceType":"Document","requestStage":stage})
}
None => json!({"urlPattern":"*/never-matches","resourceType":"Document"}),
};
ctx.process_async(json!({"id":42_002,"sessionId":"SID-1","method":"Fetch.enable","params":{"patterns":[pattern]}})).await;
ctx.expect_result(42_002, json!({}), Some("SID-1"));
ctx.sent.clear();
ctx.process_async(
json!({"id":42_003,"sessionId":"SID-1","method":"Page.navigate",
"params":{"url":format!("http://{addr}/controlled")}}),
)
.await;
if let Some(stage) = stage {
crate::testing::wait_until_scheduler_message(
&mut ctx,
"controlled navigation pause",
|event| event["method"] == "Fetch.requestPaused" && event["sessionId"] == "SID-1",
)
.await;
let pause = ctx.take_first_matching("controlled navigation pause", |event| {
event["method"] == "Fetch.requestPaused"
});
if stage == "Response" {
ctx.process_async(json!({"id":42_005,"sessionId":"SID-1","method":"Fetch.getResponseBody","params":{"requestId":pause["params"]["requestId"]}})).await;
crate::testing::wait_until_scheduler_message(
&mut ctx,
"captured controlled response body",
|event| event["id"] == 42_005,
)
.await;
let body = take_response_by_id(&mut ctx, 42_005);
assert!(body.get("result").is_some(), "{body:?}");
}
ctx.process_async(json!({"id":42_006,"sessionId":"SID-1","method":if stage == "Response" {"Fetch.continueResponse"} else {"Fetch.continueRequest"},"params":{"requestId":pause["params"]["requestId"]}})).await;
ctx.expect_result(42_006, json!({}), Some("SID-1"));
}
crate::testing::wait_until_frame_stopped_loading(&mut ctx, "TID-1").await;
ctx.process_async(
json!({"id":42_004,"sessionId":"SID-1","method":"Runtime.evaluate",
"params":{"expression":"[document.body.textContent, navigator.serviceWorker.controller !== null]","returnByValue":true}}),
)
.await;
let result = take_response_by_id(&mut ctx, 42_004);
assert_eq!(
result["result"]["result"]["value"],
json!(["service worker navigation", true])
);
server.abort();
}
#[tokio::test(flavor = "multi_thread")]
async fn unmatched_navigation_interception_preserves_service_worker_response() {
service_worker_navigation_contract(None).await;
}
#[tokio::test(flavor = "multi_thread")]
async fn request_interception_preserves_service_worker_navigation() {
service_worker_navigation_contract(Some("Request")).await;
}
#[tokio::test(flavor = "multi_thread")]
async fn response_interception_preserves_service_worker_client_after_body_capture() {
service_worker_navigation_contract(Some("Response")).await;
}
+3 -3
View File
@@ -510,7 +510,7 @@ impl TargetNetworkAgentState {
.record_fetch_pause_announced_request_id(request_id);
}
pub(crate) fn replace_child_document_fetch_request_id(
pub(crate) fn replace_navigation_fetch_request_id(
&mut self,
network_id: String,
session_id: Option<String>,
@@ -518,7 +518,7 @@ impl TargetNetworkAgentState {
) -> Option<String> {
self.artifacts
.subresource_network_artifacts
.replace_child_document_fetch_request_id(network_id, session_id, fetch_id)
.replace_navigation_fetch_request_id(network_id, session_id, fetch_id)
}
pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool {
@@ -1935,7 +1935,7 @@ impl SubresourceNetworkArtifacts {
.or_default();
}
fn replace_child_document_fetch_request_id(
fn replace_navigation_fetch_request_id(
&mut self,
network_id: String,
session_id: Option<String>,
@@ -603,6 +603,7 @@ struct MainDocumentLiveNetworkProgressSource {
timestamp: f64,
initial_request_headers: Vec<(String, String)>,
initial_request_cookie_report: Option<StoredCookieQueryReport>,
announced_redirects: usize,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
@@ -1003,6 +1004,7 @@ impl MainDocumentLiveNetworkProgressSource {
loader_id: state.loader_id.clone(),
frame_id: state.frame_id.clone(),
timestamp: state.timestamp,
announced_redirects: state.announced_redirects,
initial_request_headers: state.request_headers.to_byte_strings(),
initial_request_cookie_report: initial_request_cookie_report.cloned(),
})
@@ -1138,30 +1140,32 @@ impl MainDocumentLiveNetworkProgressSource {
},
);
}
events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
target: target.clone(),
url: redirect.to_url.clone(),
method: final_request_method.to_owned(),
request_body: request_body.map(str::to_owned),
request_headers: final_request_headers.to_vec(),
request_initiator_type: SubresourceRequestInitiatorType::Other,
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
url: redirect.from_url.clone(),
status: redirect.status,
status_text: critical_client_hint_internal_redirect_status_text(
&redirect.from_url,
&redirect.to_url,
redirect.status,
redirect.network_extra_info_available,
observed_response_available,
),
headers: redirect.headers.clone(),
from_cache: redirect.from_cache,
negotiated_http_version: redirect.negotiated_http_version,
})),
redirect_has_extra_info: redirect.network_extra_info_available,
cookie_access_report: redirect.request_cookie_report.clone(),
});
if index >= self.announced_redirects {
events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
target: target.clone(),
url: redirect.to_url.clone(),
method: final_request_method.to_owned(),
request_body: request_body.map(str::to_owned),
request_headers: final_request_headers.to_vec(),
request_initiator_type: SubresourceRequestInitiatorType::Other,
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
url: redirect.from_url.clone(),
status: redirect.status,
status_text: critical_client_hint_internal_redirect_status_text(
&redirect.from_url,
&redirect.to_url,
redirect.status,
redirect.network_extra_info_available,
observed_response_available,
),
headers: redirect.headers.clone(),
from_cache: redirect.from_cache,
negotiated_http_version: redirect.negotiated_http_version,
})),
redirect_has_extra_info: redirect.network_extra_info_available,
cookie_access_report: redirect.request_cookie_report.clone(),
});
}
if observed_response_available && redirect.network_extra_info_available {
let (status, headers) = observed_response_metadata(
network_observation_journal,
@@ -1667,6 +1671,7 @@ struct CompletedMainDocumentProgressContext {
session_ids: Vec<Option<String>>,
request_id: Option<String>,
request_announced: bool,
announced_redirects: usize,
requested_url: Url,
request_method: String,
request_body: Option<String>,
@@ -1682,7 +1687,7 @@ impl CompletedMainDocumentProgressContext {
state: &NavigationDispatchState,
request_id: Option<String>,
) -> Self {
Self::new(
let mut context = Self::new(
main_document_network_event_session_ids(conn, state.session_id.as_deref()),
request_id,
state.request_announced,
@@ -1693,7 +1698,9 @@ impl CompletedMainDocumentProgressContext {
state.loader_id.clone(),
state.frame_id.clone(),
state.timestamp,
)
);
context.announced_redirects = state.announced_redirects;
context
}
fn new(
@@ -1712,6 +1719,7 @@ impl CompletedMainDocumentProgressContext {
session_ids,
request_id,
request_announced,
announced_redirects: 0,
requested_url,
request_method,
request_body,
@@ -1808,30 +1816,32 @@ impl CompletedMainDocumentProgressContext {
},
);
}
progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
target: target.clone(),
url: redirect.to_url.clone(),
method: events.request_method.clone(),
request_body: self.request_body.clone(),
request_headers: events.request_headers.clone(),
request_initiator_type: SubresourceRequestInitiatorType::Other,
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
url: redirect.from_url.clone(),
status: redirect.status,
status_text: critical_client_hint_internal_redirect_status_text(
&redirect.from_url,
&redirect.to_url,
redirect.status,
redirect.network_extra_info_available,
observed_response_available,
),
headers: redirect.headers.clone(),
from_cache: redirect.from_cache,
negotiated_http_version: redirect.negotiated_http_version,
})),
redirect_has_extra_info: redirect.network_extra_info_available,
cookie_access_report: redirect.request_cookie_report.clone(),
});
if index >= self.announced_redirects {
progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
target: target.clone(),
url: redirect.to_url.clone(),
method: events.request_method.clone(),
request_body: self.request_body.clone(),
request_headers: events.request_headers.clone(),
request_initiator_type: SubresourceRequestInitiatorType::Other,
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
url: redirect.from_url.clone(),
status: redirect.status,
status_text: critical_client_hint_internal_redirect_status_text(
&redirect.from_url,
&redirect.to_url,
redirect.status,
redirect.network_extra_info_available,
observed_response_available,
),
headers: redirect.headers.clone(),
from_cache: redirect.from_cache,
negotiated_http_version: redirect.negotiated_http_version,
})),
redirect_has_extra_info: redirect.network_extra_info_available,
cookie_access_report: redirect.request_cookie_report.clone(),
});
}
if observed_response_available && redirect.network_extra_info_available {
let (status, headers) = observed_response_metadata(
&events.network_observation_journal,
@@ -120,6 +120,7 @@ fn live_progress_source() -> (
let (sender, receiver) = unbounded_channel();
(
MainDocumentLiveNetworkProgressSource {
announced_redirects: 0,
sender: Some(sender),
progress_queue: super::gate::MainDocumentProgressQueueHandle::from_source(
MainDocumentProgressSource::streaming(),
@@ -1604,6 +1605,7 @@ fn no_content_navigation_retains_response_metadata_with_live_progress() {
request_id: Some("REQ-1".to_owned()),
loader_id: "LOADER-1".to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url: Url::parse("http://example.test/start").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -666,6 +666,7 @@ async fn clear_browser_cache_keeps_pending_response_navigation_transfer() {
request_id: Some("REQ-1".to_owned()),
loader_id: LOADER_ID.to_owned(),
request_announced: true,
announced_redirects: 0,
requested_url: url.clone(),
request_method: "GET".to_owned(),
request_body: None,
@@ -524,6 +524,7 @@ async fn get_request_post_data_returns_main_document_navigation_post_body() {
request_id: Some("REQ-main-post".to_owned()),
loader_id: LOADER_ID.to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url,
request_method: "POST".to_owned(),
request_body: Some("username=alice&pw=s3cret".to_owned()),
@@ -620,6 +621,7 @@ async fn get_request_post_data_uses_text_projection_while_bidi_collector_keeps_t
request_id: Some("REQ-multipart".to_owned()),
loader_id: LOADER_ID.to_owned(),
request_announced: false,
announced_redirects: 0,
requested_url,
request_method: "POST".to_owned(),
request_body: Some(multipart_text.to_owned()),
+18 -5
View File
@@ -2654,6 +2654,7 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
request_id: None,
loader_id: document_loader_id.clone(),
request_announced: false,
announced_redirects: 0,
requested_url: requested_url.clone(),
request_method: request_method.to_owned(),
request_body: request_body
@@ -2699,6 +2700,10 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
));
};
pending_fetch_navigation = Some(PendingFetchNavigation {
reserved_service_worker_client: None,
interception: None,
remaining_request_stages: Vec::new(),
redirected_request_id: None,
auth_retry_budget: Default::default(),
fetch_request_id,
interception_session_id: preflight
@@ -2791,7 +2796,8 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
}
if let Some(mut pending) = pending_fetch_navigation {
if fetch_request_stage == FetchRequestStage::Request {
let request_pause = fetch::prepare_navigation_request_pause(conn, &mut pending);
if fetch_request_stage == FetchRequestStage::Request && request_pause {
pending.request_cookie_report = navigation_cookie_access_report_for_owner(
conn,
&pending.navigation.owner,
@@ -2810,14 +2816,13 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
);
}
pending.navigation.request_announced = pending.navigation.request_id.is_some();
let _ = conn.register_pending_fetch_navigation_request_for_owner(
&pending.navigation.owner,
pending.clone(),
);
out.push(fetch::request_paused_background_event(
pending.interception_session_id.as_deref(),
&pending,
));
let pending_owner = pending.navigation.owner.clone();
let _ =
conn.register_pending_fetch_navigation_request_for_owner(&pending_owner, pending);
let mut output = CommandOutputBuffer::default();
output.extend_background_events_after_messages(out);
return NavigateCommandStart::CompleteImmediate(output.into_plan());
@@ -2840,6 +2845,14 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
);
}
pending.navigation.request_announced = pending.navigation.request_id.is_some();
if let Some(sender) = conn.background_event_sender() {
// Retire the previous source and announce the request before
// waiting for headers. A held transport must not retain either
// the old Worker targets or this navigation's initial events.
for event in out.drain(..) {
let _ = sender.send(event);
}
}
return NavigateCommandStart::PendingContinueWithoutRequestPause(Box::new(
PendingContinueNavigationWithoutRequestPauseCommand {
prefix_events: out,
@@ -681,6 +681,7 @@ async fn page_navigate_network_failure_commits_error_document() {
request_id: Some(stale_request_id.to_owned()),
loader_id: old_document_token.loader_id.clone(),
request_announced: true,
announced_redirects: 0,
requested_url: url::Url::parse("https://stale.example.test/old-body").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
@@ -207,19 +207,24 @@ impl JsContextHost {
let fetch = ChildDocumentFetch {
target,
network_attribution,
navigation_url: target_url,
network_info: None,
intercept_redirects: false,
redirect: None,
request: child_document_load_request(bootstrap).map(|request| {
configure_child_document_navigation_request(
request,
&initiator_url,
&browser_context,
)
.with_page_network_policy()
.with_network_partition_key(network_partition_key)
}),
request: child_document_load_request(bootstrap)
.map(|request| {
configure_child_document_navigation_request(
request,
&initiator_url,
&browser_context,
)
.with_page_network_policy()
.with_network_partition_key(network_partition_key)
})
.map(|request| {
crate::network::navigation::NavigationInterception::new(
resource_loader.clone(),
request,
)
}),
service_worker_client_id,
browser_context_runtime,
resource_loader,
@@ -801,9 +806,11 @@ impl JsContextHost {
initiating_loader: &crate::network::context::DocumentResourceLoader,
mut fetch: ChildDocumentFetch,
) -> Option<u64> {
fetch.intercept_redirects = false;
if let Some(request) = fetch.request.as_mut() {
request.set_interception(false);
}
let Some(request) = fetch.request.as_ref().filter(|request| {
matches!(request.url.scheme(), "http" | "https")
matches!(request.request().url.scheme(), "http" | "https")
&& self.should_intercept_subresource(SubresourceResourceType::Document)
}) else {
fetch.start();
@@ -827,16 +834,17 @@ impl JsContextHost {
internal_id: 0,
network_request_handle: None,
frame_id: Some(fetch.network_attribution.frame_id().to_owned()),
document_url: request.url.clone(),
url: request.url.clone(),
document_url: request.request().url.clone(),
url: request.request().url.clone(),
websocket_socket_id: None,
method: request.method.clone(),
request_headers: request.request_headers.clone(),
method: request.request().method.clone(),
request_headers: request.request().request_headers.clone(),
request_body: request
.request()
.body
.as_ref()
.map(|body| String::from_utf8_lossy(body).into_owned()),
request_body_bytes: request.body.clone(),
request_body_bytes: request.request().body.clone(),
resource_type: SubresourceResourceType::Document,
request_cookie_report: None,
};
@@ -862,12 +870,13 @@ impl JsContextHost {
headers: moli_fetch::RequestHeaders,
body: Option<bytes::Bytes>,
) {
if let Some(mut request) = fetch.request.take() {
if let Some(interception) = fetch.request.as_mut() {
let request = interception.request_mut();
request.url = url;
request.method = method;
request.request_headers = headers;
request.body = body;
request = request.with_redirect_headers(redirect_headers);
*request = request.clone().with_redirect_headers(redirect_headers);
if let Some(info) = fetch.network_info.as_mut() {
info.url = request.url.clone();
info.method = request.method.clone();
@@ -878,7 +887,6 @@ impl JsContextHost {
.as_ref()
.map(|body| String::from_utf8_lossy(body).into_owned());
}
fetch.request = Some(request);
}
// A superseded navigation has already settled without this request.
if self
@@ -942,10 +950,8 @@ const UNSUPPORTED_CHILD_DOCUMENT_BOOTSTRAP: &str =
pub(crate) struct ChildDocumentFetch {
target: ChildDocumentNavigationFetchTarget,
network_attribution: ChildDocumentLoadNetworkAttribution,
request: Option<moli_fetch::Request>,
navigation_url: url::Url,
request: Option<crate::network::navigation::NavigationInterception>,
network_info: Option<PendingSubresourceFetchInfo>,
intercept_redirects: bool,
redirect: Option<crate::protocol_types::NavigationRedirect>,
service_worker_client_id: Option<crate::service_worker_runtime::ServiceWorkerClientId>,
browser_context_runtime: crate::runtime::RendererBrowserContextRuntime,
@@ -959,7 +965,7 @@ impl std::fmt::Debug for ChildDocumentFetch {
formatter
.debug_struct("ChildDocumentFetch")
.field("target", &self.target)
.field("navigation_url", &self.navigation_url)
.field("request", &self.request)
.finish_non_exhaustive()
}
}
@@ -967,7 +973,9 @@ impl std::fmt::Debug for ChildDocumentFetch {
impl ChildDocumentFetch {
pub(crate) fn set_network_info(&mut self, info: PendingSubresourceFetchInfo) {
self.network_info = Some(info);
self.intercept_redirects = true;
if let Some(request) = self.request.as_mut() {
request.set_interception(true);
}
}
pub(crate) fn take_request_metadata(
@@ -997,7 +1005,7 @@ impl ChildDocumentFetch {
.browser_context_runtime
.fetch_service_worker_child_main_resource_for_reserved_client(
client_id,
&request,
request.request(),
self.resource_loader.request_client(),
self.resource_loader.task_runner(),
self.completion_tx.clone(),
@@ -1011,14 +1019,8 @@ impl ChildDocumentFetch {
let (head, body) = response.into_body();
return self.accept_response(request, head, body);
}
let transport_request = if self.intercept_redirects {
request.clone().with_single_hop()
} else {
request.clone()
};
let response = self
.resource_loader
.fetch_raw(transport_request)
let response = request
.fetch_raw()
.await
.map_err(|error| error.to_string())?;
let (head, body) = response.into_body();
@@ -1027,58 +1029,23 @@ impl ChildDocumentFetch {
fn accept_response(
&mut self,
mut request: moli_fetch::Request,
mut request: crate::network::navigation::NavigationInterception,
head: moli_fetch::ResponseHead,
body: moli_fetch::ResponseBody,
) -> Result<Option<ChildDocumentLoadOutcome>, String> {
if self.intercept_redirects
&& let Some(next_url) = crate::network_host::next_redirect_url(
&head.final_url,
head.status,
&head.headers,
request.redirect_count(),
)?
if let Some(redirect) = request
.follow_redirect(&head)
.map_err(|error| error.to_string())?
{
if !matches!(next_url.scheme(), "http" | "https") {
return Err(format!(
"child document redirect to unsupported URL {next_url}"
));
}
let redirect = moli_fetch::RedirectInfo {
source: moli_fetch::RedirectSource::Network,
from_url: head.final_url.clone(),
to_url: next_url.clone(),
status: head.status,
headers: head.headers.clone(),
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: head.request_cookie_report,
cookie_set_reports: head.cookie_set_reports,
from_cache: head.from_cache,
negotiated_http_version: head.negotiated_http_version,
};
request.update_referrer_for_redirect(&head.final_url, &head.headers);
request
.follow_redirect(redirect.clone())
.map_err(|error| error.to_string())?;
self.resource_loader
.prepare_redirect()
.map_err(|error| error.to_string())?;
self.navigation_url = next_url;
self.redirect = Some(redirect.into());
self.request = Some(request);
return Ok(None);
}
let navigation_url = if !self.intercept_redirects && head.redirected {
head.final_url.clone()
} else {
self.navigation_url.clone()
};
let navigation_url = request.navigation_url(&head);
let request = request.request();
child_document_load_outcome_from_response(
request.url.as_str().to_owned(),
request.method,
request.method.clone(),
request.request_headers.to_byte_strings(),
head,
body,
@@ -1300,11 +1267,12 @@ mod tests {
None,
"retired-loader".to_owned(),
),
navigation_url: url.clone(),
network_info: None,
intercept_redirects: false,
redirect: None,
request: Some(moli_fetch::Request::get_with_url(url)),
request: Some(crate::network::navigation::NavigationInterception::new(
resource_loader.clone(),
moli_fetch::Request::get_with_url(url),
)),
service_worker_client_id: None,
browser_context_runtime: host.browser_context_runtime(),
resource_loader: resource_loader.clone(),
@@ -2431,20 +2431,16 @@ impl JsContextHost {
task_resource_loader.note_external_response_ready()?;
response
}
None => task_resource_loader
.fetch(
None => {
crate::network::navigation::NavigationInterception::new(
task_resource_loader.clone(),
request
.with_request_origin(request_origin)
.with_page_network_policy()
.with_top_level_navigation_cookie_context(),
)
.await
.map(|response| {
moli_fetch::RawResponse::from_head_and_body(
response.head(),
response.body_bytes().to_vec(),
)
})?,
.fetch_raw()
.await?
}
};
let head = response.head();
if popup_document_response_should_ignore_navigation(response.status, &head.headers)
@@ -0,0 +1,161 @@
use anyhow::{Result, bail};
use moli_fetch::{
NetworkFetchResult, RawResponse, RedirectInfo, Request, Response, ResponseHead,
StreamingRawResponse,
};
use url::Url;
use super::NavigationResourceLoader;
/// Request execution shared by top-level, child and auxiliary navigations.
/// The navigation owner decides when a hop may start and where its result can
/// commit. This object owns transport routing, request overrides and redirects;
/// it deliberately has no DevTools session or frame-lifecycle state.
#[derive(Clone, Debug)]
pub struct NavigationInterception {
loader: NavigationResourceLoader,
request: Request,
navigation_url: Url,
intercept_redirects: bool,
}
struct CancelOnDrop(Option<NavigationResourceLoader>);
impl Drop for CancelOnDrop {
fn drop(&mut self) {
if let Some(loader) = self.0.take() {
loader.cancel();
}
}
}
impl NavigationInterception {
pub fn new(loader: NavigationResourceLoader, request: Request) -> Self {
Self {
navigation_url: request.url.clone(),
loader,
request,
intercept_redirects: false,
}
}
pub fn resource_loader(&self) -> &NavigationResourceLoader {
&self.loader
}
pub fn request(&self) -> &Request {
&self.request
}
pub fn request_mut(&mut self) -> &mut Request {
&mut self.request
}
pub fn set_interception(&mut self, enabled: bool) {
self.intercept_redirects = enabled;
}
pub fn navigation_url(&self, response: &ResponseHead) -> Url {
if !self.intercept_redirects && response.redirected {
response.final_url.clone()
} else {
self.navigation_url.clone()
}
}
fn transport_request(&self) -> Result<Request> {
moli_url_policy::route_navigation_url(
&self.request.url,
moli_url_policy::LocalFileNavigationAccess::Denied,
)?;
Ok(if self.intercept_redirects {
self.request.clone().with_single_hop()
} else {
self.request.clone()
})
}
async fn execute<T>(
&self,
operation: impl std::future::Future<Output = Result<T>>,
) -> Result<T> {
let mut guard = CancelOnDrop(Some(self.loader.clone()));
let result = operation.await;
guard.0.take();
result
}
pub async fn fetch_text(&self) -> Result<NetworkFetchResult<Response>> {
self.execute(
self.loader
.fetch_with_network_metadata(self.transport_request()?),
)
.await
}
pub async fn fetch_raw(&self) -> Result<RawResponse> {
self.execute(self.loader.fetch_raw(self.transport_request()?))
.await
}
pub async fn fetch_raw_stream(&self) -> Result<NetworkFetchResult<StreamingRawResponse>> {
self.execute(
self.loader
.fetch_raw_stream_with_network_metadata(self.transport_request()?),
)
.await
}
pub async fn fetch_buffered(&self) -> Result<NetworkFetchResult<RawResponse>> {
self.execute(
self.loader
.fetch_raw_with_network_metadata(self.transport_request()?),
)
.await
}
/// Returns the next hop to its owner before any transport is started.
/// Synthetic and network responses use exactly the same redirect rules.
pub fn follow_redirect(&mut self, head: &ResponseHead) -> Result<Option<RedirectInfo>> {
if !self.intercept_redirects {
return Ok(None);
}
let Some(next_url) = crate::network_host::next_redirect_url(
&head.final_url,
head.status,
&head.headers,
self.request.redirect_count(),
)
.map_err(anyhow::Error::msg)?
else {
return Ok(None);
};
if !matches!(next_url.scheme(), "http" | "https") {
bail!("document redirect to unsupported URL {next_url}");
}
let redirect = RedirectInfo {
source: moli_fetch::RedirectSource::Network,
from_url: head.final_url.clone(),
to_url: next_url.clone(),
status: head.status,
headers: head.headers.clone(),
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: head.request_cookie_report.clone(),
cookie_set_reports: head.cookie_set_reports.clone(),
from_cache: head.from_cache,
negotiated_http_version: head.negotiated_http_version,
};
self.request
.update_referrer_for_redirect(&head.final_url, &head.headers);
self.request.follow_redirect(redirect.clone())?;
self.loader.prepare_redirect()?;
self.navigation_url = next_url;
Ok(Some(redirect))
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,104 @@
use super::*;
use crate::network::{RendererResourceTaskRunner, ResourceRequestClient};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
async fn request_head(socket: &mut tokio::net::TcpStream) -> String {
let mut head = Vec::new();
let mut bytes = [0u8; 4096];
while !head.ends_with(b"\r\n\r\n") {
let length = socket.read(&mut bytes).await.unwrap();
assert!(length > 0, "connection closed before request headers");
head.extend_from_slice(&bytes[..length]);
}
String::from_utf8(head).unwrap()
}
#[tokio::test]
async fn redirect_body_cancellation_does_not_cancel_the_next_navigation_hop() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut first, _) = listener.accept().await.unwrap();
assert!(request_head(&mut first).await.starts_with("GET /redirect "));
first
.write_all(b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 1000000\r\n\r\n")
.await
.unwrap();
let (mut second, _) = listener.accept().await.unwrap();
assert!(request_head(&mut second).await.starts_with("GET /final "));
second
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
.await
.unwrap();
});
let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap();
let request = Request::get(&format!("http://{addr}/redirect")).unwrap();
let origin = moli_url::WebOrigin::from_url(&request.url);
let request = request.with_request_origin(origin);
let loader = NavigationResourceLoader::new(
owner.handle(),
request.url.clone(),
RendererResourceTaskRunner::from_current_tokio().unwrap(),
);
let mut navigation = NavigationInterception::new(loader.clone(), request);
navigation.set_interception(true);
let response = navigation.fetch_raw_stream().await.unwrap();
let redirect = navigation
.follow_redirect(&response.response().head())
.unwrap()
.unwrap();
assert_eq!(redirect.to_url.path(), "/final");
drop(response);
let response = tokio::time::timeout(std::time::Duration::from_secs(10), navigation.fetch_raw())
.await
.unwrap()
.unwrap();
let (_, body) = response.into_body();
assert_eq!(body.try_into_materialized_bytes().unwrap(), b"ok");
assert_eq!(
loader.state(),
super::super::NavigationResourceLoaderState::ResponseReady
);
server.await.unwrap();
}
#[tokio::test]
async fn dropping_a_request_future_cancels_transport_while_its_navigation_owner_survives() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (received, request_received) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut bytes = [0u8; 4096];
assert!(socket.read(&mut bytes).await.unwrap() > 0);
received.send(()).unwrap();
let read =
tokio::time::timeout(std::time::Duration::from_secs(10), socket.read(&mut bytes))
.await
.unwrap();
assert!(
matches!(read, Ok(0)) || read.is_err(),
"cancelled navigation must close the transport"
);
});
let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap();
let request = Request::get(&format!("http://{addr}/pending")).unwrap();
let origin = moli_url::WebOrigin::from_url(&request.url);
let request = request.with_request_origin(origin);
let loader = NavigationResourceLoader::new(
owner.handle(),
request.url.clone(),
RendererResourceTaskRunner::from_current_tokio().unwrap(),
);
let navigation = NavigationInterception::new(loader.clone(), request);
let task = tokio::spawn(async move { navigation.fetch_raw_stream().await });
request_received.await.unwrap();
task.abort();
assert!(task.await.is_err_and(|error| error.is_cancelled()));
assert_eq!(
loader.state(),
super::super::NavigationResourceLoaderState::Cancelled
);
server.await.unwrap();
}
@@ -277,6 +277,30 @@ impl NavigationResourceLoader {
Ok(())
}
pub async fn fetch_raw_with_network_metadata(
&self,
request: Request,
) -> Result<NetworkFetchResult<RawResponse>> {
self.begin_fetch()?;
let result = self
.request_client
.fetch_raw_with_cancel_and_network_metadata(
request.with_page_network_policy(),
self.inner.cancel.clone(),
)
.await;
match result {
Ok(response) => {
self.finish_response_ready()?;
Ok(response)
}
Err(error) => {
self.finish_failed();
Err(error)
}
}
}
pub async fn fetch_raw_stream(&self, request: Request) -> Result<StreamingRawResponse> {
self.fetch_raw_stream_with_network_metadata(request)
.await
@@ -288,16 +312,30 @@ impl NavigationResourceLoader {
request: Request,
) -> Result<NetworkFetchResult<StreamingRawResponse>> {
self.begin_fetch()?;
let cancellation = if request.transport_follows_redirects() {
self.inner.cancel.clone()
} else {
self.inner.cancel.child_for_subrequest()
};
match self
.request_client
.fetch_raw_stream_with_cancel_and_network_metadata(
request.with_page_network_policy(),
self.inner.cancel.clone(),
cancellation.clone(),
)
.await
{
Ok(response) => {
self.finish_response_ready()?;
if !matches!(response.response().status, 301 | 302 | 303 | 307 | 308)
|| !response
.response()
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("location"))
{
self.inner.cancel.adopt_response_progress(&cancellation);
}
Ok(response)
}
Err(error) => {
@@ -1,7 +1,9 @@
mod context;
mod interception;
mod loader;
pub use context::DocumentFetchContextSeed;
pub use interception::NavigationInterception;
pub use loader::{
NavigationResourceLoader, NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState,
};