mirror of
https://github.com/lexmount/moli.git
synced 2026-11-18 00:09:07 +00:00
refactor(navigation): share document request execution
Reuse NavigationInterception for main, child-frame and auxiliary document requests, replacing the protocol-only PreparedNavigationRequest wrapper. Reevaluate main-document Fetch interception per redirect and share per-session hop identities. Preserve Service Worker clients through response handling, publish source retirement before waiting for headers, and keep transport failures on the existing error-document path.
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
pub use moli_renderer_v8::network::{
|
||||
BrowserResourceRuntime, BrowserResourceRuntimeOwner, ResourceRequestClient,
|
||||
SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation, WebStorageMutationRecord,
|
||||
WebStorageMutationSubscription, WebStorageString,
|
||||
BrowserResourceRuntime, BrowserResourceRuntimeOwner, RendererResourceTaskRunner,
|
||||
ResourceRequestClient, SharedWebStorageStore, WebStorageAreaKind, WebStorageMutation,
|
||||
WebStorageMutationRecord, WebStorageMutationSubscription, WebStorageString,
|
||||
context::{DocumentResourceLoader, DocumentResourceLoaderDiagnostics},
|
||||
deep_clone_shared_web_storage_store,
|
||||
navigation::{
|
||||
DocumentFetchContextSeed, NavigationResourceLoader, NavigationResourceLoaderDiagnostics,
|
||||
NavigationResourceLoaderState,
|
||||
DocumentFetchContextSeed, NavigationInterception, NavigationResourceLoader,
|
||||
NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState,
|
||||
},
|
||||
new_shared_web_storage_store, web_storage_partitioned_area_key,
|
||||
};
|
||||
|
||||
@@ -1458,11 +1458,23 @@ fn external_raw_document_body_from_streaming_response_with_observer_and_progress
|
||||
body_stream
|
||||
}
|
||||
|
||||
fn streaming_raw_response_from_navigation_response(
|
||||
response: NavigationResponse,
|
||||
pub fn streaming_raw_response_from_navigation_response(
|
||||
mut response: NavigationResponse,
|
||||
) -> Result<StreamingRawResponse> {
|
||||
let head = response.head();
|
||||
let body = response.clone_body_bytes();
|
||||
let body = response.take_body_bytes();
|
||||
streaming_raw_response_from_raw_response(moli_fetch::RawResponse::from_head_and_body(
|
||||
head, body,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn streaming_raw_response_from_raw_response(
|
||||
response: moli_fetch::RawResponse,
|
||||
) -> Result<StreamingRawResponse> {
|
||||
let (head, body) = response.into_body();
|
||||
let body = body
|
||||
.try_into_materialized_bytes()
|
||||
.map_err(|_| anyhow!("navigation response body must be materialized"))?;
|
||||
let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
if !body.is_empty() {
|
||||
body_tx
|
||||
|
||||
@@ -74,28 +74,6 @@ pub enum CommittedDocumentResourceSource {
|
||||
Synthetic,
|
||||
}
|
||||
|
||||
fn streaming_raw_response_from_navigation_response(
|
||||
response: NavigationResponse,
|
||||
) -> Result<StreamingRawResponse> {
|
||||
let head = response.head();
|
||||
let body = response.clone_body_bytes();
|
||||
let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
if !body.is_empty() {
|
||||
body_tx
|
||||
.send(body)
|
||||
.map_err(|_| anyhow!("failed to enqueue service worker main resource body"))?;
|
||||
}
|
||||
drop(body_tx);
|
||||
let (completion_tx, completion_rx) = tokio::sync::oneshot::channel();
|
||||
let _ = completion_tx.send(Ok(()));
|
||||
Ok(StreamingRawResponse::new_with_head(
|
||||
head,
|
||||
body_rx,
|
||||
FetchCancelHandle::new(),
|
||||
completion_rx,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NavigationResourceStorageHandles {
|
||||
cookie_store: SharedBrowserCookieStore,
|
||||
@@ -1207,10 +1185,13 @@ impl NavigationEngine {
|
||||
request.url.clone(),
|
||||
FetchCancelHandle::new(),
|
||||
)?;
|
||||
navigation_loader
|
||||
.fetch_with_network_metadata(request)
|
||||
.await
|
||||
.map(|result| result.map_response(NavigationResponse::from))
|
||||
moli_renderer_v8::network::navigation::NavigationInterception::new(
|
||||
navigation_loader,
|
||||
request,
|
||||
)
|
||||
.fetch_text()
|
||||
.await
|
||||
.map(|result| result.map_response(NavigationResponse::from))
|
||||
}
|
||||
|
||||
pub async fn fetch_navigation_response_with_storage_async(
|
||||
@@ -1301,15 +1282,18 @@ impl NavigationEngine {
|
||||
let final_url = response.final_url.clone();
|
||||
return Ok(NavigationStreamingRawResponse {
|
||||
fetch_result: NetworkFetchResult::without_request_observation(
|
||||
streaming_raw_response_from_navigation_response(response)?,
|
||||
super::streaming_raw_response_from_navigation_response(response)?,
|
||||
),
|
||||
reserved_service_worker_client: reserved_client,
|
||||
document_fetch_context_seed: navigation_loader.commit(final_url)?,
|
||||
});
|
||||
}
|
||||
let fetch_result = navigation_loader
|
||||
.fetch_raw_stream_with_network_metadata(request)
|
||||
.await?;
|
||||
let fetch_result = moli_renderer_v8::network::navigation::NavigationInterception::new(
|
||||
navigation_loader.clone(),
|
||||
request,
|
||||
)
|
||||
.fetch_raw_stream()
|
||||
.await?;
|
||||
let final_url = fetch_result.response().final_url.clone();
|
||||
Ok(NavigationStreamingRawResponse {
|
||||
fetch_result,
|
||||
|
||||
@@ -683,7 +683,7 @@ pub(crate) use runtime_load::{
|
||||
BackgroundNavigationLoadJob, CapturedPageReplacement, CommittedNavigationPage,
|
||||
CompletedInitialDocumentPageBuild, FailedInitialDocumentPageBuild,
|
||||
InitialDocumentPageInstallResult, InitialDocumentPageOwner, PendingInitialDocumentPageBuild,
|
||||
PreparedCapturedNavigationResponse, PreparedNavigationLoadError, PreparedNavigationRequest,
|
||||
PreparedCapturedNavigationResponse, PreparedNavigationLoadError,
|
||||
PreparedStreamingNavigationResponse, ResponseCommitReady, StablePageNavigationCommitTarget,
|
||||
};
|
||||
use scheduler_hooks::CdpSchedulerHooks;
|
||||
@@ -4271,3 +4271,5 @@ pub struct PermissionOverride {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) use state::TargetFetchSubresourceInterceptionSnapshot;
|
||||
|
||||
@@ -576,25 +576,6 @@ impl CdpConnection {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn register_pending_fetch_response_navigation_for_owner(
|
||||
&mut self,
|
||||
owner: &CommandOwnerScope,
|
||||
request_id: String,
|
||||
document_navigation_token: Option<crate::conn::DocumentNavigationToken>,
|
||||
navigation: crate::conn::NavigationDispatchState,
|
||||
body: crate::conn::DocumentBodySource,
|
||||
) -> bool {
|
||||
self.target_session_owner_mut_for_owner(owner)
|
||||
.is_some_and(|mut owner| {
|
||||
owner.register_pending_fetch_response_navigation(
|
||||
request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
body,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn take_fetch_response_transfer_for_cancellation_for_owner(
|
||||
&mut self,
|
||||
owner: &CommandOwnerScope,
|
||||
@@ -1430,34 +1411,6 @@ impl TargetSessionOwnerMut<'_> {
|
||||
true
|
||||
}
|
||||
|
||||
fn register_pending_fetch_response_navigation(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
document_navigation_token: Option<crate::conn::DocumentNavigationToken>,
|
||||
navigation: crate::conn::NavigationDispatchState,
|
||||
body: crate::conn::DocumentBodySource,
|
||||
) -> bool {
|
||||
if let Some(token) = document_navigation_token.as_ref()
|
||||
&& let crate::conn::DocumentBodySource::StreamingRaw { response, .. } = &body
|
||||
{
|
||||
// Navigation cancellation also covers a response already handed
|
||||
// to a body reader or a continuation outside the Fetch owner.
|
||||
self.runtime_slot_mut()
|
||||
.page_slot_mut()
|
||||
.add_document_navigation_cancellation(token, response.cancellation_handle());
|
||||
}
|
||||
let Some(mut owner) = self.pending_fetch_owner_mut() else {
|
||||
return false;
|
||||
};
|
||||
owner.register_pending_fetch_response_navigation(
|
||||
request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
body,
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
fn take_fetch_response_transfer_for_cancellation(
|
||||
&mut self,
|
||||
request_id: &str,
|
||||
@@ -1479,6 +1432,13 @@ impl TargetSessionOwnerMut<'_> {
|
||||
request_id: String,
|
||||
transfer: PausedDocumentTransfer,
|
||||
) -> bool {
|
||||
if let Some(token) = transfer.navigation_token()
|
||||
&& let Some(cancellation) = transfer.cancellation_handle()
|
||||
{
|
||||
self.runtime_slot_mut()
|
||||
.page_slot_mut()
|
||||
.add_document_navigation_cancellation(token, cancellation);
|
||||
}
|
||||
let Some(mut owner) = self.pending_fetch_owner_mut() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -1105,7 +1105,12 @@ impl<'a> TargetSessionOwnerMut<'a> {
|
||||
.map(|_| FetchRequestStage::Response)
|
||||
})
|
||||
.or_else(|| {
|
||||
document_fetch_response_stage_candidate.then_some(FetchRequestStage::Response)
|
||||
(document_fetch_response_stage_candidate
|
||||
|| (!is_data_url
|
||||
&& fetch_snapshot.has_request_stage_candidate(
|
||||
DevToolsNetworkResourceType::Document,
|
||||
)))
|
||||
.then_some(FetchRequestStage::Response)
|
||||
});
|
||||
let document_fetch_event_session_id = document_request_pause
|
||||
.as_ref()
|
||||
|
||||
@@ -197,6 +197,10 @@ async fn bidi_fetch_control_resolves_background_request_owner() {
|
||||
conn.register_pending_fetch_navigation_request_for_owner(
|
||||
&crate::conn::CommandOwnerScope::for_session("SID-background"),
|
||||
PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: "FETCH-background".to_owned(),
|
||||
interception_session_id: Some("bidi-session-1".to_owned()),
|
||||
@@ -216,6 +220,7 @@ async fn bidi_fetch_control_resolves_background_request_owner() {
|
||||
request_id: Some("NETWORK-background".to_owned()),
|
||||
loader_id: "LOADER-background".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: url::Url::parse("https://example.test/background").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -222,8 +222,16 @@ impl PendingSubresourceFetchOwnerKind {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct PendingFetchNavigation {
|
||||
pub(crate) reserved_service_worker_client:
|
||||
Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
pub(crate) interception: Option<(
|
||||
moli_core::network::NavigationInterception,
|
||||
moli_fetch::NetworkObservationJournal,
|
||||
)>,
|
||||
pub(crate) remaining_request_stages: Vec<PendingSubresourceFetchRequestStage>,
|
||||
pub(crate) redirected_request_id: Option<String>,
|
||||
/// Authentication restart budget for this logical navigation, independent of pause IDs.
|
||||
pub(crate) auth_retry_budget: moli_fetch::AuthRetryBudget,
|
||||
pub fetch_request_id: String,
|
||||
@@ -300,6 +308,7 @@ impl PendingFetchAuthNavigation {
|
||||
/// Protocol keeps navigation ownership even while a task exclusively reads the body.
|
||||
#[derive(Debug)]
|
||||
pub struct PausedDocumentTransfer {
|
||||
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
fetch_request_id: String,
|
||||
document_navigation_token: Option<DocumentNavigationToken>,
|
||||
navigation: NavigationDispatchState,
|
||||
@@ -510,6 +519,7 @@ impl PausedDocumentTransfers {
|
||||
self.transfers.insert(request_id, transfer);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn register_pending_navigation(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
@@ -737,6 +747,8 @@ impl PendingDocumentBodyRead {
|
||||
}
|
||||
|
||||
pub(crate) struct PendingStreamingDocumentResponseNavigation {
|
||||
pub(crate) reserved_service_worker_client:
|
||||
Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
pub(crate) document_navigation_token: DocumentNavigationToken,
|
||||
pub(crate) navigation: NavigationDispatchState,
|
||||
pub(crate) response: StreamingRawResponse,
|
||||
@@ -745,6 +757,32 @@ pub(crate) struct PendingStreamingDocumentResponseNavigation {
|
||||
}
|
||||
|
||||
impl PausedDocumentTransfer {
|
||||
pub(crate) fn with_service_worker_client(
|
||||
mut self,
|
||||
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
) -> Self {
|
||||
self.reserved_service_worker_client = client;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn cancellation_handle(&self) -> Option<moli_fetch::FetchCancelHandle> {
|
||||
match &self.state {
|
||||
PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) => {
|
||||
match body.as_ref() {
|
||||
DocumentBodySource::StreamingRaw { response, .. } => {
|
||||
Some(response.cancellation_handle())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
PausedDocumentTransferState::Available(PausedDocumentBody::ActiveBodyStream {
|
||||
stream,
|
||||
..
|
||||
}) => Some(stream.response.cancellation_handle()),
|
||||
PausedDocumentTransferState::Reading(reading) => reading.cancellation.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn navigation_token(&self) -> Option<&DocumentNavigationToken> {
|
||||
self.document_navigation_token.as_ref()
|
||||
}
|
||||
@@ -757,7 +795,7 @@ impl PausedDocumentTransfer {
|
||||
// Otherwise the same owned preparation accepts captured or streaming bodies.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub(crate) fn prepare_response(
|
||||
self,
|
||||
mut self,
|
||||
conn: &mut CdpConnection,
|
||||
response_code: Option<u16>,
|
||||
response_headers: Vec<(String, Vec<u8>)>,
|
||||
@@ -813,7 +851,12 @@ impl PausedDocumentTransfer {
|
||||
body_progress_source,
|
||||
response_code,
|
||||
response_headers,
|
||||
),
|
||||
)
|
||||
.map(|response| {
|
||||
response.with_service_worker_client(
|
||||
self.reserved_service_worker_client.take(),
|
||||
)
|
||||
}),
|
||||
)),
|
||||
);
|
||||
}
|
||||
@@ -844,11 +887,17 @@ impl PausedDocumentTransfer {
|
||||
progress,
|
||||
)
|
||||
};
|
||||
Ok(crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new(response)))
|
||||
Ok(
|
||||
crate::domains::fetch::PreparedFetchNavigationResponse::Captured(Box::new(
|
||||
response.map(|response| {
|
||||
response.with_service_worker_client(self.reserved_service_worker_client.take())
|
||||
}),
|
||||
)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_synthetic_response(
|
||||
&self,
|
||||
mut self,
|
||||
conn: &mut CdpConnection,
|
||||
response_code: u16,
|
||||
response_headers: Vec<(String, Vec<u8>)>,
|
||||
@@ -883,6 +932,9 @@ impl PausedDocumentTransfer {
|
||||
NetworkObservationJournal::default(),
|
||||
body_progress_source,
|
||||
)
|
||||
.map(|response| {
|
||||
response.with_service_worker_client(self.reserved_service_worker_client.take())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn into_navigation(
|
||||
@@ -899,6 +951,7 @@ impl PausedDocumentTransfer {
|
||||
body: DocumentBodySource,
|
||||
) -> Self {
|
||||
Self {
|
||||
reserved_service_worker_client: None,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -936,6 +989,7 @@ impl PausedDocumentTransfer {
|
||||
handle: String,
|
||||
) -> Result<PendingFetchResponseOpenedBodyStream, Box<Self>> {
|
||||
let Self {
|
||||
reserved_service_worker_client,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -944,6 +998,7 @@ impl PausedDocumentTransfer {
|
||||
let PausedDocumentTransferState::Available(PausedDocumentBody::Pending(body)) = state
|
||||
else {
|
||||
return Err(Box::new(Self {
|
||||
reserved_service_worker_client,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -985,6 +1040,7 @@ impl PausedDocumentTransfer {
|
||||
handle,
|
||||
buffered_body,
|
||||
transfer: Self {
|
||||
reserved_service_worker_client,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -997,6 +1053,7 @@ impl PausedDocumentTransfer {
|
||||
self,
|
||||
) -> Result<PendingStreamingDocumentResponseNavigation, Box<Self>> {
|
||||
let Self {
|
||||
reserved_service_worker_client,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -1009,6 +1066,7 @@ impl PausedDocumentTransfer {
|
||||
) if matches!(body.as_ref(), DocumentBodySource::StreamingRaw { .. }) => (token, body),
|
||||
(document_navigation_token, state) => {
|
||||
return Err(Box::new(Self {
|
||||
reserved_service_worker_client,
|
||||
fetch_request_id,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
@@ -1026,6 +1084,7 @@ impl PausedDocumentTransfer {
|
||||
unreachable!("the response was checked before transferring it")
|
||||
};
|
||||
Ok(PendingStreamingDocumentResponseNavigation {
|
||||
reserved_service_worker_client,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
response,
|
||||
|
||||
@@ -380,6 +380,7 @@ mod tests {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: url.clone(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -903,52 +903,6 @@ struct BackgroundNavigationClaim {
|
||||
cancellation: FetchCancelHandle,
|
||||
}
|
||||
|
||||
/// Transport inputs captured while the navigation owner is available. Waiting
|
||||
/// for response headers owns neither a connection nor a renderer Page.
|
||||
pub(crate) struct PreparedNavigationRequest {
|
||||
client: moli_core::network::ResourceRequestClient,
|
||||
request: Request,
|
||||
cancellation: FetchCancelHandle,
|
||||
}
|
||||
|
||||
struct CancelNavigationRequestOnDrop(Option<FetchCancelHandle>);
|
||||
|
||||
impl Drop for CancelNavigationRequestOnDrop {
|
||||
fn drop(&mut self) {
|
||||
if let Some(cancellation) = self.0.take() {
|
||||
cancellation.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PreparedNavigationRequest {
|
||||
pub(crate) async fn wait(self) -> Result<NetworkFetchResult<StreamingRawResponse>> {
|
||||
let mut cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone()));
|
||||
let url = self.request.url.clone();
|
||||
let response = self
|
||||
.client
|
||||
.fetch_raw_stream_with_cancel_and_network_metadata(self.request, self.cancellation)
|
||||
.await
|
||||
.with_context(|| format!("failed to fetch page `{url}`"));
|
||||
if response.is_ok() {
|
||||
// The streaming response now owns transport cancellation.
|
||||
cancellation.0.take();
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_buffered(self) -> Result<NetworkFetchResult<RawResponse>> {
|
||||
let _cancellation = CancelNavigationRequestOnDrop(Some(self.cancellation.clone()));
|
||||
let url = self.request.url.clone();
|
||||
// libcurl owns Digest's intermediate challenges. The exact navigation
|
||||
// cancellation reaches that transport as well as this waiting future.
|
||||
self.client
|
||||
.fetch_raw_with_cancel_and_network_metadata(self.request, self.cancellation)
|
||||
.await
|
||||
.with_context(|| format!("failed to fetch page `{url}`"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct BackgroundNavigationLoadJob {
|
||||
engine: NavigationEngine,
|
||||
page_reservation: RendererDocumentPreparationTarget,
|
||||
@@ -969,6 +923,7 @@ pub(crate) struct BackgroundNavigationLoadJob {
|
||||
}
|
||||
|
||||
pub(crate) struct PreparedStreamingNavigationResponse {
|
||||
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
engine: NavigationEngine,
|
||||
page_reservation: RendererDocumentPreparationTarget,
|
||||
load_inputs: TargetNavigationLoadInputs,
|
||||
@@ -1284,6 +1239,14 @@ impl BackgroundNavigationLoadJob {
|
||||
}
|
||||
|
||||
impl PreparedStreamingNavigationResponse {
|
||||
pub(crate) fn with_service_worker_client(
|
||||
mut self,
|
||||
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
) -> Self {
|
||||
self.reserved_service_worker_client = client;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) async fn run(
|
||||
mut self,
|
||||
body_completion_sink: Option<BackgroundNavigationBodyCompletionSink>,
|
||||
@@ -1307,7 +1270,7 @@ impl PreparedStreamingNavigationResponse {
|
||||
self.response_headers_override,
|
||||
self.body_progress_source,
|
||||
body_completion_sink,
|
||||
None,
|
||||
self.reserved_service_worker_client,
|
||||
CommittedDocumentResourceSource::Synthetic,
|
||||
self.reply_boundary,
|
||||
)
|
||||
@@ -2717,6 +2680,7 @@ impl CdpConnection {
|
||||
let (response, network_observation_journal) =
|
||||
response.into_parts_with_observation_journal();
|
||||
Ok(PreparedStreamingNavigationResponse {
|
||||
reserved_service_worker_client: None,
|
||||
engine,
|
||||
page_reservation,
|
||||
load_inputs,
|
||||
@@ -2756,6 +2720,7 @@ impl CdpConnection {
|
||||
let page_reservation =
|
||||
self.reserve_renderer_page_for_owner(&navigation.owner, &load_inputs, &engine);
|
||||
Ok(PreparedStreamingNavigationResponse {
|
||||
reserved_service_worker_client: None,
|
||||
engine,
|
||||
page_reservation,
|
||||
load_inputs,
|
||||
@@ -3225,7 +3190,7 @@ impl CdpConnection {
|
||||
token: &DocumentNavigationToken,
|
||||
navigation: &NavigationDispatchState,
|
||||
auth: Option<SubresourceAuthCredentials>,
|
||||
) -> Result<PreparedNavigationRequest> {
|
||||
) -> Result<moli_core::network::NavigationInterception> {
|
||||
let load_inputs = self.navigation_load_inputs_for_navigation(Some(token), navigation)?;
|
||||
let cancellation = self
|
||||
.runtime_session_owner_slot_for_owner(&navigation.owner)
|
||||
@@ -3243,11 +3208,15 @@ impl CdpConnection {
|
||||
let client = self
|
||||
.ensure_resource_request_client_for_navigation_load_inputs(&load_inputs)?
|
||||
.frozen_request_client();
|
||||
Ok(PreparedNavigationRequest {
|
||||
let loader = moli_core::network::NavigationResourceLoader::new_with_cancel_handle(
|
||||
client,
|
||||
request,
|
||||
request.url.clone(),
|
||||
moli_core::network::RendererResourceTaskRunner::from_current_tokio()?,
|
||||
cancellation,
|
||||
})
|
||||
);
|
||||
Ok(moli_core::network::NavigationInterception::new(
|
||||
loader, request,
|
||||
))
|
||||
}
|
||||
|
||||
async fn fetch_navigation_streaming_raw_response_with_load_inputs_async(
|
||||
@@ -3271,8 +3240,13 @@ impl CdpConnection {
|
||||
let loader = self
|
||||
.ensure_resource_request_client_for_navigation_load_inputs(load_inputs)?
|
||||
.clone();
|
||||
loader
|
||||
.fetch_raw_stream_with_cancel_and_network_metadata(request, FetchCancelHandle::new())
|
||||
let navigation = moli_core::network::NavigationResourceLoader::new(
|
||||
loader,
|
||||
request.url.clone(),
|
||||
moli_core::network::RendererResourceTaskRunner::from_current_tokio()?,
|
||||
);
|
||||
moli_core::network::NavigationInterception::new(navigation, request)
|
||||
.fetch_raw_stream()
|
||||
.await
|
||||
.with_context(|| format!("failed to fetch page `{raw_url}`"))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::*;
|
||||
/// A captured response's parser inputs and exact renderer reservation. Taking
|
||||
/// this value ends protocol state access; preparation owns all of its waits.
|
||||
pub(crate) struct PreparedCapturedNavigationResponse {
|
||||
reserved_service_worker_client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
engine: NavigationEngine,
|
||||
page_reservation: Option<RendererDocumentPreparationTarget>,
|
||||
load_inputs: TargetNavigationLoadInputs,
|
||||
@@ -45,6 +46,14 @@ impl PreparedNavigationLoadError {
|
||||
}
|
||||
|
||||
impl PreparedCapturedNavigationResponse {
|
||||
pub(crate) fn with_service_worker_client(
|
||||
mut self,
|
||||
client: Option<moli_core::runtime::RendererReservedServiceWorkerClient>,
|
||||
) -> Self {
|
||||
self.reserved_service_worker_client = client;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) async fn run(mut self) -> Result<NavigationLoadOutcome> {
|
||||
if super::super::downloads::response_headers_indicate_download(&self.head.headers) {
|
||||
complete_auxiliary_download_response(&self.load_inputs, self.head.clone());
|
||||
@@ -71,6 +80,34 @@ impl PreparedCapturedNavigationResponse {
|
||||
&self.body_progress_source,
|
||||
));
|
||||
}
|
||||
if let Some(client) = self.reserved_service_worker_client.take() {
|
||||
let body = self
|
||||
.body
|
||||
.process_in_worker(|body| body.materialize_bytes())
|
||||
.await?;
|
||||
let response = moli_core::runtime::streaming_raw_response_from_raw_response(
|
||||
RawResponse::from_head_and_body(self.head, body),
|
||||
)?;
|
||||
return build_navigation_from_streaming_raw_response_with_engine_async(
|
||||
&mut self.engine,
|
||||
self.page_reservation
|
||||
.expect("a Document response must reserve its renderer Page"),
|
||||
&self.load_inputs,
|
||||
self.requested_url,
|
||||
self.request_method,
|
||||
self.request_headers,
|
||||
response,
|
||||
self.network_observation_journal,
|
||||
None,
|
||||
Vec::new(),
|
||||
self.body_progress_source,
|
||||
None,
|
||||
Some(client),
|
||||
CommittedDocumentResourceSource::Synthetic,
|
||||
RendererReplyBoundary::Stage,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
prepare_navigation_from_captured_raw_response_with_engine_async(
|
||||
&mut self.engine,
|
||||
self.page_reservation
|
||||
@@ -99,9 +136,34 @@ impl CdpConnection {
|
||||
navigation: &NavigationDispatchState,
|
||||
error: anyhow::Error,
|
||||
) -> Result<PreparedNavigationLoadError> {
|
||||
let Some(network_error) = error.downcast_ref::<NavigationNetworkError>() else {
|
||||
return Err(error);
|
||||
};
|
||||
let (unreachable_url, request_method, request_headers, error_text) =
|
||||
if let Some(network_error) = error.downcast_ref::<NavigationNetworkError>() {
|
||||
(
|
||||
network_error.unreachable_url.clone(),
|
||||
network_error.request_method.clone(),
|
||||
network_error.request_headers.clone(),
|
||||
network_error.kind.error_text().to_owned(),
|
||||
)
|
||||
} else if !moli_fetch::is_fetch_cancelled(&error)
|
||||
&& let Some(failure) = error.downcast_ref::<NetworkFetchFailureContext>()
|
||||
{
|
||||
match failure.request_context() {
|
||||
Some(request) => (
|
||||
request.current_url().clone(),
|
||||
request.request_method().to_owned(),
|
||||
request.request_headers().clone(),
|
||||
failure.network_error_text().to_owned(),
|
||||
),
|
||||
None => (
|
||||
navigation.requested_url.clone(),
|
||||
navigation.request_method.clone(),
|
||||
navigation.request_headers.clone(),
|
||||
failure.network_error_text().to_owned(),
|
||||
),
|
||||
}
|
||||
} else {
|
||||
return Err(error);
|
||||
};
|
||||
let load_inputs = self.navigation_load_inputs_for_navigation(token, navigation)?;
|
||||
let engine = self.navigation_engine_handle_for_load_inputs(&load_inputs);
|
||||
let page_reservation =
|
||||
@@ -110,10 +172,10 @@ impl CdpConnection {
|
||||
engine,
|
||||
page_reservation,
|
||||
load_inputs,
|
||||
unreachable_url: network_error.unreachable_url.clone(),
|
||||
request_method: network_error.request_method.clone(),
|
||||
request_headers: network_error.request_headers.clone(),
|
||||
error_text: network_error.kind.error_text().to_owned(),
|
||||
unreachable_url,
|
||||
request_method,
|
||||
request_headers,
|
||||
error_text,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -201,6 +263,7 @@ impl CdpConnection {
|
||||
&& !response_status_has_no_content(head.status))
|
||||
.then(|| self.reserve_renderer_page_for_owner(owner, &load_inputs, &engine));
|
||||
PreparedCapturedNavigationResponse {
|
||||
reserved_service_worker_client: None,
|
||||
engine,
|
||||
page_reservation,
|
||||
load_inputs,
|
||||
|
||||
@@ -4,14 +4,13 @@ use serde_json::{Value, json};
|
||||
use url::Url;
|
||||
|
||||
use super::super::fetch_support::{
|
||||
ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, DocumentBodySource,
|
||||
InFlightSubresourceFetchRequest, PausedDocumentTransfer, PausedDocumentTransfers,
|
||||
PendingDocumentBodyRead, PendingFetchAuthNavigation, PendingFetchNavigation,
|
||||
PendingFetchResponseBodyStreamRead, PendingFetchResponseBodyStreamReadStart,
|
||||
PendingSubresourceFetchAuthRequest, PendingSubresourceFetchOwnerKind,
|
||||
PendingSubresourceFetchRequest, PendingSubresourceFetchResponseRequest,
|
||||
ResponseStageUrlMatchPolicy, fetch_subresource_interception_config_for_patterns,
|
||||
matching_fetch_pattern,
|
||||
ClaimedSubresourceContinueRequest, CompletedDocumentBodyRead, InFlightSubresourceFetchRequest,
|
||||
PausedDocumentTransfer, PausedDocumentTransfers, PendingDocumentBodyRead,
|
||||
PendingFetchAuthNavigation, PendingFetchNavigation, PendingFetchResponseBodyStreamRead,
|
||||
PendingFetchResponseBodyStreamReadStart, PendingSubresourceFetchAuthRequest,
|
||||
PendingSubresourceFetchOwnerKind, PendingSubresourceFetchRequest,
|
||||
PendingSubresourceFetchResponseRequest, ResponseStageUrlMatchPolicy,
|
||||
fetch_subresource_interception_config_for_patterns, matching_fetch_pattern,
|
||||
};
|
||||
pub(super) use super::super::fetch_support::{
|
||||
FetchInterceptionPattern, FetchRequestStage, FetchResourceTypeFilter,
|
||||
@@ -20,6 +19,8 @@ use super::DocumentNavigationToken;
|
||||
use super::navigation_outcome::NavigationDispatchState;
|
||||
use super::runtime_slot::TargetRuntimeSlot;
|
||||
use crate::automation::{DevToolsNetworkInterceptId, DevToolsNetworkResourceType};
|
||||
#[cfg(test)]
|
||||
use crate::conn::DocumentBodySource;
|
||||
use moli_fetch::url_pattern_matches;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -293,6 +294,12 @@ impl TargetFetchState {
|
||||
}
|
||||
|
||||
fn prune_request_stage_chains_for_session(&mut self, session_id: Option<&str>) {
|
||||
for pending in self.pending_fetch_navigations.values_mut() {
|
||||
pending.remaining_request_stages.retain(|stage| {
|
||||
!stage.owner_kind.drains_on_fetch_disable()
|
||||
|| stage.session_id.as_deref() != session_id
|
||||
});
|
||||
}
|
||||
for pending in self.pending_subresource_fetches.values_mut() {
|
||||
if let Some(chain) = pending.request_stage_chain.as_mut() {
|
||||
chain.remaining_sessions.retain(|stage| {
|
||||
@@ -561,6 +568,7 @@ impl TargetFetchState {
|
||||
.insert(request_id, pending);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn register_pending_fetch_response_navigation(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
@@ -1726,6 +1734,19 @@ impl TargetFetchSubresourceInterceptionSnapshot {
|
||||
matching_auth_required_network_intercepts(&self.network_intercepts, url)
|
||||
}
|
||||
|
||||
pub(crate) fn has_request_stage_candidate(
|
||||
&self,
|
||||
resource_type: DevToolsNetworkResourceType,
|
||||
) -> bool {
|
||||
self.enabled
|
||||
&& self.patterns.iter().any(|pattern| {
|
||||
pattern.request_stage == FetchRequestStage::Request
|
||||
&& pattern
|
||||
.resource_type_filter
|
||||
.is_none_or(|filter| filter.matches_resource_type(resource_type))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn has_response_stage_candidate(
|
||||
&self,
|
||||
resource_type: DevToolsNetworkResourceType,
|
||||
@@ -1978,6 +1999,7 @@ impl TargetFetchOwner {
|
||||
.register_pending_fetch_auth_navigation(request_id, pending);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn register_pending_fetch_response_navigation(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
@@ -2548,6 +2570,7 @@ mod tests {
|
||||
request_id: Some(format!("NETWORK-{request_id}")),
|
||||
loader_id: "LOADER-1".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: test_url("auth-page"),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -11,6 +11,10 @@ fn navigation(token: &DocumentNavigationToken, id: &str) -> NavigationDispatchSt
|
||||
|
||||
fn request(token: &DocumentNavigationToken, id: &str) -> PendingFetchNavigation {
|
||||
PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: id.to_owned(),
|
||||
interception_session_id: Some("SID-A".to_owned()),
|
||||
|
||||
@@ -374,6 +374,7 @@ pub struct NavigationDispatchState {
|
||||
pub request_id: Option<String>,
|
||||
pub loader_id: String,
|
||||
pub request_announced: bool,
|
||||
pub(crate) announced_redirects: usize,
|
||||
pub requested_url: Url,
|
||||
pub request_method: String,
|
||||
/// Text projection used by CDP request events and Fetch interception.
|
||||
|
||||
@@ -441,6 +441,10 @@ impl TargetRuntimeSlot {
|
||||
target_id: String,
|
||||
loader_id: String,
|
||||
) -> DocumentNavigationToken {
|
||||
if let Some(previous) = self.current_document_loader_id().map(str::to_owned) {
|
||||
self.network_agent
|
||||
.clear_fetch_pause_announced_request_id(&previous);
|
||||
}
|
||||
self.devtools_renderer_channel.reopen_after_target_crash();
|
||||
let token = self
|
||||
.page_slot
|
||||
@@ -929,14 +933,14 @@ impl TargetRuntimeSlot {
|
||||
.record_fetch_pause_announced_request_id(request_id);
|
||||
}
|
||||
|
||||
pub(crate) fn replace_child_document_fetch_request_id(
|
||||
pub(crate) fn replace_navigation_fetch_request_id(
|
||||
&mut self,
|
||||
network_id: String,
|
||||
session_id: Option<String>,
|
||||
fetch_id: String,
|
||||
) -> Option<String> {
|
||||
self.network_agent
|
||||
.replace_child_document_fetch_request_id(network_id, session_id, fetch_id)
|
||||
.replace_navigation_fetch_request_id(network_id, session_id, fetch_id)
|
||||
}
|
||||
|
||||
pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool {
|
||||
@@ -975,14 +979,22 @@ impl TargetRuntimeSlot {
|
||||
&mut self,
|
||||
token: &DocumentNavigationToken,
|
||||
) -> bool {
|
||||
self.page_slot
|
||||
.commit_pending_document_navigation_if_matches(token)
|
||||
let committed = self
|
||||
.page_slot
|
||||
.commit_pending_document_navigation_if_matches(token);
|
||||
if committed {
|
||||
self.network_agent
|
||||
.clear_fetch_pause_announced_request_id(&token.loader_id);
|
||||
}
|
||||
committed
|
||||
}
|
||||
|
||||
pub(crate) fn clear_pending_document_navigation_if_loader_matches(
|
||||
&mut self,
|
||||
loader_id: &str,
|
||||
) -> bool {
|
||||
self.network_agent
|
||||
.clear_fetch_pause_announced_request_id(loader_id);
|
||||
self.page_slot
|
||||
.clear_pending_document_navigation_if_loader_matches(loader_id)
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ fn test_navigation_dispatch_state(fetch_request_id: &str) -> NavigationDispatchS
|
||||
request_id: Some(format!("NETWORK-{fetch_request_id}")),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("https://example.test/").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
@@ -812,6 +813,10 @@ fn target_fetch_owner_reports_matching_auth_required_network_intercepts() {
|
||||
fn pending_navigation_rejects_generic_request_action_without_consuming_id() {
|
||||
let mut state = TargetFetchState::default();
|
||||
state.register_pending_fetch_navigation_request(PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: "FETCH-NAV".to_owned(),
|
||||
interception_session_id: Some("SID-fetch".to_owned()),
|
||||
|
||||
@@ -322,6 +322,20 @@ impl ProtocolWorkflowState<'_> {
|
||||
.prepare_navigation_load_error(Some(&token), &navigation, error)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_fetch_navigation_redirect(
|
||||
self,
|
||||
pending: super::PendingFetchNavigation,
|
||||
executor: moli_core::network::NavigationInterception,
|
||||
prior: moli_fetch::NetworkObservationJournal,
|
||||
) -> crate::domains::fetch::RedirectNavigation {
|
||||
crate::domains::fetch::prepare_navigation_redirect(
|
||||
self.connection,
|
||||
pending,
|
||||
executor,
|
||||
prior,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn accept_fetch_navigation_response(
|
||||
self,
|
||||
pending: super::PendingFetchNavigation,
|
||||
|
||||
@@ -1904,6 +1904,7 @@ fn materialized_navigation_test_state(
|
||||
request_id: Some(loader_id.to_owned()),
|
||||
loader_id: loader_id.to_owned(),
|
||||
request_announced: true,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse(requested_url).unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -234,7 +234,7 @@ async fn prepared_navigation_request_freezes_policy_and_cancels_before_headers_o
|
||||
ctx.process_async(json!({"id": 11, "sessionId": "SID-1", "method": "Network.setExtraHTTPHeaders", "params": {"headers": {"X-Snapshot": "after"}}})).await;
|
||||
ctx.expect_result(11, json!({}), Some("SID-1"));
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
let (result, ()) = tokio::join!(job.wait(), async {
|
||||
let (result, ()) = tokio::join!(job.fetch_raw_stream(), async {
|
||||
requested_rx.await.unwrap();
|
||||
ctx.with_fixture(|conn| {
|
||||
conn.start_document_navigation_for_owner(&navigation.owner, "replacement".into())
|
||||
@@ -269,7 +269,7 @@ async fn dropping_prepared_navigation_header_wait_cancels_its_transport() {
|
||||
.unwrap()
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
let mut wait = Box::pin(job.wait());
|
||||
let mut wait = Box::pin(job.fetch_raw_stream());
|
||||
tokio::select! {
|
||||
result = &mut wait => panic!("headers must stay held: {result:?}"),
|
||||
requested = requested_rx => requested.unwrap(),
|
||||
@@ -580,6 +580,7 @@ pub(super) fn navigation_fixture() -> (
|
||||
request_id: Some("LID-test".to_owned()),
|
||||
loader_id: "LID-test".to_owned(),
|
||||
request_announced: true,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("https://original.example/document").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
@@ -1014,14 +1015,14 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context(
|
||||
let buffered_error = async {
|
||||
let request =
|
||||
ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?;
|
||||
request.wait_buffered().await
|
||||
request.fetch_buffered().await
|
||||
}
|
||||
.await
|
||||
.expect_err("offline response fetch must fail");
|
||||
let streaming_error = async {
|
||||
let request =
|
||||
ctx.with_fixture(|conn| conn.prepare_navigation_request(&token, &navigation, None))?;
|
||||
request.wait().await
|
||||
request.fetch_raw_stream().await
|
||||
}
|
||||
.await
|
||||
.expect_err("offline streaming response fetch must fail");
|
||||
@@ -1038,7 +1039,7 @@ async fn offline_navigation_loaders_preserve_typed_error_causes_through_context(
|
||||
}),
|
||||
)
|
||||
})?;
|
||||
request.wait_buffered().await
|
||||
request.fetch_buffered().await
|
||||
}
|
||||
.await
|
||||
.expect_err("offline authenticated response fetch must fail");
|
||||
|
||||
@@ -119,6 +119,7 @@ async fn buffered_navigation_for_inactive_session_retains_its_target_engine() {
|
||||
request_id: Some("LOADER-target".to_owned()),
|
||||
loader_id: "LOADER-target".to_owned(),
|
||||
request_announced: true,
|
||||
announced_redirects: 0,
|
||||
requested_url: requested_url.clone(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -780,6 +780,7 @@ mod tests {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: "LID-1".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("https://example.test/start").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -10,6 +10,8 @@ mod document_navigation;
|
||||
mod document_request;
|
||||
mod helpers;
|
||||
mod navigation;
|
||||
mod navigation_interception;
|
||||
pub(crate) use navigation_interception::prepare_navigation_request_pause;
|
||||
mod params;
|
||||
mod patterns;
|
||||
mod state;
|
||||
@@ -43,7 +45,8 @@ pub(crate) use crate::conn::PendingFetchAuthNavigation;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::conn::PendingFetchNavigation;
|
||||
pub(crate) use document_request::{
|
||||
FetchNavigationResponse, PreparedFetchNavigationResponse, accept_navigation_response,
|
||||
FetchNavigationResponse, PreparedFetchNavigationResponse, RedirectNavigation,
|
||||
accept_navigation_response, prepare_navigation_redirect,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use helpers::encode_basic_auth;
|
||||
|
||||
@@ -51,6 +51,10 @@ pub(super) fn navigation_for_auth_retry(
|
||||
pending: crate::conn::PendingFetchAuthNavigation,
|
||||
) -> PendingFetchNavigation {
|
||||
PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: pending.auth_retry_budget,
|
||||
fetch_request_id: pending.response_stage_request_id,
|
||||
interception_session_id: pending.interception_session_id,
|
||||
@@ -75,6 +79,10 @@ pub(super) fn canceled_auth_response(
|
||||
};
|
||||
(
|
||||
PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: pending.auth_retry_budget,
|
||||
fetch_request_id: pending.response_stage_request_id,
|
||||
interception_session_id: pending.interception_session_id,
|
||||
|
||||
@@ -403,6 +403,21 @@ fn start_devtools_continue_intercepted_request_command(
|
||||
pending.navigation.request_load_policy,
|
||||
None,
|
||||
);
|
||||
if !pending.remaining_request_stages.is_empty() {
|
||||
let stage = pending.remaining_request_stages.remove(0);
|
||||
pending.fetch_request_id = stage.request_id;
|
||||
pending.interception_session_id = stage.session_id;
|
||||
pending.redirected_request_id = stage.redirected_request_id;
|
||||
let event = super::request_paused_background_event(
|
||||
pending.interception_session_id.as_deref(),
|
||||
&pending,
|
||||
);
|
||||
let navigation_owner = pending.navigation.owner.clone();
|
||||
conn.register_pending_fetch_navigation_request_for_owner(&navigation_owner, pending);
|
||||
let mut plan = CommandOutputPlan::success();
|
||||
plan.push_background_event(event);
|
||||
return FetchCommandTaskStep::Complete(plan);
|
||||
}
|
||||
let termination = super::navigation::NavigationTermination::new(
|
||||
conn,
|
||||
owner,
|
||||
@@ -1560,6 +1575,7 @@ pub(super) fn continue_streaming_document_response_in_background(
|
||||
response_headers: Vec<(String, Vec<u8>)>,
|
||||
) {
|
||||
let PendingStreamingDocumentResponseNavigation {
|
||||
reserved_service_worker_client,
|
||||
document_navigation_token,
|
||||
navigation,
|
||||
response,
|
||||
@@ -1575,7 +1591,7 @@ pub(super) fn continue_streaming_document_response_in_background(
|
||||
response_headers,
|
||||
body_progress_source,
|
||||
) {
|
||||
Ok(job) => job,
|
||||
Ok(job) => job.with_service_worker_client(reserved_service_worker_client),
|
||||
Err(BackgroundNavigationClaimError::AlreadyClaimed) => return,
|
||||
Err(BackgroundNavigationClaimError::Stale) => {
|
||||
let _ = sender.send(page::BackgroundNavigationCompletion::new(
|
||||
|
||||
@@ -498,17 +498,23 @@ pub(super) fn start_fulfill(
|
||||
response_code,
|
||||
response_headers,
|
||||
decoded_body,
|
||||
} => conn.prepare_synthetic_navigation_response(
|
||||
Some(&token),
|
||||
&navigation,
|
||||
navigation.requested_url.clone(),
|
||||
response_code,
|
||||
response_headers,
|
||||
crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body(decoded_body),
|
||||
pending.request_cookie_report,
|
||||
Default::default(),
|
||||
network::MainDocumentBodyProgressSource::default(),
|
||||
),
|
||||
} => {
|
||||
let client = pending.reserved_service_worker_client;
|
||||
conn.prepare_synthetic_navigation_response(
|
||||
Some(&token),
|
||||
&navigation,
|
||||
navigation.requested_url.clone(),
|
||||
response_code,
|
||||
response_headers,
|
||||
crate::conn::CapturedBody::from_optional_renderer_synthetic_response_body(
|
||||
decoded_body,
|
||||
),
|
||||
pending.request_cookie_report,
|
||||
Default::default(),
|
||||
network::MainDocumentBodyProgressSource::default(),
|
||||
)
|
||||
.map(|response| response.with_service_worker_client(client))
|
||||
}
|
||||
DocumentFulfillment::ResponseTransfer {
|
||||
transfer,
|
||||
response_code,
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use crate::conn::{
|
||||
CdpConnection, NavigationLoadOutcome, PendingFetchNavigation,
|
||||
PendingStreamingDocumentResponseNavigation, PreparedCapturedNavigationResponse,
|
||||
PreparedNavigationRequest, ProtocolWorkflowDriver,
|
||||
ProtocolWorkflowDriver,
|
||||
};
|
||||
use crate::domains::{
|
||||
command_output::CommandOutputBuffer, command_output::CommandOutputPlan, network,
|
||||
};
|
||||
use moli_core::network::NavigationInterception;
|
||||
use moli_core::page::SubresourceAuthCredentials;
|
||||
use moli_fetch::{
|
||||
NetworkFetchResult, NetworkObservationJournal, RawResponse, StreamingRawResponse,
|
||||
@@ -23,7 +24,8 @@ use super::{
|
||||
/// awaits; only interpreting the accepted response needs another state turn.
|
||||
pub(crate) struct PreparedInterceptedNavigationRequest {
|
||||
pending: PendingFetchNavigation,
|
||||
request: Option<Result<PreparedNavigationRequest>>,
|
||||
request: Option<Result<NavigationInterception>>,
|
||||
renderer_runtime: Option<moli_core::runtime::RendererBrowserContextRuntime>,
|
||||
buffered: bool,
|
||||
handle_auth: bool,
|
||||
prior_journal: Option<NetworkObservationJournal>,
|
||||
@@ -62,6 +64,11 @@ impl PreparedInterceptedNavigationRequest {
|
||||
&pending.navigation.requested_url,
|
||||
);
|
||||
let buffered = !navigation::navigation_response_stage_auth_can_stream(auth.as_ref());
|
||||
let mut prior_journal = prior_journal;
|
||||
let renderer_runtime = conn
|
||||
.target_owner_identity_for_owner(&pending.navigation.owner)
|
||||
.and_then(|(id, _)| conn.browser_context_by_id(&id))
|
||||
.map(|context| context.renderer_runtime());
|
||||
let request = if auth.is_some() && !pending.auth_retry_budget.try_restart() {
|
||||
None
|
||||
} else if handle_auth && buffered && pending.intercept_response {
|
||||
@@ -73,7 +80,24 @@ impl PreparedInterceptedNavigationRequest {
|
||||
.into()))
|
||||
} else {
|
||||
network::record_main_document_request_body(conn, &pending.navigation);
|
||||
Some(
|
||||
let prepared = if let Some((mut interception, journal)) = pending.interception.take() {
|
||||
prior_journal = Some(navigation::merge_network_observations(
|
||||
prior_journal,
|
||||
journal,
|
||||
));
|
||||
let request = interception.request_mut();
|
||||
request.url = pending.navigation.requested_url.clone();
|
||||
request.method = pending.navigation.request_method.clone();
|
||||
request.request_headers = pending.navigation.request_headers.clone();
|
||||
request.body = pending.navigation.clone_request_body_bytes();
|
||||
*request = request
|
||||
.clone()
|
||||
.with_redirect_headers(pending.navigation.redirect_headers.clone());
|
||||
if let Some(auth) = auth {
|
||||
request.set_auth(Some(auth.into()));
|
||||
}
|
||||
Ok(interception)
|
||||
} else {
|
||||
conn.prepare_navigation_request(
|
||||
pending
|
||||
.document_navigation_token
|
||||
@@ -81,12 +105,26 @@ impl PreparedInterceptedNavigationRequest {
|
||||
.expect("request token"),
|
||||
&pending.navigation,
|
||||
auth,
|
||||
),
|
||||
)
|
||||
)
|
||||
};
|
||||
Some(prepared.map(|mut prepared| {
|
||||
let intercept = conn
|
||||
.target_fetch_subresource_interception_snapshot_for_owner(
|
||||
&pending.navigation.owner,
|
||||
)
|
||||
.is_some_and(|snapshot| {
|
||||
snapshot.has_request_stage_candidate(
|
||||
crate::automation::DevToolsNetworkResourceType::Document,
|
||||
)
|
||||
});
|
||||
prepared.set_interception(intercept);
|
||||
prepared
|
||||
}))
|
||||
};
|
||||
Self {
|
||||
pending,
|
||||
request,
|
||||
renderer_runtime,
|
||||
buffered,
|
||||
handle_auth,
|
||||
prior_journal,
|
||||
@@ -95,43 +133,134 @@ impl PreparedInterceptedNavigationRequest {
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare(
|
||||
self,
|
||||
mut self,
|
||||
driver: &mut ProtocolWorkflowDriver,
|
||||
) -> PreparedFetchNavigationResponse {
|
||||
let Some(request) = self.request else {
|
||||
let Some(request) = self.request.take() else {
|
||||
return PreparedFetchNavigationResponse::Result(Ok(
|
||||
NavigationLoadOutcome::network_failure(
|
||||
moli_fetch::AuthRetryBudget::ERROR_TEXT.to_owned(),
|
||||
),
|
||||
));
|
||||
};
|
||||
let response = match request {
|
||||
Ok(request) if self.buffered => request
|
||||
.wait_buffered()
|
||||
.await
|
||||
.map(FetchNavigationResponse::Captured),
|
||||
Ok(request) => match request.wait().await {
|
||||
Ok(response)
|
||||
if moli_web_mime::response_headers_indicate_attachment_download(
|
||||
&response.response().headers,
|
||||
) || (self.handle_auth
|
||||
&& matches!(response.response().status, 401 | 407)
|
||||
&& extract_auth_challenge(&response.response().headers).is_some()) =>
|
||||
let mut request = match request {
|
||||
Ok(request) => request,
|
||||
Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)),
|
||||
};
|
||||
let response = loop {
|
||||
let service_worker_response = if let Some(runtime) = &self.renderer_runtime {
|
||||
match runtime
|
||||
.fetch_service_worker_main_resource_for_navigation(
|
||||
request.request(),
|
||||
request.resource_loader(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let (response, journal) = response.into_parts_with_observation_journal();
|
||||
response
|
||||
.into_materialized_raw_response()
|
||||
.await
|
||||
.map(|response| {
|
||||
FetchNavigationResponse::Captured(
|
||||
NetworkFetchResult::with_observation_journal(response, journal),
|
||||
Ok(fetch) => {
|
||||
self.pending.reserved_service_worker_client = fetch.reserved_client;
|
||||
fetch.response
|
||||
}
|
||||
Err(error) => break Err(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let response = if self.buffered && service_worker_response.is_none() {
|
||||
request
|
||||
.fetch_buffered()
|
||||
.await
|
||||
.with_context(|| format!("failed to fetch page `{}`", request.request().url))
|
||||
.map(FetchNavigationResponse::Captured)
|
||||
} else {
|
||||
let response = if let Some(response) = service_worker_response {
|
||||
request
|
||||
.resource_loader()
|
||||
.note_service_worker_response_ready()
|
||||
.and_then(|()| {
|
||||
moli_core::runtime::streaming_raw_response_from_navigation_response(
|
||||
response,
|
||||
)
|
||||
})
|
||||
.map_err(|error| error.context("failed to read page body from stream"))
|
||||
.map(NetworkFetchResult::without_request_observation)
|
||||
} else {
|
||||
request.fetch_raw_stream().await
|
||||
};
|
||||
match response
|
||||
.with_context(|| format!("failed to fetch page `{}`", request.request().url))
|
||||
{
|
||||
Ok(response)
|
||||
if moli_web_mime::response_headers_indicate_attachment_download(
|
||||
&response.response().headers,
|
||||
) || (self.handle_auth
|
||||
&& matches!(response.response().status, 401 | 407)
|
||||
&& extract_auth_challenge(&response.response().headers).is_some()) =>
|
||||
{
|
||||
let (response, journal) = response.into_parts_with_observation_journal();
|
||||
response
|
||||
.into_materialized_raw_response()
|
||||
.await
|
||||
.map(|response| {
|
||||
FetchNavigationResponse::Captured(
|
||||
NetworkFetchResult::with_observation_journal(response, journal),
|
||||
)
|
||||
})
|
||||
.map_err(|error| error.context("failed to read page body from stream"))
|
||||
}
|
||||
response => response.map(FetchNavigationResponse::Streaming),
|
||||
}
|
||||
response => response.map(FetchNavigationResponse::Streaming),
|
||||
},
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
let head = match &response {
|
||||
Ok(FetchNavigationResponse::Streaming(response)) => {
|
||||
Some(response.response().head())
|
||||
}
|
||||
Ok(FetchNavigationResponse::Captured(response)) => Some(response.response().head()),
|
||||
Err(_) => None,
|
||||
};
|
||||
match head.as_ref().map(|head| request.follow_redirect(head)) {
|
||||
Some(Ok(Some(_))) => {}
|
||||
Some(Err(error)) => break Err(error),
|
||||
_ => break response,
|
||||
}
|
||||
let journal = match response.expect("redirect has a response") {
|
||||
FetchNavigationResponse::Streaming(response) => {
|
||||
response.into_parts_with_observation_journal().1
|
||||
}
|
||||
FetchNavigationResponse::Captured(response) => {
|
||||
response.into_parts_with_observation_journal().1
|
||||
}
|
||||
};
|
||||
self.prior_journal = Some(navigation::merge_network_observations(
|
||||
self.prior_journal.take(),
|
||||
journal,
|
||||
));
|
||||
let hop = request.request();
|
||||
self.pending.navigation.requested_url = hop.url.clone();
|
||||
self.pending.navigation.request_method = hop.method.clone();
|
||||
self.pending.navigation.request_headers = hop.request_headers.clone();
|
||||
self.pending.navigation.request_body_bytes = hop.body.clone();
|
||||
self.pending.navigation.request_body = hop
|
||||
.body
|
||||
.as_ref()
|
||||
.map(|body| String::from_utf8_lossy(body).into_owned());
|
||||
self.pending.navigation.redirect_chain = hop.redirect_chain().to_vec();
|
||||
self.pending.navigation.redirect_headers = None;
|
||||
let pending = self.pending;
|
||||
let executor = request.clone();
|
||||
let prior = self.prior_journal.clone().unwrap_or_default();
|
||||
let redirected = driver.with_state(move |state| {
|
||||
state.prepare_fetch_navigation_redirect(pending, executor, prior)
|
||||
});
|
||||
match redirected {
|
||||
Ok(RedirectNavigation::Continue(pending)) => self.pending = *pending,
|
||||
Ok(RedirectNavigation::Paused(plan)) => {
|
||||
self.transferred.mark_transferred();
|
||||
return PreparedFetchNavigationResponse::Transferred(*plan);
|
||||
}
|
||||
Ok(RedirectNavigation::Superseded) => {
|
||||
return PreparedFetchNavigationResponse::Superseded;
|
||||
}
|
||||
Err(error) => return PreparedFetchNavigationResponse::Result(Err(error)),
|
||||
}
|
||||
};
|
||||
let result = driver.with_state(move |state| {
|
||||
let result = state.accept_fetch_navigation_response(
|
||||
@@ -219,7 +348,12 @@ pub(crate) fn accept_navigation_response(
|
||||
),
|
||||
journal,
|
||||
network::MainDocumentBodyProgressSource::default(),
|
||||
),
|
||||
)
|
||||
.map(|response| {
|
||||
response.with_service_worker_client(
|
||||
pending.reserved_service_worker_client.take(),
|
||||
)
|
||||
}),
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -248,7 +382,12 @@ pub(crate) fn accept_navigation_response(
|
||||
network::MainDocumentBodyProgressSource::default(),
|
||||
None,
|
||||
Vec::new(),
|
||||
),
|
||||
)
|
||||
.map(|response| {
|
||||
response.with_service_worker_client(
|
||||
pending.reserved_service_worker_client.take(),
|
||||
)
|
||||
}),
|
||||
));
|
||||
};
|
||||
let (response, network_observation_journal) =
|
||||
@@ -257,6 +396,9 @@ pub(crate) fn accept_navigation_response(
|
||||
conn,
|
||||
sender,
|
||||
PendingStreamingDocumentResponseNavigation {
|
||||
reserved_service_worker_client: pending
|
||||
.reserved_service_worker_client
|
||||
.take(),
|
||||
document_navigation_token: pending
|
||||
.document_navigation_token
|
||||
.expect("request token"),
|
||||
@@ -273,3 +415,103 @@ pub(crate) fn accept_navigation_response(
|
||||
}
|
||||
PreparedFetchNavigationResponse::Transferred(output.into_plan())
|
||||
}
|
||||
|
||||
pub(crate) enum RedirectNavigation {
|
||||
Continue(Box<PendingFetchNavigation>),
|
||||
Paused(Box<crate::domains::command_output::CommandOutputPlan>),
|
||||
Superseded,
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_navigation_redirect(
|
||||
conn: &mut CdpConnection,
|
||||
mut pending: PendingFetchNavigation,
|
||||
executor: moli_core::network::NavigationInterception,
|
||||
prior: NetworkObservationJournal,
|
||||
) -> RedirectNavigation {
|
||||
if !pending
|
||||
.document_navigation_token
|
||||
.as_ref()
|
||||
.is_some_and(|token| {
|
||||
conn.accepts_pending_document_navigation_for_owner(&pending.navigation.owner, token)
|
||||
})
|
||||
{
|
||||
return RedirectNavigation::Superseded;
|
||||
}
|
||||
let Some((id, _)) =
|
||||
conn.allocate_pending_subresource_fetch_request_ids_for_owner(&pending.navigation.owner)
|
||||
else {
|
||||
return RedirectNavigation::Superseded;
|
||||
};
|
||||
pending.fetch_request_id = id;
|
||||
// The same selector and per-handler hop IDs serve root and child Documents.
|
||||
let paused = super::prepare_navigation_request_pause(conn, &mut pending);
|
||||
if !paused {
|
||||
return RedirectNavigation::Continue(Box::new(pending));
|
||||
}
|
||||
pending.interception = Some((executor, prior));
|
||||
let owner = pending.navigation.owner.clone();
|
||||
let event = super::request_paused_background_event(
|
||||
pending.interception_session_id.as_deref(),
|
||||
&pending,
|
||||
);
|
||||
let mut plan = crate::domains::command_output::CommandOutputPlan::default();
|
||||
append_redirect_request_events(conn, &mut plan, &mut pending);
|
||||
if conn
|
||||
.register_pending_fetch_navigation_request_for_owner(&owner, pending)
|
||||
.is_none()
|
||||
{
|
||||
return RedirectNavigation::Superseded;
|
||||
}
|
||||
plan.push_background_event(event);
|
||||
RedirectNavigation::Paused(Box::new(plan))
|
||||
}
|
||||
|
||||
fn append_redirect_request_events(
|
||||
conn: &CdpConnection,
|
||||
plan: &mut crate::domains::command_output::CommandOutputPlan,
|
||||
pending: &mut PendingFetchNavigation,
|
||||
) {
|
||||
let nav = &pending.navigation;
|
||||
let Some(network_id) = &nav.request_id else {
|
||||
return;
|
||||
};
|
||||
let info = moli_core::page::PendingSubresourceFetchInfo {
|
||||
internal_id: 0,
|
||||
network_request_handle: None,
|
||||
frame_id: Some(nav.frame_id.clone()),
|
||||
document_url: nav.requested_url.clone(),
|
||||
url: nav.requested_url.clone(),
|
||||
websocket_socket_id: None,
|
||||
method: nav.request_method.clone(),
|
||||
request_headers: nav.request_headers.clone(),
|
||||
request_body: nav.request_body.clone(),
|
||||
request_body_bytes: nav.clone_request_body_bytes(),
|
||||
resource_type: moli_core::page::SubresourceResourceType::Document,
|
||||
request_cookie_report: pending.request_cookie_report.clone(),
|
||||
};
|
||||
for redirect in nav.redirect_chain.iter().skip(nav.announced_redirects) {
|
||||
let mut info = info.clone();
|
||||
info.url = redirect.to_url.clone();
|
||||
let output = network::TargetSubresourceFetchPauseNetworkOutput::from_pending_fetch_info(
|
||||
network_id.clone(),
|
||||
nav.frame_id.clone(),
|
||||
nav.loader_id.clone(),
|
||||
crate::conn::monotonic_timestamp_seconds(),
|
||||
nav.requested_url.clone(),
|
||||
&info,
|
||||
)
|
||||
.with_child_document(Some(moli_core::page::ChildDocumentFetchRequest {
|
||||
loader_id: nav.loader_id.clone(),
|
||||
redirect: Some(redirect.clone().into()),
|
||||
}));
|
||||
for session in conn.network_event_session_ids_for_owner(&nav.owner) {
|
||||
plan.extend_background_events(
|
||||
network::fetch_subresource_initial_request_network_events(
|
||||
session.as_deref(),
|
||||
&output,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
pending.navigation.announced_redirects = pending.navigation.redirect_chain.len();
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use moli_core::page::{
|
||||
SubresourceAuthChallenge, SubresourceAuthCredentials, extract_subresource_auth_challenge,
|
||||
subresource_auth_credentials_for_challenge,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use serde_json::{Value, json};
|
||||
use url::Url;
|
||||
|
||||
use super::params::HeaderEntry;
|
||||
@@ -518,7 +518,10 @@ pub(crate) fn request_paused_background_event(
|
||||
.map(DevToolsRequestId::from),
|
||||
auth_challenge: None,
|
||||
};
|
||||
let payload = network::fetch_request_paused_params(&network_event);
|
||||
let mut payload = network::fetch_request_paused_params(&network_event);
|
||||
if let Some(previous) = &pending.redirected_request_id {
|
||||
payload["redirectedRequestId"] = json!(previous);
|
||||
}
|
||||
let message = build_event("Fetch.requestPaused", payload, session_id);
|
||||
BackgroundProtocolEvent::immediate_automation_event(
|
||||
message,
|
||||
|
||||
@@ -143,7 +143,7 @@ pub(super) fn navigation_response_stage_auth_can_stream(
|
||||
pub(super) fn pause_streaming_response_stage_navigation_into_buffer(
|
||||
conn: &mut CdpConnection,
|
||||
out: &mut CommandOutputBuffer,
|
||||
pending: PendingFetchNavigation,
|
||||
mut pending: PendingFetchNavigation,
|
||||
response: NetworkFetchResult<StreamingRawResponse>,
|
||||
) {
|
||||
let response_head = response.response();
|
||||
@@ -176,19 +176,23 @@ pub(super) fn pause_streaming_response_stage_navigation_into_buffer(
|
||||
// here would wait on the old document's debugger pause before we could even
|
||||
// publish Fetch.requestPaused. Continue/fulfill owns document replacement.
|
||||
let (response, network_observation_journal) = response.into_parts_with_observation_journal();
|
||||
conn.register_pending_fetch_response_navigation_for_owner(
|
||||
conn.register_pending_fetch_response_transfer_for_owner(
|
||||
&pending.navigation.owner,
|
||||
pending.fetch_request_id.clone(),
|
||||
pending.document_navigation_token.clone(),
|
||||
pending.navigation.clone(),
|
||||
DocumentBodySource::StreamingRaw {
|
||||
requested_url: pending.navigation.requested_url.clone(),
|
||||
request_method: pending.navigation.request_method.clone(),
|
||||
request_headers: pending.navigation.request_headers.clone(),
|
||||
response,
|
||||
network_observation_journal,
|
||||
body_progress_source,
|
||||
},
|
||||
crate::conn::PausedDocumentTransfer::pending(
|
||||
pending.fetch_request_id.clone(),
|
||||
pending.document_navigation_token.clone(),
|
||||
pending.navigation.clone(),
|
||||
DocumentBodySource::StreamingRaw {
|
||||
requested_url: pending.navigation.requested_url.clone(),
|
||||
request_method: pending.navigation.request_method.clone(),
|
||||
request_headers: pending.navigation.request_headers.clone(),
|
||||
response,
|
||||
network_observation_journal,
|
||||
body_progress_source,
|
||||
},
|
||||
)
|
||||
.with_service_worker_client(pending.reserved_service_worker_client.take()),
|
||||
);
|
||||
out.extend_background_events_after_messages([navigation_response_stage_request_paused_event(
|
||||
pending.interception_session_id.as_deref(),
|
||||
@@ -244,7 +248,7 @@ pub(super) fn pause_data_url_response_stage_navigation_into_buffer(
|
||||
pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
|
||||
conn: &mut CdpConnection,
|
||||
out: &mut CommandOutputBuffer,
|
||||
pending: PendingFetchNavigation,
|
||||
mut pending: PendingFetchNavigation,
|
||||
response: NetworkFetchResult<RawResponse>,
|
||||
) {
|
||||
let response_status = response.response().status;
|
||||
@@ -252,18 +256,22 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
|
||||
let response_headers = response.response().headers.clone();
|
||||
let request_cookie_report = response.response().request_cookie_report.clone();
|
||||
let (response, network_observation_journal) = response.into_parts_with_observation_journal();
|
||||
conn.register_pending_fetch_response_navigation_for_owner(
|
||||
conn.register_pending_fetch_response_transfer_for_owner(
|
||||
&pending.navigation.owner,
|
||||
pending.fetch_request_id.clone(),
|
||||
pending.document_navigation_token.clone(),
|
||||
pending.navigation.clone(),
|
||||
DocumentBodySource::BufferedRaw {
|
||||
requested_url: pending.navigation.requested_url.clone(),
|
||||
request_method: pending.navigation.request_method.clone(),
|
||||
request_headers: pending.navigation.request_headers.clone(),
|
||||
response,
|
||||
network_observation_journal,
|
||||
},
|
||||
crate::conn::PausedDocumentTransfer::pending(
|
||||
pending.fetch_request_id.clone(),
|
||||
pending.document_navigation_token.clone(),
|
||||
pending.navigation.clone(),
|
||||
DocumentBodySource::BufferedRaw {
|
||||
requested_url: pending.navigation.requested_url.clone(),
|
||||
request_method: pending.navigation.request_method.clone(),
|
||||
request_headers: pending.navigation.request_headers.clone(),
|
||||
response,
|
||||
network_observation_journal,
|
||||
},
|
||||
)
|
||||
.with_service_worker_client(pending.reserved_service_worker_client.take()),
|
||||
);
|
||||
out.extend_background_events_after_messages([navigation_response_stage_request_paused_event(
|
||||
pending.interception_session_id.as_deref(),
|
||||
@@ -276,6 +284,15 @@ pub(super) fn pause_buffered_raw_response_stage_navigation_into_buffer(
|
||||
)]);
|
||||
}
|
||||
|
||||
pub(super) fn merge_network_observations(
|
||||
prior: Option<NetworkObservationJournal>,
|
||||
current: NetworkObservationJournal,
|
||||
) -> NetworkObservationJournal {
|
||||
let mut journal = prior.unwrap_or_default();
|
||||
journal.append(current);
|
||||
journal
|
||||
}
|
||||
|
||||
pub(super) fn append_prior_network_observations<R>(
|
||||
response: NetworkFetchResult<R>,
|
||||
prior: Option<NetworkObservationJournal>,
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::automation::DevToolsNetworkResourceType;
|
||||
use crate::conn::{
|
||||
CdpConnection, CommandOwnerScope, PendingFetchNavigation, PendingSubresourceFetchOwnerKind,
|
||||
PendingSubresourceFetchRequestStage, TargetFetchSubresourceInterceptionSnapshot,
|
||||
};
|
||||
|
||||
type NavigationHopIds = HashMap<Option<String>, (String, Option<String>)>;
|
||||
|
||||
/// Every Fetch handler advances its own hop identity, including unmatched
|
||||
/// hops. The transport's navigation identity is independent of these IDs.
|
||||
pub(super) fn navigation_hop_ids(
|
||||
conn: &mut CdpConnection,
|
||||
owner: &CommandOwnerScope,
|
||||
snapshot: &TargetFetchSubresourceInterceptionSnapshot,
|
||||
network_id: &str,
|
||||
first_id: &str,
|
||||
redirected: bool,
|
||||
) -> Option<NavigationHopIds> {
|
||||
let mut ids = HashMap::new();
|
||||
for (index, session_id) in snapshot
|
||||
.fetch_session_ids(owner.session_id())
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let id = if index == 0 {
|
||||
first_id.to_owned()
|
||||
} else {
|
||||
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)?
|
||||
.0
|
||||
};
|
||||
let previous = conn
|
||||
.runtime_session_owner_slot_mut_for_owner(owner)
|
||||
.and_then(|slot| {
|
||||
slot.replace_navigation_fetch_request_id(
|
||||
network_id.to_owned(),
|
||||
session_id.clone(),
|
||||
id.clone(),
|
||||
)
|
||||
});
|
||||
ids.insert(session_id, (id, previous.filter(|_| redirected)));
|
||||
}
|
||||
Some(ids)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_navigation_request_pause(
|
||||
conn: &mut CdpConnection,
|
||||
pending: &mut PendingFetchNavigation,
|
||||
) -> bool {
|
||||
let owner = &pending.navigation.owner;
|
||||
let Some(snapshot) = conn.target_fetch_subresource_interception_snapshot_for_owner(owner)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let redirected = !pending.navigation.redirect_chain.is_empty();
|
||||
let Some(ids) = navigation_hop_ids(
|
||||
conn,
|
||||
owner,
|
||||
&snapshot,
|
||||
&pending.navigation.loader_id,
|
||||
&pending.fetch_request_id,
|
||||
redirected,
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
let blocked = snapshot.matching_network_intercepts(
|
||||
crate::conn::FetchRequestStage::Request,
|
||||
DevToolsNetworkResourceType::Document,
|
||||
&pending.navigation.requested_url,
|
||||
);
|
||||
let mut stages = Vec::new();
|
||||
for stage in snapshot.matching_request_stage_pause_sessions(
|
||||
owner.session_id(),
|
||||
DevToolsNetworkResourceType::Document,
|
||||
&pending.navigation.requested_url,
|
||||
) {
|
||||
let (id, previous) = if stage.owner_kind == PendingSubresourceFetchOwnerKind::Fetch {
|
||||
ids.get(&stage.session_id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| (pending.fetch_request_id.clone(), None))
|
||||
} else {
|
||||
let Some((id, _)) =
|
||||
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
(id, None)
|
||||
};
|
||||
stages.push(PendingSubresourceFetchRequestStage {
|
||||
session_id: stage.session_id,
|
||||
owner_kind: stage.owner_kind,
|
||||
request_id: id,
|
||||
redirected_request_id: previous,
|
||||
blocked_intercepts: blocked.clone(),
|
||||
});
|
||||
}
|
||||
if stages.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let stage = stages.remove(0);
|
||||
pending.fetch_request_id = stage.request_id;
|
||||
pending.interception_session_id = stage.session_id;
|
||||
pending.redirected_request_id = stage.redirected_request_id;
|
||||
pending.remaining_request_stages = stages;
|
||||
true
|
||||
}
|
||||
@@ -193,35 +193,17 @@ fn prepare_subresource_fetch_pause_sources(
|
||||
// including hops that do not match that handler's URL patterns.
|
||||
let mut child_fetch_ids = HashMap::new();
|
||||
if let Some(document) = child_document.as_ref() {
|
||||
for (index, fetch_session_id) in fetch_snapshot
|
||||
.fetch_session_ids(session_id)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let fetch_id = if index == 0 {
|
||||
request_id.clone()
|
||||
} else {
|
||||
let Some((id, _)) =
|
||||
conn.allocate_pending_subresource_fetch_request_ids_for_owner(owner)
|
||||
else {
|
||||
return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic);
|
||||
};
|
||||
id
|
||||
};
|
||||
let previous = conn
|
||||
.runtime_session_owner_slot_mut_for_owner(owner)
|
||||
.and_then(|slot| {
|
||||
slot.replace_child_document_fetch_request_id(
|
||||
network_request_id.clone(),
|
||||
fetch_session_id.clone(),
|
||||
fetch_id.clone(),
|
||||
)
|
||||
});
|
||||
child_fetch_ids.insert(
|
||||
fetch_session_id,
|
||||
(fetch_id, previous.filter(|_| document.redirect.is_some())),
|
||||
);
|
||||
}
|
||||
let Some(ids) = super::navigation_interception::navigation_hop_ids(
|
||||
conn,
|
||||
owner,
|
||||
&fetch_snapshot,
|
||||
&network_request_id,
|
||||
&request_id,
|
||||
document.redirect.is_some(),
|
||||
) else {
|
||||
return network::NetworkPreparedOutputs::from_subresource_fetch_pauses_and_continuations(outputs, automatic);
|
||||
};
|
||||
child_fetch_ids = ids;
|
||||
}
|
||||
if request_stage_pause_sessions.is_empty() {
|
||||
if child_document.is_some() {
|
||||
|
||||
@@ -1212,6 +1212,10 @@ async fn disable_clears_fetch_state() {
|
||||
bc.active_page_target_mut()
|
||||
.fetch_owner
|
||||
.register_pending_fetch_navigation_request(PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: "INT-1".to_owned(),
|
||||
interception_session_id: Some("SID-1".to_owned()),
|
||||
@@ -1235,6 +1239,7 @@ async fn disable_clears_fetch_state() {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/auth").unwrap(),
|
||||
request_method: "POST".to_owned(),
|
||||
request_body: Some("payload".to_owned()),
|
||||
@@ -1282,6 +1287,7 @@ async fn disable_clears_fetch_state() {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/auth").unwrap(),
|
||||
request_method: "POST".to_owned(),
|
||||
request_body: Some("payload".to_owned()),
|
||||
@@ -1530,6 +1536,7 @@ async fn continue_with_auth_rejects_invalid_response_without_consuming_pending_a
|
||||
request_id: Some("REQ-8".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/auth").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
@@ -1614,6 +1621,7 @@ async fn continue_with_auth_unsupported_challenge_preserves_pending_auth_navigat
|
||||
request_id: Some("REQ-9".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/auth").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
@@ -1774,6 +1782,10 @@ fn extract_auth_challenge_preserves_quoted_commas_in_realm() {
|
||||
#[test]
|
||||
fn emit_auth_required_preserves_request_headers_and_post_data_shape() {
|
||||
let pending = PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: "INT-11".to_owned(),
|
||||
interception_session_id: Some("SID-1".to_owned()),
|
||||
@@ -1795,6 +1807,7 @@ fn emit_auth_required_preserves_request_headers_and_post_data_shape() {
|
||||
request_id: Some("REQ-11".to_owned()),
|
||||
loader_id: "LID-0000000001".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/auth").unwrap(),
|
||||
request_method: "POST".to_owned(),
|
||||
request_body: Some("payload".to_owned()),
|
||||
@@ -1867,6 +1880,7 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig
|
||||
request_id: Some("REQ-retired".to_owned()),
|
||||
loader_id: "LID-retired".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("https://retired.example/document").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
@@ -1913,6 +1927,10 @@ async fn untracked_fetch_actions_acknowledge_the_handler_and_abort_only_the_orig
|
||||
conn.register_pending_fetch_navigation_request_for_owner(
|
||||
&owner,
|
||||
PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id: "FETCH-retired".to_owned(),
|
||||
interception_session_id: Some("SID-handler".to_owned()),
|
||||
|
||||
@@ -366,6 +366,34 @@ async fn navigation_header_override_expires_on_redirect_before_auth_retry() {
|
||||
)
|
||||
.await;
|
||||
ctx.expect_result(70302, json!({}), Some("SID-1"));
|
||||
crate::testing::wait_until_scheduler_message(&mut ctx, "redirect request pause", |message| {
|
||||
message["method"] == "Fetch.requestPaused"
|
||||
&& message["sessionId"] == "SID-1"
|
||||
&& message["params"]["request"]["url"] == format!("http://{addr}/protected")
|
||||
})
|
||||
.await;
|
||||
let redirected = ctx.take_first_matching("redirect request pause", |message| {
|
||||
message["method"] == "Fetch.requestPaused"
|
||||
&& message["params"]["request"]["url"] == format!("http://{addr}/protected")
|
||||
});
|
||||
assert_eq!(
|
||||
redirected["params"]["request"]["url"],
|
||||
format!("http://{addr}/protected")
|
||||
);
|
||||
assert_eq!(
|
||||
redirected["params"]["redirectedRequestId"],
|
||||
paused["params"]["requestId"]
|
||||
);
|
||||
assert_ne!(
|
||||
redirected["params"]["requestId"],
|
||||
paused["params"]["requestId"]
|
||||
);
|
||||
ctx.process_async(
|
||||
json!({"id":70304,"sessionId":"SID-1","method":"Fetch.continueRequest",
|
||||
"params":{"requestId":redirected["params"]["requestId"]}}),
|
||||
)
|
||||
.await;
|
||||
ctx.expect_result(70304, json!({}), Some("SID-1"));
|
||||
wait_until_message(&mut ctx, "SID-1", "redirected navigation auth", |message| {
|
||||
message["method"] == "Fetch.authRequired"
|
||||
})
|
||||
|
||||
@@ -176,3 +176,6 @@ mod runtime_subresources;
|
||||
mod session_interception;
|
||||
#[path = "runtime_fetch/worker_requests.rs"]
|
||||
mod worker_requests;
|
||||
|
||||
#[path = "runtime_fetch/unified_navigation.rs"]
|
||||
mod unified_navigation;
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum NavigationEntry {
|
||||
Main,
|
||||
Frame,
|
||||
Popup,
|
||||
}
|
||||
|
||||
async fn navigation_redirect_contract(entry: NavigationEntry, fail: bool) {
|
||||
let hits = Arc::new(AtomicUsize::new(0));
|
||||
let observed_hits = hits.clone();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/page",
|
||||
get(|| async {
|
||||
(
|
||||
[(CONTENT_TYPE.as_str(), "text/html")],
|
||||
"<!doctype html><body>opener<iframe></iframe></body>",
|
||||
)
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/redirect",
|
||||
get(|| async { (StatusCode::FOUND, [("location", "/destination")]) }),
|
||||
)
|
||||
.route(
|
||||
"/destination",
|
||||
get(move || {
|
||||
let hits = observed_hits.clone();
|
||||
async move {
|
||||
hits.fetch_add(1, Ordering::SeqCst);
|
||||
(
|
||||
[(CONTENT_TYPE.as_str(), "text/html")],
|
||||
"<!doctype html><title>destination</title><body>shared navigation</body>",
|
||||
)
|
||||
}
|
||||
}),
|
||||
);
|
||||
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
|
||||
let mut ctx = TestContext::new();
|
||||
with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await;
|
||||
enable_runtime_async(&mut ctx, "SID-1", 41_000).await;
|
||||
let redirect = format!("http://{addr}/redirect");
|
||||
let destination = format!("http://{addr}/destination");
|
||||
let session = if matches!(entry, NavigationEntry::Popup) {
|
||||
ctx.enable_background_navigation_scheduler_for_test();
|
||||
ctx.process_async(json!({"id":41_001,"method":"Target.setAutoAttach",
|
||||
"params":{"autoAttach":true,"waitForDebuggerOnStart":true,"flatten":true}}))
|
||||
.await;
|
||||
ctx.expect_result(41_001, json!({}), None);
|
||||
let (_, session) =
|
||||
open_auto_attached_popup_from_session(&mut ctx, 41_002, "SID-1", &redirect).await;
|
||||
ctx.process_async(json!({"id":41_003,"method":"Network.enable","sessionId":session}))
|
||||
.await;
|
||||
ctx.expect_result(41_003, json!({}), Some(&session));
|
||||
ctx.process_async(json!({"id":41_004,"method":"Page.enable","sessionId":session}))
|
||||
.await;
|
||||
ctx.expect_result(41_004, json!({}), Some(&session));
|
||||
session
|
||||
} else {
|
||||
"SID-1".to_owned()
|
||||
};
|
||||
ctx.sent.clear();
|
||||
ctx.process_async(
|
||||
json!({"id":41_005,"method":"Fetch.enable","sessionId":session,
|
||||
"params":{"patterns":[{"urlPattern":"*/destination","resourceType":"Document"}]}}),
|
||||
)
|
||||
.await;
|
||||
ctx.expect_result(41_005, json!({}), Some(&session));
|
||||
match entry {
|
||||
NavigationEntry::Main => ctx.process_async(json!({"id":41_006,"method":"Page.navigate","sessionId":session,"params":{"url":redirect}})).await,
|
||||
NavigationEntry::Frame => ctx.process_async(json!({"id":41_006,"method":"Runtime.evaluate","sessionId":session,"params":{"expression":"document.querySelector('iframe').src = '/redirect'"}})).await,
|
||||
NavigationEntry::Popup => ctx.process_async(json!({"id":41_006,"method":"Runtime.runIfWaitingForDebugger","sessionId":session})).await,
|
||||
}
|
||||
crate::testing::wait_until_scheduler_message(
|
||||
&mut ctx,
|
||||
"destination redirect must pause for every navigation entry",
|
||||
|event| {
|
||||
event["method"] == "Fetch.requestPaused"
|
||||
&& event["sessionId"] == session
|
||||
&& event["params"]["request"]["url"] == destination
|
||||
&& event["params"]["resourceType"] == "Document"
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let paused = ctx
|
||||
.sent
|
||||
.iter()
|
||||
.find(|event| {
|
||||
event["method"] == "Fetch.requestPaused"
|
||||
&& event["sessionId"] == session
|
||||
&& event["params"]["request"]["url"] == destination
|
||||
})
|
||||
.unwrap()
|
||||
.clone();
|
||||
assert_eq!(
|
||||
hits.load(Ordering::SeqCst),
|
||||
0,
|
||||
"{entry:?} sent a redirect destination before interception"
|
||||
);
|
||||
assert!(
|
||||
paused["params"]["redirectedRequestId"].is_string(),
|
||||
"{entry:?} must retain the hidden first-hop Fetch ID: {paused}"
|
||||
);
|
||||
let network_id = paused["params"]["networkId"].as_str().unwrap().to_owned();
|
||||
assert!(
|
||||
ctx.sent
|
||||
.iter()
|
||||
.any(|event| event["method"] == "Network.requestWillBeSent"
|
||||
&& event["params"]["requestId"] == network_id
|
||||
&& event["params"]["request"]["url"] == destination
|
||||
&& event["params"]["redirectResponse"]["status"] == 302),
|
||||
"{entry:?}: redirect Network event must precede pause: {:?}",
|
||||
ctx.sent
|
||||
);
|
||||
let (method, params) = if fail {
|
||||
(
|
||||
"Fetch.failRequest",
|
||||
json!({"requestId":paused["params"]["requestId"],"errorReason":"BlockedByClient"}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"Fetch.continueRequest",
|
||||
json!({"requestId":paused["params"]["requestId"]}),
|
||||
)
|
||||
};
|
||||
ctx.process_async(json!({"id":41_007,"method":method,"sessionId":session,"params":params}))
|
||||
.await;
|
||||
ctx.expect_result(41_007, json!({}), Some(&session));
|
||||
wait_until_message(
|
||||
&mut ctx,
|
||||
session.as_str(),
|
||||
"one navigation Network terminal",
|
||||
|event| {
|
||||
matches!(
|
||||
event["method"].as_str(),
|
||||
Some("Network.loadingFinished" | "Network.loadingFailed")
|
||||
) && event["params"]["requestId"] == network_id
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let terminals = ctx
|
||||
.sent
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
matches!(
|
||||
event["method"].as_str(),
|
||||
Some("Network.loadingFinished" | "Network.loadingFailed")
|
||||
) && event["params"]["requestId"] == network_id
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1, "{entry:?}: exactly one terminal");
|
||||
assert_eq!(
|
||||
terminals[0]["method"],
|
||||
if fail {
|
||||
"Network.loadingFailed"
|
||||
} else {
|
||||
"Network.loadingFinished"
|
||||
}
|
||||
);
|
||||
assert_eq!(hits.load(Ordering::SeqCst), usize::from(!fail));
|
||||
let redirects = ctx
|
||||
.sent
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
event["method"] == "Network.requestWillBeSent"
|
||||
&& event["params"]["requestId"] == network_id
|
||||
&& event["params"]["request"]["url"] == destination
|
||||
})
|
||||
.count();
|
||||
assert_eq!(
|
||||
redirects, 1,
|
||||
"{entry:?}: redirect must not be announced again at completion"
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn main_navigation_uses_shared_redirect_interception() {
|
||||
for fail in [true, false] {
|
||||
navigation_redirect_contract(NavigationEntry::Main, fail).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn iframe_navigation_uses_shared_redirect_interception() {
|
||||
for fail in [true, false] {
|
||||
navigation_redirect_contract(NavigationEntry::Frame, fail).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn popup_navigation_uses_shared_redirect_interception() {
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
for fail in [true, false] {
|
||||
navigation_redirect_contract(NavigationEntry::Popup, fail).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn service_worker_navigation_contract(stage: Option<&str>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(
|
||||
listener,
|
||||
Router::new()
|
||||
.route(
|
||||
"/page",
|
||||
get(|| async {
|
||||
(
|
||||
[(CONTENT_TYPE.as_str(), "text/html")],
|
||||
"<!doctype html><body>register worker</body>",
|
||||
)
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/worker.js",
|
||||
get(|| async {
|
||||
(
|
||||
[(CONTENT_TYPE.as_str(), "text/javascript")],
|
||||
r#"
|
||||
self.addEventListener('install', event => event.waitUntil(self.skipWaiting()));
|
||||
self.addEventListener('activate', event => event.waitUntil(clients.claim()));
|
||||
self.addEventListener('fetch', event => {
|
||||
if (new URL(event.request.url).pathname === '/controlled') {
|
||||
event.respondWith(new Response('<!doctype html><body>service worker navigation</body>', {
|
||||
headers: {'Content-Type': 'text/html'}
|
||||
}));
|
||||
}
|
||||
});
|
||||
"#,
|
||||
)
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/controlled",
|
||||
get(|| async {
|
||||
(
|
||||
[(CONTENT_TYPE.as_str(), "text/html")],
|
||||
"<!doctype html><body>network navigation</body>",
|
||||
)
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let mut ctx = TestContext::new();
|
||||
with_loaded_http_document(&mut ctx, &format!("http://{addr}/page"), "SID-1", "TID-1").await;
|
||||
enable_runtime_async(&mut ctx, "SID-1", 42_000).await;
|
||||
ctx.process_async(json!({"id":42_001,"sessionId":"SID-1","method":"Runtime.evaluate",
|
||||
"params":{"expression":"(async () => { await navigator.serviceWorker.register('/worker.js'); await navigator.serviceWorker.ready; return true; })()", "awaitPromise":true,"returnByValue":true}})).await;
|
||||
crate::testing::wait_until_scheduler_message(
|
||||
&mut ctx,
|
||||
"service worker registration",
|
||||
|event| event["id"] == 42_001,
|
||||
)
|
||||
.await;
|
||||
let registered = take_response_by_id(&mut ctx, 42_001);
|
||||
assert_eq!(registered["result"]["result"]["value"], true);
|
||||
let pattern = match stage {
|
||||
Some(stage) => {
|
||||
json!({"urlPattern":"*/controlled","resourceType":"Document","requestStage":stage})
|
||||
}
|
||||
None => json!({"urlPattern":"*/never-matches","resourceType":"Document"}),
|
||||
};
|
||||
ctx.process_async(json!({"id":42_002,"sessionId":"SID-1","method":"Fetch.enable","params":{"patterns":[pattern]}})).await;
|
||||
ctx.expect_result(42_002, json!({}), Some("SID-1"));
|
||||
ctx.sent.clear();
|
||||
ctx.process_async(
|
||||
json!({"id":42_003,"sessionId":"SID-1","method":"Page.navigate",
|
||||
"params":{"url":format!("http://{addr}/controlled")}}),
|
||||
)
|
||||
.await;
|
||||
if let Some(stage) = stage {
|
||||
crate::testing::wait_until_scheduler_message(
|
||||
&mut ctx,
|
||||
"controlled navigation pause",
|
||||
|event| event["method"] == "Fetch.requestPaused" && event["sessionId"] == "SID-1",
|
||||
)
|
||||
.await;
|
||||
let pause = ctx.take_first_matching("controlled navigation pause", |event| {
|
||||
event["method"] == "Fetch.requestPaused"
|
||||
});
|
||||
if stage == "Response" {
|
||||
ctx.process_async(json!({"id":42_005,"sessionId":"SID-1","method":"Fetch.getResponseBody","params":{"requestId":pause["params"]["requestId"]}})).await;
|
||||
crate::testing::wait_until_scheduler_message(
|
||||
&mut ctx,
|
||||
"captured controlled response body",
|
||||
|event| event["id"] == 42_005,
|
||||
)
|
||||
.await;
|
||||
let body = take_response_by_id(&mut ctx, 42_005);
|
||||
assert!(body.get("result").is_some(), "{body:?}");
|
||||
}
|
||||
ctx.process_async(json!({"id":42_006,"sessionId":"SID-1","method":if stage == "Response" {"Fetch.continueResponse"} else {"Fetch.continueRequest"},"params":{"requestId":pause["params"]["requestId"]}})).await;
|
||||
ctx.expect_result(42_006, json!({}), Some("SID-1"));
|
||||
}
|
||||
crate::testing::wait_until_frame_stopped_loading(&mut ctx, "TID-1").await;
|
||||
ctx.process_async(
|
||||
json!({"id":42_004,"sessionId":"SID-1","method":"Runtime.evaluate",
|
||||
"params":{"expression":"[document.body.textContent, navigator.serviceWorker.controller !== null]","returnByValue":true}}),
|
||||
)
|
||||
.await;
|
||||
let result = take_response_by_id(&mut ctx, 42_004);
|
||||
assert_eq!(
|
||||
result["result"]["result"]["value"],
|
||||
json!(["service worker navigation", true])
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn unmatched_navigation_interception_preserves_service_worker_response() {
|
||||
service_worker_navigation_contract(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn request_interception_preserves_service_worker_navigation() {
|
||||
service_worker_navigation_contract(Some("Request")).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn response_interception_preserves_service_worker_client_after_body_capture() {
|
||||
service_worker_navigation_contract(Some("Response")).await;
|
||||
}
|
||||
@@ -510,7 +510,7 @@ impl TargetNetworkAgentState {
|
||||
.record_fetch_pause_announced_request_id(request_id);
|
||||
}
|
||||
|
||||
pub(crate) fn replace_child_document_fetch_request_id(
|
||||
pub(crate) fn replace_navigation_fetch_request_id(
|
||||
&mut self,
|
||||
network_id: String,
|
||||
session_id: Option<String>,
|
||||
@@ -518,7 +518,7 @@ impl TargetNetworkAgentState {
|
||||
) -> Option<String> {
|
||||
self.artifacts
|
||||
.subresource_network_artifacts
|
||||
.replace_child_document_fetch_request_id(network_id, session_id, fetch_id)
|
||||
.replace_navigation_fetch_request_id(network_id, session_id, fetch_id)
|
||||
}
|
||||
|
||||
pub(crate) fn request_was_announced_by_fetch_pause(&self, request_id: &str) -> bool {
|
||||
@@ -1935,7 +1935,7 @@ impl SubresourceNetworkArtifacts {
|
||||
.or_default();
|
||||
}
|
||||
|
||||
fn replace_child_document_fetch_request_id(
|
||||
fn replace_navigation_fetch_request_id(
|
||||
&mut self,
|
||||
network_id: String,
|
||||
session_id: Option<String>,
|
||||
|
||||
@@ -603,6 +603,7 @@ struct MainDocumentLiveNetworkProgressSource {
|
||||
timestamp: f64,
|
||||
initial_request_headers: Vec<(String, String)>,
|
||||
initial_request_cookie_report: Option<StoredCookieQueryReport>,
|
||||
announced_redirects: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
@@ -1003,6 +1004,7 @@ impl MainDocumentLiveNetworkProgressSource {
|
||||
loader_id: state.loader_id.clone(),
|
||||
frame_id: state.frame_id.clone(),
|
||||
timestamp: state.timestamp,
|
||||
announced_redirects: state.announced_redirects,
|
||||
initial_request_headers: state.request_headers.to_byte_strings(),
|
||||
initial_request_cookie_report: initial_request_cookie_report.cloned(),
|
||||
})
|
||||
@@ -1138,30 +1140,32 @@ impl MainDocumentLiveNetworkProgressSource {
|
||||
},
|
||||
);
|
||||
}
|
||||
events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
|
||||
target: target.clone(),
|
||||
url: redirect.to_url.clone(),
|
||||
method: final_request_method.to_owned(),
|
||||
request_body: request_body.map(str::to_owned),
|
||||
request_headers: final_request_headers.to_vec(),
|
||||
request_initiator_type: SubresourceRequestInitiatorType::Other,
|
||||
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
|
||||
url: redirect.from_url.clone(),
|
||||
status: redirect.status,
|
||||
status_text: critical_client_hint_internal_redirect_status_text(
|
||||
&redirect.from_url,
|
||||
&redirect.to_url,
|
||||
redirect.status,
|
||||
redirect.network_extra_info_available,
|
||||
observed_response_available,
|
||||
),
|
||||
headers: redirect.headers.clone(),
|
||||
from_cache: redirect.from_cache,
|
||||
negotiated_http_version: redirect.negotiated_http_version,
|
||||
})),
|
||||
redirect_has_extra_info: redirect.network_extra_info_available,
|
||||
cookie_access_report: redirect.request_cookie_report.clone(),
|
||||
});
|
||||
if index >= self.announced_redirects {
|
||||
events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
|
||||
target: target.clone(),
|
||||
url: redirect.to_url.clone(),
|
||||
method: final_request_method.to_owned(),
|
||||
request_body: request_body.map(str::to_owned),
|
||||
request_headers: final_request_headers.to_vec(),
|
||||
request_initiator_type: SubresourceRequestInitiatorType::Other,
|
||||
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
|
||||
url: redirect.from_url.clone(),
|
||||
status: redirect.status,
|
||||
status_text: critical_client_hint_internal_redirect_status_text(
|
||||
&redirect.from_url,
|
||||
&redirect.to_url,
|
||||
redirect.status,
|
||||
redirect.network_extra_info_available,
|
||||
observed_response_available,
|
||||
),
|
||||
headers: redirect.headers.clone(),
|
||||
from_cache: redirect.from_cache,
|
||||
negotiated_http_version: redirect.negotiated_http_version,
|
||||
})),
|
||||
redirect_has_extra_info: redirect.network_extra_info_available,
|
||||
cookie_access_report: redirect.request_cookie_report.clone(),
|
||||
});
|
||||
}
|
||||
if observed_response_available && redirect.network_extra_info_available {
|
||||
let (status, headers) = observed_response_metadata(
|
||||
network_observation_journal,
|
||||
@@ -1667,6 +1671,7 @@ struct CompletedMainDocumentProgressContext {
|
||||
session_ids: Vec<Option<String>>,
|
||||
request_id: Option<String>,
|
||||
request_announced: bool,
|
||||
announced_redirects: usize,
|
||||
requested_url: Url,
|
||||
request_method: String,
|
||||
request_body: Option<String>,
|
||||
@@ -1682,7 +1687,7 @@ impl CompletedMainDocumentProgressContext {
|
||||
state: &NavigationDispatchState,
|
||||
request_id: Option<String>,
|
||||
) -> Self {
|
||||
Self::new(
|
||||
let mut context = Self::new(
|
||||
main_document_network_event_session_ids(conn, state.session_id.as_deref()),
|
||||
request_id,
|
||||
state.request_announced,
|
||||
@@ -1693,7 +1698,9 @@ impl CompletedMainDocumentProgressContext {
|
||||
state.loader_id.clone(),
|
||||
state.frame_id.clone(),
|
||||
state.timestamp,
|
||||
)
|
||||
);
|
||||
context.announced_redirects = state.announced_redirects;
|
||||
context
|
||||
}
|
||||
|
||||
fn new(
|
||||
@@ -1712,6 +1719,7 @@ impl CompletedMainDocumentProgressContext {
|
||||
session_ids,
|
||||
request_id,
|
||||
request_announced,
|
||||
announced_redirects: 0,
|
||||
requested_url,
|
||||
request_method,
|
||||
request_body,
|
||||
@@ -1808,30 +1816,32 @@ impl CompletedMainDocumentProgressContext {
|
||||
},
|
||||
);
|
||||
}
|
||||
progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
|
||||
target: target.clone(),
|
||||
url: redirect.to_url.clone(),
|
||||
method: events.request_method.clone(),
|
||||
request_body: self.request_body.clone(),
|
||||
request_headers: events.request_headers.clone(),
|
||||
request_initiator_type: SubresourceRequestInitiatorType::Other,
|
||||
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
|
||||
url: redirect.from_url.clone(),
|
||||
status: redirect.status,
|
||||
status_text: critical_client_hint_internal_redirect_status_text(
|
||||
&redirect.from_url,
|
||||
&redirect.to_url,
|
||||
redirect.status,
|
||||
redirect.network_extra_info_available,
|
||||
observed_response_available,
|
||||
),
|
||||
headers: redirect.headers.clone(),
|
||||
from_cache: redirect.from_cache,
|
||||
negotiated_http_version: redirect.negotiated_http_version,
|
||||
})),
|
||||
redirect_has_extra_info: redirect.network_extra_info_available,
|
||||
cookie_access_report: redirect.request_cookie_report.clone(),
|
||||
});
|
||||
if index >= self.announced_redirects {
|
||||
progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
|
||||
target: target.clone(),
|
||||
url: redirect.to_url.clone(),
|
||||
method: events.request_method.clone(),
|
||||
request_body: self.request_body.clone(),
|
||||
request_headers: events.request_headers.clone(),
|
||||
request_initiator_type: SubresourceRequestInitiatorType::Other,
|
||||
redirect_response: Box::new(Some(MainDocumentRedirectResponse {
|
||||
url: redirect.from_url.clone(),
|
||||
status: redirect.status,
|
||||
status_text: critical_client_hint_internal_redirect_status_text(
|
||||
&redirect.from_url,
|
||||
&redirect.to_url,
|
||||
redirect.status,
|
||||
redirect.network_extra_info_available,
|
||||
observed_response_available,
|
||||
),
|
||||
headers: redirect.headers.clone(),
|
||||
from_cache: redirect.from_cache,
|
||||
negotiated_http_version: redirect.negotiated_http_version,
|
||||
})),
|
||||
redirect_has_extra_info: redirect.network_extra_info_available,
|
||||
cookie_access_report: redirect.request_cookie_report.clone(),
|
||||
});
|
||||
}
|
||||
if observed_response_available && redirect.network_extra_info_available {
|
||||
let (status, headers) = observed_response_metadata(
|
||||
&events.network_observation_journal,
|
||||
|
||||
@@ -120,6 +120,7 @@ fn live_progress_source() -> (
|
||||
let (sender, receiver) = unbounded_channel();
|
||||
(
|
||||
MainDocumentLiveNetworkProgressSource {
|
||||
announced_redirects: 0,
|
||||
sender: Some(sender),
|
||||
progress_queue: super::gate::MainDocumentProgressQueueHandle::from_source(
|
||||
MainDocumentProgressSource::streaming(),
|
||||
@@ -1604,6 +1605,7 @@ fn no_content_navigation_retains_response_metadata_with_live_progress() {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: "LOADER-1".to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: Url::parse("http://example.test/start").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -666,6 +666,7 @@ async fn clear_browser_cache_keeps_pending_response_navigation_transfer() {
|
||||
request_id: Some("REQ-1".to_owned()),
|
||||
loader_id: LOADER_ID.to_owned(),
|
||||
request_announced: true,
|
||||
announced_redirects: 0,
|
||||
requested_url: url.clone(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -524,6 +524,7 @@ async fn get_request_post_data_returns_main_document_navigation_post_body() {
|
||||
request_id: Some("REQ-main-post".to_owned()),
|
||||
loader_id: LOADER_ID.to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url,
|
||||
request_method: "POST".to_owned(),
|
||||
request_body: Some("username=alice&pw=s3cret".to_owned()),
|
||||
@@ -620,6 +621,7 @@ async fn get_request_post_data_uses_text_projection_while_bidi_collector_keeps_t
|
||||
request_id: Some("REQ-multipart".to_owned()),
|
||||
loader_id: LOADER_ID.to_owned(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url,
|
||||
request_method: "POST".to_owned(),
|
||||
request_body: Some(multipart_text.to_owned()),
|
||||
|
||||
@@ -2654,6 +2654,7 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
|
||||
request_id: None,
|
||||
loader_id: document_loader_id.clone(),
|
||||
request_announced: false,
|
||||
announced_redirects: 0,
|
||||
requested_url: requested_url.clone(),
|
||||
request_method: request_method.to_owned(),
|
||||
request_body: request_body
|
||||
@@ -2699,6 +2700,10 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
|
||||
));
|
||||
};
|
||||
pending_fetch_navigation = Some(PendingFetchNavigation {
|
||||
reserved_service_worker_client: None,
|
||||
interception: None,
|
||||
remaining_request_stages: Vec::new(),
|
||||
redirected_request_id: None,
|
||||
auth_retry_budget: Default::default(),
|
||||
fetch_request_id,
|
||||
interception_session_id: preflight
|
||||
@@ -2791,7 +2796,8 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
|
||||
}
|
||||
|
||||
if let Some(mut pending) = pending_fetch_navigation {
|
||||
if fetch_request_stage == FetchRequestStage::Request {
|
||||
let request_pause = fetch::prepare_navigation_request_pause(conn, &mut pending);
|
||||
if fetch_request_stage == FetchRequestStage::Request && request_pause {
|
||||
pending.request_cookie_report = navigation_cookie_access_report_for_owner(
|
||||
conn,
|
||||
&pending.navigation.owner,
|
||||
@@ -2810,14 +2816,13 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
|
||||
);
|
||||
}
|
||||
pending.navigation.request_announced = pending.navigation.request_id.is_some();
|
||||
let _ = conn.register_pending_fetch_navigation_request_for_owner(
|
||||
&pending.navigation.owner,
|
||||
pending.clone(),
|
||||
);
|
||||
out.push(fetch::request_paused_background_event(
|
||||
pending.interception_session_id.as_deref(),
|
||||
&pending,
|
||||
));
|
||||
let pending_owner = pending.navigation.owner.clone();
|
||||
let _ =
|
||||
conn.register_pending_fetch_navigation_request_for_owner(&pending_owner, pending);
|
||||
let mut output = CommandOutputBuffer::default();
|
||||
output.extend_background_events_after_messages(out);
|
||||
return NavigateCommandStart::CompleteImmediate(output.into_plan());
|
||||
@@ -2840,6 +2845,14 @@ fn start_navigate_to_url_command_with_background_policy_and_request(
|
||||
);
|
||||
}
|
||||
pending.navigation.request_announced = pending.navigation.request_id.is_some();
|
||||
if let Some(sender) = conn.background_event_sender() {
|
||||
// Retire the previous source and announce the request before
|
||||
// waiting for headers. A held transport must not retain either
|
||||
// the old Worker targets or this navigation's initial events.
|
||||
for event in out.drain(..) {
|
||||
let _ = sender.send(event);
|
||||
}
|
||||
}
|
||||
return NavigateCommandStart::PendingContinueWithoutRequestPause(Box::new(
|
||||
PendingContinueNavigationWithoutRequestPauseCommand {
|
||||
prefix_events: out,
|
||||
|
||||
@@ -681,6 +681,7 @@ async fn page_navigate_network_failure_commits_error_document() {
|
||||
request_id: Some(stale_request_id.to_owned()),
|
||||
loader_id: old_document_token.loader_id.clone(),
|
||||
request_announced: true,
|
||||
announced_redirects: 0,
|
||||
requested_url: url::Url::parse("https://stale.example.test/old-body").unwrap(),
|
||||
request_method: "GET".to_owned(),
|
||||
request_body: None,
|
||||
|
||||
@@ -207,19 +207,24 @@ impl JsContextHost {
|
||||
let fetch = ChildDocumentFetch {
|
||||
target,
|
||||
network_attribution,
|
||||
navigation_url: target_url,
|
||||
network_info: None,
|
||||
intercept_redirects: false,
|
||||
redirect: None,
|
||||
request: child_document_load_request(bootstrap).map(|request| {
|
||||
configure_child_document_navigation_request(
|
||||
request,
|
||||
&initiator_url,
|
||||
&browser_context,
|
||||
)
|
||||
.with_page_network_policy()
|
||||
.with_network_partition_key(network_partition_key)
|
||||
}),
|
||||
request: child_document_load_request(bootstrap)
|
||||
.map(|request| {
|
||||
configure_child_document_navigation_request(
|
||||
request,
|
||||
&initiator_url,
|
||||
&browser_context,
|
||||
)
|
||||
.with_page_network_policy()
|
||||
.with_network_partition_key(network_partition_key)
|
||||
})
|
||||
.map(|request| {
|
||||
crate::network::navigation::NavigationInterception::new(
|
||||
resource_loader.clone(),
|
||||
request,
|
||||
)
|
||||
}),
|
||||
service_worker_client_id,
|
||||
browser_context_runtime,
|
||||
resource_loader,
|
||||
@@ -801,9 +806,11 @@ impl JsContextHost {
|
||||
initiating_loader: &crate::network::context::DocumentResourceLoader,
|
||||
mut fetch: ChildDocumentFetch,
|
||||
) -> Option<u64> {
|
||||
fetch.intercept_redirects = false;
|
||||
if let Some(request) = fetch.request.as_mut() {
|
||||
request.set_interception(false);
|
||||
}
|
||||
let Some(request) = fetch.request.as_ref().filter(|request| {
|
||||
matches!(request.url.scheme(), "http" | "https")
|
||||
matches!(request.request().url.scheme(), "http" | "https")
|
||||
&& self.should_intercept_subresource(SubresourceResourceType::Document)
|
||||
}) else {
|
||||
fetch.start();
|
||||
@@ -827,16 +834,17 @@ impl JsContextHost {
|
||||
internal_id: 0,
|
||||
network_request_handle: None,
|
||||
frame_id: Some(fetch.network_attribution.frame_id().to_owned()),
|
||||
document_url: request.url.clone(),
|
||||
url: request.url.clone(),
|
||||
document_url: request.request().url.clone(),
|
||||
url: request.request().url.clone(),
|
||||
websocket_socket_id: None,
|
||||
method: request.method.clone(),
|
||||
request_headers: request.request_headers.clone(),
|
||||
method: request.request().method.clone(),
|
||||
request_headers: request.request().request_headers.clone(),
|
||||
request_body: request
|
||||
.request()
|
||||
.body
|
||||
.as_ref()
|
||||
.map(|body| String::from_utf8_lossy(body).into_owned()),
|
||||
request_body_bytes: request.body.clone(),
|
||||
request_body_bytes: request.request().body.clone(),
|
||||
resource_type: SubresourceResourceType::Document,
|
||||
request_cookie_report: None,
|
||||
};
|
||||
@@ -862,12 +870,13 @@ impl JsContextHost {
|
||||
headers: moli_fetch::RequestHeaders,
|
||||
body: Option<bytes::Bytes>,
|
||||
) {
|
||||
if let Some(mut request) = fetch.request.take() {
|
||||
if let Some(interception) = fetch.request.as_mut() {
|
||||
let request = interception.request_mut();
|
||||
request.url = url;
|
||||
request.method = method;
|
||||
request.request_headers = headers;
|
||||
request.body = body;
|
||||
request = request.with_redirect_headers(redirect_headers);
|
||||
*request = request.clone().with_redirect_headers(redirect_headers);
|
||||
if let Some(info) = fetch.network_info.as_mut() {
|
||||
info.url = request.url.clone();
|
||||
info.method = request.method.clone();
|
||||
@@ -878,7 +887,6 @@ impl JsContextHost {
|
||||
.as_ref()
|
||||
.map(|body| String::from_utf8_lossy(body).into_owned());
|
||||
}
|
||||
fetch.request = Some(request);
|
||||
}
|
||||
// A superseded navigation has already settled without this request.
|
||||
if self
|
||||
@@ -942,10 +950,8 @@ const UNSUPPORTED_CHILD_DOCUMENT_BOOTSTRAP: &str =
|
||||
pub(crate) struct ChildDocumentFetch {
|
||||
target: ChildDocumentNavigationFetchTarget,
|
||||
network_attribution: ChildDocumentLoadNetworkAttribution,
|
||||
request: Option<moli_fetch::Request>,
|
||||
navigation_url: url::Url,
|
||||
request: Option<crate::network::navigation::NavigationInterception>,
|
||||
network_info: Option<PendingSubresourceFetchInfo>,
|
||||
intercept_redirects: bool,
|
||||
redirect: Option<crate::protocol_types::NavigationRedirect>,
|
||||
service_worker_client_id: Option<crate::service_worker_runtime::ServiceWorkerClientId>,
|
||||
browser_context_runtime: crate::runtime::RendererBrowserContextRuntime,
|
||||
@@ -959,7 +965,7 @@ impl std::fmt::Debug for ChildDocumentFetch {
|
||||
formatter
|
||||
.debug_struct("ChildDocumentFetch")
|
||||
.field("target", &self.target)
|
||||
.field("navigation_url", &self.navigation_url)
|
||||
.field("request", &self.request)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
@@ -967,7 +973,9 @@ impl std::fmt::Debug for ChildDocumentFetch {
|
||||
impl ChildDocumentFetch {
|
||||
pub(crate) fn set_network_info(&mut self, info: PendingSubresourceFetchInfo) {
|
||||
self.network_info = Some(info);
|
||||
self.intercept_redirects = true;
|
||||
if let Some(request) = self.request.as_mut() {
|
||||
request.set_interception(true);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn take_request_metadata(
|
||||
@@ -997,7 +1005,7 @@ impl ChildDocumentFetch {
|
||||
.browser_context_runtime
|
||||
.fetch_service_worker_child_main_resource_for_reserved_client(
|
||||
client_id,
|
||||
&request,
|
||||
request.request(),
|
||||
self.resource_loader.request_client(),
|
||||
self.resource_loader.task_runner(),
|
||||
self.completion_tx.clone(),
|
||||
@@ -1011,14 +1019,8 @@ impl ChildDocumentFetch {
|
||||
let (head, body) = response.into_body();
|
||||
return self.accept_response(request, head, body);
|
||||
}
|
||||
let transport_request = if self.intercept_redirects {
|
||||
request.clone().with_single_hop()
|
||||
} else {
|
||||
request.clone()
|
||||
};
|
||||
let response = self
|
||||
.resource_loader
|
||||
.fetch_raw(transport_request)
|
||||
let response = request
|
||||
.fetch_raw()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let (head, body) = response.into_body();
|
||||
@@ -1027,58 +1029,23 @@ impl ChildDocumentFetch {
|
||||
|
||||
fn accept_response(
|
||||
&mut self,
|
||||
mut request: moli_fetch::Request,
|
||||
mut request: crate::network::navigation::NavigationInterception,
|
||||
head: moli_fetch::ResponseHead,
|
||||
body: moli_fetch::ResponseBody,
|
||||
) -> Result<Option<ChildDocumentLoadOutcome>, String> {
|
||||
if self.intercept_redirects
|
||||
&& let Some(next_url) = crate::network_host::next_redirect_url(
|
||||
&head.final_url,
|
||||
head.status,
|
||||
&head.headers,
|
||||
request.redirect_count(),
|
||||
)?
|
||||
if let Some(redirect) = request
|
||||
.follow_redirect(&head)
|
||||
.map_err(|error| error.to_string())?
|
||||
{
|
||||
if !matches!(next_url.scheme(), "http" | "https") {
|
||||
return Err(format!(
|
||||
"child document redirect to unsupported URL {next_url}"
|
||||
));
|
||||
}
|
||||
let redirect = moli_fetch::RedirectInfo {
|
||||
source: moli_fetch::RedirectSource::Network,
|
||||
from_url: head.final_url.clone(),
|
||||
to_url: next_url.clone(),
|
||||
status: head.status,
|
||||
headers: head.headers.clone(),
|
||||
network_extra_info_available: false,
|
||||
request_extra_info: None,
|
||||
response_extra_info: None,
|
||||
redirect_has_extra_info: false,
|
||||
request_cookie_report: head.request_cookie_report,
|
||||
cookie_set_reports: head.cookie_set_reports,
|
||||
from_cache: head.from_cache,
|
||||
negotiated_http_version: head.negotiated_http_version,
|
||||
};
|
||||
request.update_referrer_for_redirect(&head.final_url, &head.headers);
|
||||
request
|
||||
.follow_redirect(redirect.clone())
|
||||
.map_err(|error| error.to_string())?;
|
||||
self.resource_loader
|
||||
.prepare_redirect()
|
||||
.map_err(|error| error.to_string())?;
|
||||
self.navigation_url = next_url;
|
||||
self.redirect = Some(redirect.into());
|
||||
self.request = Some(request);
|
||||
return Ok(None);
|
||||
}
|
||||
let navigation_url = if !self.intercept_redirects && head.redirected {
|
||||
head.final_url.clone()
|
||||
} else {
|
||||
self.navigation_url.clone()
|
||||
};
|
||||
let navigation_url = request.navigation_url(&head);
|
||||
let request = request.request();
|
||||
child_document_load_outcome_from_response(
|
||||
request.url.as_str().to_owned(),
|
||||
request.method,
|
||||
request.method.clone(),
|
||||
request.request_headers.to_byte_strings(),
|
||||
head,
|
||||
body,
|
||||
@@ -1300,11 +1267,12 @@ mod tests {
|
||||
None,
|
||||
"retired-loader".to_owned(),
|
||||
),
|
||||
navigation_url: url.clone(),
|
||||
network_info: None,
|
||||
intercept_redirects: false,
|
||||
redirect: None,
|
||||
request: Some(moli_fetch::Request::get_with_url(url)),
|
||||
request: Some(crate::network::navigation::NavigationInterception::new(
|
||||
resource_loader.clone(),
|
||||
moli_fetch::Request::get_with_url(url),
|
||||
)),
|
||||
service_worker_client_id: None,
|
||||
browser_context_runtime: host.browser_context_runtime(),
|
||||
resource_loader: resource_loader.clone(),
|
||||
|
||||
@@ -2431,20 +2431,16 @@ impl JsContextHost {
|
||||
task_resource_loader.note_external_response_ready()?;
|
||||
response
|
||||
}
|
||||
None => task_resource_loader
|
||||
.fetch(
|
||||
None => {
|
||||
crate::network::navigation::NavigationInterception::new(
|
||||
task_resource_loader.clone(),
|
||||
request
|
||||
.with_request_origin(request_origin)
|
||||
.with_page_network_policy()
|
||||
.with_top_level_navigation_cookie_context(),
|
||||
)
|
||||
.await
|
||||
.map(|response| {
|
||||
moli_fetch::RawResponse::from_head_and_body(
|
||||
response.head(),
|
||||
response.body_bytes().to_vec(),
|
||||
)
|
||||
})?,
|
||||
.fetch_raw()
|
||||
.await?
|
||||
}
|
||||
};
|
||||
let head = response.head();
|
||||
if popup_document_response_should_ignore_navigation(response.status, &head.headers)
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
use anyhow::{Result, bail};
|
||||
use moli_fetch::{
|
||||
NetworkFetchResult, RawResponse, RedirectInfo, Request, Response, ResponseHead,
|
||||
StreamingRawResponse,
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
use super::NavigationResourceLoader;
|
||||
|
||||
/// Request execution shared by top-level, child and auxiliary navigations.
|
||||
/// The navigation owner decides when a hop may start and where its result can
|
||||
/// commit. This object owns transport routing, request overrides and redirects;
|
||||
/// it deliberately has no DevTools session or frame-lifecycle state.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NavigationInterception {
|
||||
loader: NavigationResourceLoader,
|
||||
request: Request,
|
||||
navigation_url: Url,
|
||||
intercept_redirects: bool,
|
||||
}
|
||||
|
||||
struct CancelOnDrop(Option<NavigationResourceLoader>);
|
||||
|
||||
impl Drop for CancelOnDrop {
|
||||
fn drop(&mut self) {
|
||||
if let Some(loader) = self.0.take() {
|
||||
loader.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NavigationInterception {
|
||||
pub fn new(loader: NavigationResourceLoader, request: Request) -> Self {
|
||||
Self {
|
||||
navigation_url: request.url.clone(),
|
||||
loader,
|
||||
request,
|
||||
intercept_redirects: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resource_loader(&self) -> &NavigationResourceLoader {
|
||||
&self.loader
|
||||
}
|
||||
|
||||
pub fn request(&self) -> &Request {
|
||||
&self.request
|
||||
}
|
||||
|
||||
pub fn request_mut(&mut self) -> &mut Request {
|
||||
&mut self.request
|
||||
}
|
||||
|
||||
pub fn set_interception(&mut self, enabled: bool) {
|
||||
self.intercept_redirects = enabled;
|
||||
}
|
||||
|
||||
pub fn navigation_url(&self, response: &ResponseHead) -> Url {
|
||||
if !self.intercept_redirects && response.redirected {
|
||||
response.final_url.clone()
|
||||
} else {
|
||||
self.navigation_url.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn transport_request(&self) -> Result<Request> {
|
||||
moli_url_policy::route_navigation_url(
|
||||
&self.request.url,
|
||||
moli_url_policy::LocalFileNavigationAccess::Denied,
|
||||
)?;
|
||||
Ok(if self.intercept_redirects {
|
||||
self.request.clone().with_single_hop()
|
||||
} else {
|
||||
self.request.clone()
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute<T>(
|
||||
&self,
|
||||
operation: impl std::future::Future<Output = Result<T>>,
|
||||
) -> Result<T> {
|
||||
let mut guard = CancelOnDrop(Some(self.loader.clone()));
|
||||
let result = operation.await;
|
||||
guard.0.take();
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn fetch_text(&self) -> Result<NetworkFetchResult<Response>> {
|
||||
self.execute(
|
||||
self.loader
|
||||
.fetch_with_network_metadata(self.transport_request()?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn fetch_raw(&self) -> Result<RawResponse> {
|
||||
self.execute(self.loader.fetch_raw(self.transport_request()?))
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn fetch_raw_stream(&self) -> Result<NetworkFetchResult<StreamingRawResponse>> {
|
||||
self.execute(
|
||||
self.loader
|
||||
.fetch_raw_stream_with_network_metadata(self.transport_request()?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn fetch_buffered(&self) -> Result<NetworkFetchResult<RawResponse>> {
|
||||
self.execute(
|
||||
self.loader
|
||||
.fetch_raw_with_network_metadata(self.transport_request()?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Returns the next hop to its owner before any transport is started.
|
||||
/// Synthetic and network responses use exactly the same redirect rules.
|
||||
pub fn follow_redirect(&mut self, head: &ResponseHead) -> Result<Option<RedirectInfo>> {
|
||||
if !self.intercept_redirects {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(next_url) = crate::network_host::next_redirect_url(
|
||||
&head.final_url,
|
||||
head.status,
|
||||
&head.headers,
|
||||
self.request.redirect_count(),
|
||||
)
|
||||
.map_err(anyhow::Error::msg)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !matches!(next_url.scheme(), "http" | "https") {
|
||||
bail!("document redirect to unsupported URL {next_url}");
|
||||
}
|
||||
let redirect = RedirectInfo {
|
||||
source: moli_fetch::RedirectSource::Network,
|
||||
from_url: head.final_url.clone(),
|
||||
to_url: next_url.clone(),
|
||||
status: head.status,
|
||||
headers: head.headers.clone(),
|
||||
network_extra_info_available: false,
|
||||
request_extra_info: None,
|
||||
response_extra_info: None,
|
||||
redirect_has_extra_info: false,
|
||||
request_cookie_report: head.request_cookie_report.clone(),
|
||||
cookie_set_reports: head.cookie_set_reports.clone(),
|
||||
from_cache: head.from_cache,
|
||||
negotiated_http_version: head.negotiated_http_version,
|
||||
};
|
||||
self.request
|
||||
.update_referrer_for_redirect(&head.final_url, &head.headers);
|
||||
self.request.follow_redirect(redirect.clone())?;
|
||||
self.loader.prepare_redirect()?;
|
||||
self.navigation_url = next_url;
|
||||
Ok(Some(redirect))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,104 @@
|
||||
use super::*;
|
||||
use crate::network::{RendererResourceTaskRunner, ResourceRequestClient};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
async fn request_head(socket: &mut tokio::net::TcpStream) -> String {
|
||||
let mut head = Vec::new();
|
||||
let mut bytes = [0u8; 4096];
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let length = socket.read(&mut bytes).await.unwrap();
|
||||
assert!(length > 0, "connection closed before request headers");
|
||||
head.extend_from_slice(&bytes[..length]);
|
||||
}
|
||||
String::from_utf8(head).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redirect_body_cancellation_does_not_cancel_the_next_navigation_hop() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut first, _) = listener.accept().await.unwrap();
|
||||
assert!(request_head(&mut first).await.starts_with("GET /redirect "));
|
||||
first
|
||||
.write_all(b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 1000000\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
let (mut second, _) = listener.accept().await.unwrap();
|
||||
assert!(request_head(&mut second).await.starts_with("GET /final "));
|
||||
second
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap();
|
||||
let request = Request::get(&format!("http://{addr}/redirect")).unwrap();
|
||||
let origin = moli_url::WebOrigin::from_url(&request.url);
|
||||
let request = request.with_request_origin(origin);
|
||||
let loader = NavigationResourceLoader::new(
|
||||
owner.handle(),
|
||||
request.url.clone(),
|
||||
RendererResourceTaskRunner::from_current_tokio().unwrap(),
|
||||
);
|
||||
let mut navigation = NavigationInterception::new(loader.clone(), request);
|
||||
navigation.set_interception(true);
|
||||
let response = navigation.fetch_raw_stream().await.unwrap();
|
||||
let redirect = navigation
|
||||
.follow_redirect(&response.response().head())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(redirect.to_url.path(), "/final");
|
||||
drop(response);
|
||||
let response = tokio::time::timeout(std::time::Duration::from_secs(10), navigation.fetch_raw())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (_, body) = response.into_body();
|
||||
assert_eq!(body.try_into_materialized_bytes().unwrap(), b"ok");
|
||||
assert_eq!(
|
||||
loader.state(),
|
||||
super::super::NavigationResourceLoaderState::ResponseReady
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_a_request_future_cancels_transport_while_its_navigation_owner_survives() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (received, request_received) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut bytes = [0u8; 4096];
|
||||
assert!(socket.read(&mut bytes).await.unwrap() > 0);
|
||||
received.send(()).unwrap();
|
||||
let read =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), socket.read(&mut bytes))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(read, Ok(0)) || read.is_err(),
|
||||
"cancelled navigation must close the transport"
|
||||
);
|
||||
});
|
||||
let owner = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap();
|
||||
let request = Request::get(&format!("http://{addr}/pending")).unwrap();
|
||||
let origin = moli_url::WebOrigin::from_url(&request.url);
|
||||
let request = request.with_request_origin(origin);
|
||||
let loader = NavigationResourceLoader::new(
|
||||
owner.handle(),
|
||||
request.url.clone(),
|
||||
RendererResourceTaskRunner::from_current_tokio().unwrap(),
|
||||
);
|
||||
let navigation = NavigationInterception::new(loader.clone(), request);
|
||||
let task = tokio::spawn(async move { navigation.fetch_raw_stream().await });
|
||||
request_received.await.unwrap();
|
||||
task.abort();
|
||||
assert!(task.await.is_err_and(|error| error.is_cancelled()));
|
||||
assert_eq!(
|
||||
loader.state(),
|
||||
super::super::NavigationResourceLoaderState::Cancelled
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
@@ -277,6 +277,30 @@ impl NavigationResourceLoader {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn fetch_raw_with_network_metadata(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<NetworkFetchResult<RawResponse>> {
|
||||
self.begin_fetch()?;
|
||||
let result = self
|
||||
.request_client
|
||||
.fetch_raw_with_cancel_and_network_metadata(
|
||||
request.with_page_network_policy(),
|
||||
self.inner.cancel.clone(),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(response) => {
|
||||
self.finish_response_ready()?;
|
||||
Ok(response)
|
||||
}
|
||||
Err(error) => {
|
||||
self.finish_failed();
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch_raw_stream(&self, request: Request) -> Result<StreamingRawResponse> {
|
||||
self.fetch_raw_stream_with_network_metadata(request)
|
||||
.await
|
||||
@@ -288,16 +312,30 @@ impl NavigationResourceLoader {
|
||||
request: Request,
|
||||
) -> Result<NetworkFetchResult<StreamingRawResponse>> {
|
||||
self.begin_fetch()?;
|
||||
let cancellation = if request.transport_follows_redirects() {
|
||||
self.inner.cancel.clone()
|
||||
} else {
|
||||
self.inner.cancel.child_for_subrequest()
|
||||
};
|
||||
match self
|
||||
.request_client
|
||||
.fetch_raw_stream_with_cancel_and_network_metadata(
|
||||
request.with_page_network_policy(),
|
||||
self.inner.cancel.clone(),
|
||||
cancellation.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
self.finish_response_ready()?;
|
||||
if !matches!(response.response().status, 301 | 302 | 303 | 307 | 308)
|
||||
|| !response
|
||||
.response()
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("location"))
|
||||
{
|
||||
self.inner.cancel.adopt_response_progress(&cancellation);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
Err(error) => {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
mod context;
|
||||
mod interception;
|
||||
mod loader;
|
||||
|
||||
pub use context::DocumentFetchContextSeed;
|
||||
pub use interception::NavigationInterception;
|
||||
pub use loader::{
|
||||
NavigationResourceLoader, NavigationResourceLoaderDiagnostics, NavigationResourceLoaderState,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user