From 0c765c19f8bdb666e22b16b3c0ffc323dc18daa9 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Mon, 26 Oct 2026 04:53:53 +0800 Subject: [PATCH] fix(fetch): avoid deadlocks when continuing intercepted documents Reuse streaming response background navigation after request and authentication interception so parser-blocking scripts can progress without wedging other CDP clients. Cover document-only and all-request interception with authentication handling on and off. Fixes #2006. --- .../parser_replacement_and_debugger.rs | 179 ++++++++++++++++++ moli-protocol/src/domains/fetch/commands.rs | 2 +- moli-protocol/src/domains/fetch/navigation.rs | 34 +++- 3 files changed, 210 insertions(+), 5 deletions(-) diff --git a/moli-protocol-server/src/protocol_server/tests/websocket/parser_replacement_and_debugger.rs b/moli-protocol-server/src/protocol_server/tests/websocket/parser_replacement_and_debugger.rs index 866529affae..3af6c5d30bb 100644 --- a/moli-protocol-server/src/protocol_server/tests/websocket/parser_replacement_and_debugger.rs +++ b/moli-protocol-server/src/protocol_server/tests/websocket/parser_replacement_and_debugger.rs @@ -1235,6 +1235,185 @@ async fn websocket_cdp_runtime_call_function_awaitpromise_fetch_interception_unb protocol_server.abort(); } +#[tokio::test] +async fn websocket_cdp_fetch_continue_document_with_blocking_script_keeps_server_responsive() { + async fn page() -> impl IntoResponse { + ( + [(axum::http::header::CONTENT_TYPE.as_str(), "text/html")], + "\ +

loaded

", + ) + } + + for (pattern, handle_auth_requests) in + [("*", false), ("*page", false), ("*", true), ("*page", true)] + { + let script_requested = Arc::new(tokio::sync::Notify::new()); + let release_script = Arc::new(tokio::sync::Notify::new()); + let requested_for_route = Arc::clone(&script_requested); + let release_for_route = Arc::clone(&release_script); + let fixture = Router::new().route("/page", get(page)).route( + "/app.js", + get(move || { + let requested = Arc::clone(&requested_for_route); + let release = Arc::clone(&release_for_route); + async move { + requested.notify_one(); + release.notified().await; + ( + [(axum::http::header::CONTENT_TYPE.as_str(), "text/javascript")], + "globalThis.continuedScript = true;", + ) + } + }), + ); + let (fixture_addr, _fixture_server) = + spawn_dedicated_fixture_server(fixture, "fetch-continued-document"); + let (cdp_addr, protocol_server) = spawn_test_protocol_server().await; + let endpoint = format!("ws://{cdp_addr}/devtools/browser/{DEFAULT_BROWSER_ID}"); + let (mut socket, _) = connect_async(&endpoint).await.expect("connect CDP client"); + let (mut other, _) = connect_async(&endpoint) + .await + .expect("connect independent CDP client"); + let context_id = cdp_create_browser_context(&mut socket, 1).await; + let session = cdp_create_attached_target(&mut socket, 2, &context_id).await; + let _ = send_cdp_command( + &mut socket, + 4, + "Page.enable", + Some(&session.session_id), + json!({}), + ) + .await; + let _ = send_cdp_command( + &mut socket, + 5, + "Fetch.enable", + Some(&session.session_id), + json!({ + "patterns": [{"urlPattern": pattern, "requestStage": "Request"}], + "handleAuthRequests": handle_auth_requests, + }), + ) + .await; + send_cdp_command_without_wait( + &mut socket, + 6, + "Page.navigate", + Some(&session.session_id), + json!({"url": format!("http://{fixture_addr}/page")}), + ) + .await; + let mut observed = recv_until_match(&mut socket, |message| { + message["method"] == json!("Fetch.requestPaused") + && message["params"]["resourceType"] == json!("Document") + }) + .await; + let request_id = observed.last().unwrap()["params"]["requestId"].clone(); + observed.extend( + send_cdp_command( + &mut socket, + 7, + "Fetch.continueRequest", + Some(&session.session_id), + json!({"requestId": request_id}), + ) + .await, + ); + assert!( + observed + .iter() + .any(|message| { message["id"] == json!(7) && message["result"] == json!({}) }) + ); + + if pattern == "*" { + if !observed.iter().any(|message| { + message["method"] == json!("Fetch.requestPaused") + && message["params"]["resourceType"] == json!("Script") + }) { + observed.extend( + recv_until_match(&mut socket, |message| { + message["method"] == json!("Fetch.requestPaused") + && message["params"]["resourceType"] == json!("Script") + }) + .await, + ); + } + let paused = observed + .iter() + .find(|message| { + message["method"] == json!("Fetch.requestPaused") + && message["params"]["resourceType"] == json!("Script") + }) + .unwrap(); + assert_eq!(paused["sessionId"], session.session_id); + let request_id = paused["params"]["requestId"].clone(); + observed.extend( + send_cdp_command( + &mut socket, + 8, + "Fetch.continueRequest", + Some(&session.session_id), + json!({"requestId": request_id}), + ) + .await, + ); + } + timeout(Duration::from_secs(30), script_requested.notified()) + .await + .expect("continued document must start its parser-blocking script request"); + if !observed.iter().any(|message| message["id"] == json!(6)) { + observed.extend(recv_until_id(&mut socket, 6).await); + } + let targets = send_cdp_command(&mut other, 1, "Target.getTargets", None, json!({})).await; + assert!(targets.iter().any(|message| { + message["result"]["targetInfos"] + .as_array() + .is_some_and(|targets| { + targets + .iter() + .any(|target| target["targetId"] == session.target_id) + }) + })); + let pending = cdp_runtime_evaluate_string( + &mut socket, + &session.session_id, + 9, + "JSON.stringify([globalThis.continuedDocument === true, globalThis.continuedScript === true, document.readyState])", + ) + .await; + assert_eq!(pending, r#"[true,false,"loading"]"#, "pattern: {pattern}"); + release_script.notify_one(); + if !observed + .iter() + .any(|message| message["method"] == json!("Page.loadEventFired")) + { + observed.extend( + recv_until_match(&mut socket, |message| { + message["method"] == json!("Page.loadEventFired") + }) + .await, + ); + } + assert!( + observed + .iter() + .any(|message| { message["id"] == json!(6) && message.get("error").is_none() }) + ); + let completed = cdp_runtime_evaluate_string( + &mut socket, + &session.session_id, + 10, + "JSON.stringify([globalThis.continuedScript === true, document.querySelector('p').textContent])", + ) + .await; + assert_eq!(completed, r#"[true,"loaded"]"#, "pattern: {pattern}"); + let _ = other.close(None).await; + let _ = socket.close(None).await; + protocol_server.abort(); + } +} + #[tokio::test] async fn websocket_cdp_fetch_routes_pending_background_parser_script_to_exact_session() { async fn page() -> impl IntoResponse { diff --git a/moli-protocol/src/domains/fetch/commands.rs b/moli-protocol/src/domains/fetch/commands.rs index 5b30cf70ef9..731d4628827 100644 --- a/moli-protocol/src/domains/fetch/commands.rs +++ b/moli-protocol/src/domains/fetch/commands.rs @@ -1857,7 +1857,7 @@ pub(super) async fn complete_continue_response_command_async( } } -fn continue_streaming_document_response_in_background( +pub(super) fn continue_streaming_document_response_in_background( conn: &mut CdpConnection, sender: tokio::sync::mpsc::UnboundedSender, pending: PendingStreamingDocumentResponseNavigation, diff --git a/moli-protocol/src/domains/fetch/navigation.rs b/moli-protocol/src/domains/fetch/navigation.rs index dfe0771e481..1fca0050d6a 100644 --- a/moli-protocol/src/domains/fetch/navigation.rs +++ b/moli-protocol/src/domains/fetch/navigation.rs @@ -2,8 +2,9 @@ use crate::automation::{DevToolsNetworkInterceptId, DevToolsNetworkResourceType} use crate::conn::{ CdpConnection, CommandOwnerScope, DocumentBodySource, DocumentNavigationToken, FetchAuthChallenge, NavigationDispatchState, NavigationLoadOutcome, PendingFetchNavigation, - PendingSubresourceFetchAuthStage, PendingSubresourceFetchAuthStageChain, - PendingSubresourceFetchOwnerKind, PendingSubresourceFetchRequest, decode_data_url_response, + PendingStreamingDocumentResponseNavigation, PendingSubresourceFetchAuthStage, + PendingSubresourceFetchAuthStageChain, PendingSubresourceFetchOwnerKind, + PendingSubresourceFetchRequest, decode_data_url_response, }; use crate::domains::{command_output::CommandOutputBuffer, network, page}; use moli_cookie_jar::StoredCookieQueryReport; @@ -83,7 +84,7 @@ pub(crate) async fn load_or_pause_navigation_for_auth_into_buffer_async( if should_handle_auth { let has_auth_credentials = auth.is_some(); - if pending.intercept_response && navigation_response_stage_auth_can_stream(auth.as_ref()) { + if navigation_response_stage_auth_can_stream(auth.as_ref()) { match conn .fetch_navigation_streaming_raw_response_for_navigation_async( &pending.navigation, @@ -206,7 +207,9 @@ pub(crate) async fn load_or_pause_navigation_for_auth_into_buffer_async( return; } - if pending.intercept_response && pending.navigation.requested_url.scheme() != "data" { + if matches!(pending.navigation.requested_url.scheme(), "http" | "https") + || (pending.intercept_response && pending.navigation.requested_url.scheme() != "data") + { match conn .fetch_navigation_streaming_raw_response_for_navigation_async(&pending.navigation, None) .await @@ -511,6 +514,29 @@ async fn handle_streaming_response_head_for_navigation_into_buffer_async( } if !prepare_navigation_response_stage(conn, &mut pending, &response_head.final_url) { + // Parser-blocking scripts need the scheduler to process renderer + // output, so do not wait for document parsing inside this command. + if let Some(document_navigation_token) = pending.document_navigation_token.clone() + && let Some(sender) = + conn.background_navigation_completion_sender_for_owner(&pending.navigation.owner) + { + let (response, network_observation_journal) = + response.into_parts_with_observation_journal(); + super::commands::continue_streaming_document_response_in_background( + conn, + sender, + PendingStreamingDocumentResponseNavigation { + document_navigation_token, + navigation: pending.navigation, + response, + network_observation_journal, + body_progress_source: network::MainDocumentBodyProgressSource::default(), + }, + None, + Vec::new(), + ); + return; + } let navigation = conn .build_navigation_from_streaming_raw_response_for_navigation_async( pending.document_navigation_token.as_ref(),