From abcff1602808b47b2b54cc208f6632fcb2cef884 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Sat, 14 Nov 2026 23:57:07 +0800 Subject: [PATCH] fix(fetch): limit idle waiting to initial page readiness --- moli-core/src/runtime/fetch_deadline.rs | 9 +- moli/src/app.rs | 77 +++++------- moli/src/app/readiness.rs | 30 +---- moli/src/cli.rs | 12 +- moli/tests/fetch_cli/idle.rs | 160 +++++++++--------------- 5 files changed, 98 insertions(+), 190 deletions(-) diff --git a/moli-core/src/runtime/fetch_deadline.rs b/moli-core/src/runtime/fetch_deadline.rs index 6c98ff79a62..e9d50128607 100644 --- a/moli-core/src/runtime/fetch_deadline.rs +++ b/moli-core/src/runtime/fetch_deadline.rs @@ -16,8 +16,6 @@ pub enum FetchTimeoutPhase { WaitingForSubresourceResponse, WaitingForSelector, WaitingForScript, - WaitingForPageDelay, - EvaluatingScript, WaitingForIdle, } @@ -57,8 +55,6 @@ impl fmt::Display for FetchTimeoutPhase { Self::WaitingForSubresourceResponse => "waiting for a subresource response", Self::WaitingForSelector => "waiting for a selector", Self::WaitingForScript => "waiting for a script to become truthy", - Self::WaitingForPageDelay => "waiting for page delay", - Self::EvaluatingScript => "evaluating JavaScript", Self::WaitingForIdle => "waiting for idle", }) } @@ -139,10 +135,7 @@ impl FetchDeadline { self.at } - /// Enforces this absolute deadline while the supplied operation runs. - /// Callers must still close or cancel work retained by another owner when - /// the timed-out future is dropped. - pub async fn wait(self, phase: FetchTimeoutPhase, future: F) -> Result + pub(super) async fn wait(self, phase: FetchTimeoutPhase, future: F) -> Result where F: Future>, { diff --git a/moli/src/app.rs b/moli/src/app.rs index 5684a9ab902..3aa1230c3e2 100644 --- a/moli/src/app.rs +++ b/moli/src/app.rs @@ -21,7 +21,7 @@ use anyhow::Result; use anyhow::{Context, anyhow}; use clap::Parser; use moli_core::runtime::{ - Browser, FetchReadinessTimeout, FetchTimeoutPhase, FetchedDocument, NavigationRuntimeConfig, + Browser, FetchReadinessTimeout, FetchedDocument, NavigationRuntimeConfig, RawDocumentFetchPolicy, storage_partition::StoragePartitionState, }; use moli_fetch::{CookieHeaderPolicy, NetworkFetchFailureContext, Request}; @@ -146,18 +146,14 @@ pub async fn run_cli_with_config( let readiness_result = async { readiness.wait_for_page(&browser, &mut page).await?; if args.delay_ms > 0 { - readiness - .within_idle_deadline( - FetchTimeoutPhase::WaitingForPageDelay, - browser.wait_for_page_delay( - &mut page, - std::time::Duration::from_millis(args.delay_ms), - ), + browser + .wait_for_page_delay( + &mut page, + std::time::Duration::from_millis(args.delay_ms), ) .await .context("failed while waiting for page delay")?; } - readiness.wait_for_idle(&browser, &mut page).await?; Ok::<(), anyhow::Error>(()) } .await; @@ -205,49 +201,38 @@ pub async fn run_cli_with_config( let http_failure = http_status_failure(&args, page_status, final_url.as_str()); let with_output_context = |error| with_http_status_context(error, &args, page_status, final_url.as_str()); - let output_result = async { - if !args.fail || http_failure.is_none() { - let rendered = if let Some(expression) = eval_expression.as_deref() { - let value = readiness - .within_idle_deadline(FetchTimeoutPhase::EvaluatingScript, async { - if config.browser.layout_policy().uses_real_layout() { - page.publish_layout_async() - .await - .context("failed to publish layout before evaluating JavaScript")?; - } - if args.dump.is_some() { - eval_output::execute(&mut page, expression).await?; - Ok(None) - } else { - eval_output::evaluate(&mut page, expression).await.map(Some) - } - }) - .await?; - readiness.wait_for_idle(&browser, &mut page).await?; - match value { - Some(value) => value, - None => { - fetch_dump::render_page_output_async(&mut page, &config.fetch).await? - } - } + if !args.fail || http_failure.is_none() { + let rendered = if let Some(expression) = eval_expression.as_deref() { + if config.browser.layout_policy().uses_real_layout() { + page.publish_layout_async() + .await + .context("failed to publish layout before evaluating JavaScript") + .map_err(with_output_context)?; + } + if args.dump.is_some() { + eval_output::execute(&mut page, expression) + .await + .map_err(with_output_context)?; + fetch_dump::render_page_output_async(&mut page, &config.fetch).await } else { - fetch_dump::render_page_output_async(&mut page, &config.fetch).await? - }; - stdout - .write_all(&rendered) - .context("failed to write fetch output")?; - stdout - .flush() - .context("failed to flush fetch output")?; + eval_output::evaluate(&mut page, expression).await + } + } else { + fetch_dump::render_page_output_async(&mut page, &config.fetch).await } - Ok::<(), anyhow::Error>(()) + .map_err(with_output_context)?; + stdout + .write_all(&rendered) + .context("failed to write fetch output") + .map_err(with_output_context)?; + stdout + .flush() + .context("failed to flush fetch output") + .map_err(with_output_context)?; } - .await - .map_err(with_output_context); if let Err(error) = page.close_async().await { tracing::warn!(error = %error, "failed to close fetched page before browser shutdown"); } - output_result?; if let Some(error) = http_failure { return Err(error); } diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rs index a81e36815fb..999421b07ad 100644 --- a/moli/src/app/readiness.rs +++ b/moli/src/app/readiness.rs @@ -13,12 +13,11 @@ use anyhow::{Context, Result, anyhow, bail}; use moli_core::{ page::{Page, SubresourceResponseWaitCriteria}, runtime::{ - Browser, FetchDeadline, FetchTimeoutPhase, FetchedDocument, RawDocumentFetchPolicy, - RenderedDomWaitUntil, + Browser, FetchDeadline, FetchedDocument, RawDocumentFetchPolicy, RenderedDomWaitUntil, }, }; use moli_fetch::Request; -use std::{future::Future, path::Path, time::Duration}; +use std::{path::Path, time::Duration}; #[derive(Debug)] pub(super) struct ReadinessPlan { @@ -110,31 +109,6 @@ impl ReadinessPlan { Ok(()) } - - /// Include work scheduled by automation predicates, delays and --eval in - /// the explicit idle lifecycle, sharing the original readiness deadline. - pub(super) async fn wait_for_idle(&self, browser: &Browser, page: &mut Page) -> Result<()> { - if self.wait_until == RenderedDomWaitUntil::Idle { - browser - .wait_for_page_readiness_with_deadline(page, self.wait_until, self.deadline) - .await?; - } - Ok(()) - } - - /// Idle bounds execution as well as the subsequent resource observation. - /// Other modes retain their existing delay and evaluation policy. - pub(super) async fn within_idle_deadline( - &self, - phase: FetchTimeoutPhase, - operation: impl Future>, - ) -> Result { - if self.wait_until == RenderedDomWaitUntil::Idle { - self.deadline.wait(phase, operation).await - } else { - operation.await - } - } } fn rendered_wait_until(wait_until: FetchWaitUntil) -> RenderedDomWaitUntil { diff --git a/moli/src/cli.rs b/moli/src/cli.rs index a328f9633b0..ec4cae556ec 100644 --- a/moli/src/cli.rs +++ b/moli/src/cli.rs @@ -274,10 +274,9 @@ pub struct FetchArgs { /// Maximum total readiness time in milliseconds. Initial and HTTP-error /// replacement navigations, the selected lifecycle stage, response match, - /// selector, and script waits share one absolute deadline. Idle also bounds - /// --delay-ms and --eval/--eval-file. Network-idle and DOM-stable return the - /// current page when it expires; idle fails with a timeout error. Pass - /// `--log-level warn` to see timeout warnings. + /// selector, and script waits share one absolute deadline. Network-idle and + /// DOM-stable return the current page when it expires; idle fails with a + /// timeout error. Pass `--log-level warn` to see timeout warnings. #[arg( short, long = "timeout-ms", @@ -766,8 +765,9 @@ pub enum FetchWaitUntil { Load, NetworkIdle, DomStable, - /// Wait for user async work and callbacks to finish. Active intervals, - /// Workers, WebSockets and receiving MessagePorts keep the wait pending. + /// Wait for initial page async work and callbacks to finish. Active + /// intervals, Workers, WebSockets and receiving MessagePorts keep the wait + /// pending. Later script and delay options do not repeat this wait. Idle, Done, } diff --git a/moli/tests/fetch_cli/idle.rs b/moli/tests/fetch_cli/idle.rs index 00a073b9304..bb3f9b92c03 100644 --- a/moli/tests/fetch_cli/idle.rs +++ b/moli/tests/fetch_cli/idle.rs @@ -7,7 +7,7 @@ use axum::{ routing::get, }; use base64::{Engine as _, engine::general_purpose::STANDARD}; -use std::{process::Command, time::Duration}; +use std::time::Duration; fn document(script: &str) -> String { format!( @@ -423,30 +423,29 @@ fn idle_has_a_strict_timeout_for_live_workers_and_websockets() -> Result<()> { } #[test] -fn idle_waits_for_work_started_by_eval_before_exporting() -> Result<()> { - let script = - "setTimeout(() => document.querySelector('main').textContent = 'done', 100); 'value'"; +fn idle_does_not_repeat_after_eval() -> Result<()> { + let script = "setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'; 'value'"; assert_done(run( &document(""), - "3000", + "1000", &["--dump", "html", "--eval", script], )?); - let output = run(&document(""), "3000", &["--eval", script])?; + let output = run(&document(""), "1000", &["--eval", script])?; assert!(output.status.success(), "{}", clean_output(&output.stderr)); assert_eq!(clean_output(&output.stdout), "value\n"); Ok(()) } #[test] -fn idle_waits_for_work_started_by_eval_file() -> Result<()> { +fn idle_does_not_repeat_after_eval_file() -> Result<()> { let path = unique_temp_file_path("wait-idle", "eval.js")?; std::fs::write( &path, - "setTimeout(() => document.querySelector('main').textContent = 'done', 100)", + "setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'", )?; let output = run( &document(""), - "3000", + "1000", &["--dump", "html", "--eval-file", path.to_str().unwrap()], ); std::fs::remove_file(path)?; @@ -455,15 +454,15 @@ fn idle_waits_for_work_started_by_eval_file() -> Result<()> { } #[test] -fn idle_also_drains_work_created_by_the_readiness_script() -> Result<()> { +fn idle_does_not_repeat_after_readiness_script() -> Result<()> { assert_done(run( &document(""), - "3000", + "1000", &[ "--dump", "html", "--wait-script", - "(setTimeout(() => document.querySelector('main').textContent='done', 100), true)", + "(setInterval(() => {}, 60000), document.querySelector('main').textContent='done', true)", ], )?); Ok(()) @@ -488,106 +487,63 @@ fn idle_waits_for_child_window_timers_and_releases_removed_windows() -> Result<( } #[test] -fn idle_keeps_one_deadline_for_work_started_after_initial_readiness() -> Result<()> { - let output = run( - &document("setTimeout(() => {}, 300)"), - "1000", - &["--dump", "html", "--eval", "setTimeout(() => {}, 800)"], - )?; - assert!(!output.status.success()); - assert!(output.stdout.is_empty()); - assert!( - clean_output(&output.stderr).contains("timed out after 1000 ms while waiting for idle"), - "{}", - clean_output(&output.stderr) - ); - Ok(()) -} - -fn assert_idle_execution_times_out(extra: &[&str], phase: &str) -> Result<()> { - let mut command = Command::new(env!("CARGO_BIN_EXE_moli")); - command - .args([ - "fetch", - "--wait", - "idle", - "--timeout-ms", - "1000", - "--http-no-proxy", - "*", - "--log-level", - "error", - ]) - .args(extra) - .arg(document("")); - // Also bound teardown: dropping only the caller future would leave an - // awaited Promise in the renderer and hang browser shutdown. - let output = - super::subprocess::run_command_with_process_timeout(command, Duration::from_secs(5))?; - let stderr = clean_output(&output.stderr); - assert!( - !output.status.success(), - "stdout={}", - clean_output(&output.stdout) - ); - assert!(output.stdout.is_empty()); - assert!( - stderr.contains(&format!("timed out after 1000 ms while {phase}")), - "{stderr}" - ); - Ok(()) -} - -#[test] -fn idle_deadline_interrupts_async_eval_and_acknowledges_page_cleanup() -> Result<()> { - for expression in [ - "new Promise(resolve => setTimeout(() => resolve('late'), 10000))", - "new Promise(() => {})", - ] { - assert_idle_execution_times_out(&["--eval", expression], "evaluating JavaScript")?; - assert_idle_execution_times_out( - &["--dump", "html", "--eval", expression], - "evaluating JavaScript", - )?; - } - Ok(()) -} - -#[test] -fn idle_deadline_interrupts_eval_file() -> Result<()> { - let path = unique_temp_file_path("idle-eval-timeout", "eval.js")?; - std::fs::write(&path, "new Promise(resolve => setTimeout(resolve, 10000))")?; - let result = assert_idle_execution_times_out( - &["--dump", "html", "--eval-file", path.to_str().unwrap()], - "evaluating JavaScript", - ); - std::fs::remove_file(path)?; - result -} - -#[test] -fn idle_deadline_interrupts_page_delay() -> Result<()> { - assert_idle_execution_times_out( - &["--dump", "html", "--delay-ms", "10000"], - "waiting for page delay", - ) -} - -#[test] -fn idle_waits_for_async_eval_and_followup_work_that_fit_the_deadline() -> Result<()> { +fn idle_does_not_repeat_after_page_delay() -> Result<()> { assert_done(run( &document(""), - "3000", + "1000", &[ "--dump", "html", - "--eval", - "new Promise(resolve => setTimeout(() => { setTimeout(() => document.querySelector('main').textContent = 'done', 100); resolve('value'); }, 100))", + "--wait-script", + "(setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; }, 25), true)", + "--delay-ms", + "100", ], )?); Ok(()) } +#[test] +fn idle_keeps_async_eval_outside_readiness_deadline() -> Result<()> { + let expression = "new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve('value'); }, 1200))"; + assert_done(run( + &document(""), + "1000", + &["--dump", "html", "--eval", expression], + )?); + let output = run(&document(""), "1000", &["--eval", expression])?; + assert!(output.status.success(), "{}", clean_output(&output.stderr)); + assert_eq!(clean_output(&output.stdout), "value\n"); + Ok(()) +} + +#[test] +fn idle_keeps_async_eval_file_outside_readiness_deadline() -> Result<()> { + let path = unique_temp_file_path("idle-async-eval", "eval.js")?; + std::fs::write( + &path, + "new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve(); }, 1200))", + )?; + let output = run( + &document(""), + "1000", + &["--dump", "html", "--eval-file", path.to_str().unwrap()], + ); + std::fs::remove_file(path)?; + assert_done(output?); + Ok(()) +} + +#[test] +fn idle_keeps_page_delay_outside_readiness_deadline() -> Result<()> { + assert_done(run( + &document("document.querySelector('main').textContent='done'"), + "1000", + &["--dump", "html", "--delay-ms", "1200"], + )?); + Ok(()) +} + #[test] fn idle_follows_navigation_and_keeps_waiting_on_the_replacement_document() -> Result<()> { let server = Server::new("setTimeout(() => location.href = '/next', 40)")?;