fix(fetch): limit idle waiting to initial page readiness

This commit is contained in:
ldm0
2026-11-15 00:48:02 +08:00
committed by Donough Liu
parent 9247282d21
commit abcff16028
5 changed files with 98 additions and 190 deletions
+1 -8
View File
@@ -16,8 +16,6 @@ pub enum FetchTimeoutPhase {
WaitingForSubresourceResponse,
WaitingForSelector,
WaitingForScript,
WaitingForPageDelay,
EvaluatingScript,
WaitingForIdle,
}
@@ -57,8 +55,6 @@ impl fmt::Display for FetchTimeoutPhase {
Self::WaitingForSubresourceResponse => "waiting for a subresource response",
Self::WaitingForSelector => "waiting for a selector",
Self::WaitingForScript => "waiting for a script to become truthy",
Self::WaitingForPageDelay => "waiting for page delay",
Self::EvaluatingScript => "evaluating JavaScript",
Self::WaitingForIdle => "waiting for idle",
})
}
@@ -139,10 +135,7 @@ impl FetchDeadline {
self.at
}
/// Enforces this absolute deadline while the supplied operation runs.
/// Callers must still close or cancel work retained by another owner when
/// the timed-out future is dropped.
pub async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>
pub(super) async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>
where
F: Future<Output = Result<T>>,
{
+31 -46
View File
@@ -21,7 +21,7 @@ use anyhow::Result;
use anyhow::{Context, anyhow};
use clap::Parser;
use moli_core::runtime::{
Browser, FetchReadinessTimeout, FetchTimeoutPhase, FetchedDocument, NavigationRuntimeConfig,
Browser, FetchReadinessTimeout, FetchedDocument, NavigationRuntimeConfig,
RawDocumentFetchPolicy, storage_partition::StoragePartitionState,
};
use moli_fetch::{CookieHeaderPolicy, NetworkFetchFailureContext, Request};
@@ -146,18 +146,14 @@ pub async fn run_cli_with_config<W: Write>(
let readiness_result = async {
readiness.wait_for_page(&browser, &mut page).await?;
if args.delay_ms > 0 {
readiness
.within_idle_deadline(
FetchTimeoutPhase::WaitingForPageDelay,
browser.wait_for_page_delay(
&mut page,
std::time::Duration::from_millis(args.delay_ms),
),
browser
.wait_for_page_delay(
&mut page,
std::time::Duration::from_millis(args.delay_ms),
)
.await
.context("failed while waiting for page delay")?;
}
readiness.wait_for_idle(&browser, &mut page).await?;
Ok::<(), anyhow::Error>(())
}
.await;
@@ -205,49 +201,38 @@ pub async fn run_cli_with_config<W: Write>(
let http_failure = http_status_failure(&args, page_status, final_url.as_str());
let with_output_context =
|error| with_http_status_context(error, &args, page_status, final_url.as_str());
let output_result = async {
if !args.fail || http_failure.is_none() {
let rendered = if let Some(expression) = eval_expression.as_deref() {
let value = readiness
.within_idle_deadline(FetchTimeoutPhase::EvaluatingScript, async {
if config.browser.layout_policy().uses_real_layout() {
page.publish_layout_async()
.await
.context("failed to publish layout before evaluating JavaScript")?;
}
if args.dump.is_some() {
eval_output::execute(&mut page, expression).await?;
Ok(None)
} else {
eval_output::evaluate(&mut page, expression).await.map(Some)
}
})
.await?;
readiness.wait_for_idle(&browser, &mut page).await?;
match value {
Some(value) => value,
None => {
fetch_dump::render_page_output_async(&mut page, &config.fetch).await?
}
}
if !args.fail || http_failure.is_none() {
let rendered = if let Some(expression) = eval_expression.as_deref() {
if config.browser.layout_policy().uses_real_layout() {
page.publish_layout_async()
.await
.context("failed to publish layout before evaluating JavaScript")
.map_err(with_output_context)?;
}
if args.dump.is_some() {
eval_output::execute(&mut page, expression)
.await
.map_err(with_output_context)?;
fetch_dump::render_page_output_async(&mut page, &config.fetch).await
} else {
fetch_dump::render_page_output_async(&mut page, &config.fetch).await?
};
stdout
.write_all(&rendered)
.context("failed to write fetch output")?;
stdout
.flush()
.context("failed to flush fetch output")?;
eval_output::evaluate(&mut page, expression).await
}
} else {
fetch_dump::render_page_output_async(&mut page, &config.fetch).await
}
Ok::<(), anyhow::Error>(())
.map_err(with_output_context)?;
stdout
.write_all(&rendered)
.context("failed to write fetch output")
.map_err(with_output_context)?;
stdout
.flush()
.context("failed to flush fetch output")
.map_err(with_output_context)?;
}
.await
.map_err(with_output_context);
if let Err(error) = page.close_async().await {
tracing::warn!(error = %error, "failed to close fetched page before browser shutdown");
}
output_result?;
if let Some(error) = http_failure {
return Err(error);
}
+2 -28
View File
@@ -13,12 +13,11 @@ use anyhow::{Context, Result, anyhow, bail};
use moli_core::{
page::{Page, SubresourceResponseWaitCriteria},
runtime::{
Browser, FetchDeadline, FetchTimeoutPhase, FetchedDocument, RawDocumentFetchPolicy,
RenderedDomWaitUntil,
Browser, FetchDeadline, FetchedDocument, RawDocumentFetchPolicy, RenderedDomWaitUntil,
},
};
use moli_fetch::Request;
use std::{future::Future, path::Path, time::Duration};
use std::{path::Path, time::Duration};
#[derive(Debug)]
pub(super) struct ReadinessPlan {
@@ -110,31 +109,6 @@ impl ReadinessPlan {
Ok(())
}
/// Include work scheduled by automation predicates, delays and --eval in
/// the explicit idle lifecycle, sharing the original readiness deadline.
pub(super) async fn wait_for_idle(&self, browser: &Browser, page: &mut Page) -> Result<()> {
if self.wait_until == RenderedDomWaitUntil::Idle {
browser
.wait_for_page_readiness_with_deadline(page, self.wait_until, self.deadline)
.await?;
}
Ok(())
}
/// Idle bounds execution as well as the subsequent resource observation.
/// Other modes retain their existing delay and evaluation policy.
pub(super) async fn within_idle_deadline<T>(
&self,
phase: FetchTimeoutPhase,
operation: impl Future<Output = Result<T>>,
) -> Result<T> {
if self.wait_until == RenderedDomWaitUntil::Idle {
self.deadline.wait(phase, operation).await
} else {
operation.await
}
}
}
fn rendered_wait_until(wait_until: FetchWaitUntil) -> RenderedDomWaitUntil {
+6 -6
View File
@@ -274,10 +274,9 @@ pub struct FetchArgs {
/// Maximum total readiness time in milliseconds. Initial and HTTP-error
/// replacement navigations, the selected lifecycle stage, response match,
/// selector, and script waits share one absolute deadline. Idle also bounds
/// --delay-ms and --eval/--eval-file. Network-idle and DOM-stable return the
/// current page when it expires; idle fails with a timeout error. Pass
/// `--log-level warn` to see timeout warnings.
/// selector, and script waits share one absolute deadline. Network-idle and
/// DOM-stable return the current page when it expires; idle fails with a
/// timeout error. Pass `--log-level warn` to see timeout warnings.
#[arg(
short,
long = "timeout-ms",
@@ -766,8 +765,9 @@ pub enum FetchWaitUntil {
Load,
NetworkIdle,
DomStable,
/// Wait for user async work and callbacks to finish. Active intervals,
/// Workers, WebSockets and receiving MessagePorts keep the wait pending.
/// Wait for initial page async work and callbacks to finish. Active
/// intervals, Workers, WebSockets and receiving MessagePorts keep the wait
/// pending. Later script and delay options do not repeat this wait.
Idle,
Done,
}
+58 -102
View File
@@ -7,7 +7,7 @@ use axum::{
routing::get,
};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use std::{process::Command, time::Duration};
use std::time::Duration;
fn document(script: &str) -> String {
format!(
@@ -423,30 +423,29 @@ fn idle_has_a_strict_timeout_for_live_workers_and_websockets() -> Result<()> {
}
#[test]
fn idle_waits_for_work_started_by_eval_before_exporting() -> Result<()> {
let script =
"setTimeout(() => document.querySelector('main').textContent = 'done', 100); 'value'";
fn idle_does_not_repeat_after_eval() -> Result<()> {
let script = "setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'; 'value'";
assert_done(run(
&document(""),
"3000",
"1000",
&["--dump", "html", "--eval", script],
)?);
let output = run(&document(""), "3000", &["--eval", script])?;
let output = run(&document(""), "1000", &["--eval", script])?;
assert!(output.status.success(), "{}", clean_output(&output.stderr));
assert_eq!(clean_output(&output.stdout), "value\n");
Ok(())
}
#[test]
fn idle_waits_for_work_started_by_eval_file() -> Result<()> {
fn idle_does_not_repeat_after_eval_file() -> Result<()> {
let path = unique_temp_file_path("wait-idle", "eval.js")?;
std::fs::write(
&path,
"setTimeout(() => document.querySelector('main').textContent = 'done', 100)",
"setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'",
)?;
let output = run(
&document(""),
"3000",
"1000",
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
);
std::fs::remove_file(path)?;
@@ -455,15 +454,15 @@ fn idle_waits_for_work_started_by_eval_file() -> Result<()> {
}
#[test]
fn idle_also_drains_work_created_by_the_readiness_script() -> Result<()> {
fn idle_does_not_repeat_after_readiness_script() -> Result<()> {
assert_done(run(
&document(""),
"3000",
"1000",
&[
"--dump",
"html",
"--wait-script",
"(setTimeout(() => document.querySelector('main').textContent='done', 100), true)",
"(setInterval(() => {}, 60000), document.querySelector('main').textContent='done', true)",
],
)?);
Ok(())
@@ -488,106 +487,63 @@ fn idle_waits_for_child_window_timers_and_releases_removed_windows() -> Result<(
}
#[test]
fn idle_keeps_one_deadline_for_work_started_after_initial_readiness() -> Result<()> {
let output = run(
&document("setTimeout(() => {}, 300)"),
"1000",
&["--dump", "html", "--eval", "setTimeout(() => {}, 800)"],
)?;
assert!(!output.status.success());
assert!(output.stdout.is_empty());
assert!(
clean_output(&output.stderr).contains("timed out after 1000 ms while waiting for idle"),
"{}",
clean_output(&output.stderr)
);
Ok(())
}
fn assert_idle_execution_times_out(extra: &[&str], phase: &str) -> Result<()> {
let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));
command
.args([
"fetch",
"--wait",
"idle",
"--timeout-ms",
"1000",
"--http-no-proxy",
"*",
"--log-level",
"error",
])
.args(extra)
.arg(document(""));
// Also bound teardown: dropping only the caller future would leave an
// awaited Promise in the renderer and hang browser shutdown.
let output =
super::subprocess::run_command_with_process_timeout(command, Duration::from_secs(5))?;
let stderr = clean_output(&output.stderr);
assert!(
!output.status.success(),
"stdout={}",
clean_output(&output.stdout)
);
assert!(output.stdout.is_empty());
assert!(
stderr.contains(&format!("timed out after 1000 ms while {phase}")),
"{stderr}"
);
Ok(())
}
#[test]
fn idle_deadline_interrupts_async_eval_and_acknowledges_page_cleanup() -> Result<()> {
for expression in [
"new Promise(resolve => setTimeout(() => resolve('late'), 10000))",
"new Promise(() => {})",
] {
assert_idle_execution_times_out(&["--eval", expression], "evaluating JavaScript")?;
assert_idle_execution_times_out(
&["--dump", "html", "--eval", expression],
"evaluating JavaScript",
)?;
}
Ok(())
}
#[test]
fn idle_deadline_interrupts_eval_file() -> Result<()> {
let path = unique_temp_file_path("idle-eval-timeout", "eval.js")?;
std::fs::write(&path, "new Promise(resolve => setTimeout(resolve, 10000))")?;
let result = assert_idle_execution_times_out(
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
"evaluating JavaScript",
);
std::fs::remove_file(path)?;
result
}
#[test]
fn idle_deadline_interrupts_page_delay() -> Result<()> {
assert_idle_execution_times_out(
&["--dump", "html", "--delay-ms", "10000"],
"waiting for page delay",
)
}
#[test]
fn idle_waits_for_async_eval_and_followup_work_that_fit_the_deadline() -> Result<()> {
fn idle_does_not_repeat_after_page_delay() -> Result<()> {
assert_done(run(
&document(""),
"3000",
"1000",
&[
"--dump",
"html",
"--eval",
"new Promise(resolve => setTimeout(() => { setTimeout(() => document.querySelector('main').textContent = 'done', 100); resolve('value'); }, 100))",
"--wait-script",
"(setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; }, 25), true)",
"--delay-ms",
"100",
],
)?);
Ok(())
}
#[test]
fn idle_keeps_async_eval_outside_readiness_deadline() -> Result<()> {
let expression = "new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve('value'); }, 1200))";
assert_done(run(
&document(""),
"1000",
&["--dump", "html", "--eval", expression],
)?);
let output = run(&document(""), "1000", &["--eval", expression])?;
assert!(output.status.success(), "{}", clean_output(&output.stderr));
assert_eq!(clean_output(&output.stdout), "value\n");
Ok(())
}
#[test]
fn idle_keeps_async_eval_file_outside_readiness_deadline() -> Result<()> {
let path = unique_temp_file_path("idle-async-eval", "eval.js")?;
std::fs::write(
&path,
"new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve(); }, 1200))",
)?;
let output = run(
&document(""),
"1000",
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
);
std::fs::remove_file(path)?;
assert_done(output?);
Ok(())
}
#[test]
fn idle_keeps_page_delay_outside_readiness_deadline() -> Result<()> {
assert_done(run(
&document("document.querySelector('main').textContent='done'"),
"1000",
&["--dump", "html", "--delay-ms", "1200"],
)?);
Ok(())
}
#[test]
fn idle_follows_navigation_and_keeps_waiting_on_the_replacement_document() -> Result<()> {
let server = Server::new("setTimeout(() => location.href = '/next', 40)")?;