mirror of
https://github.com/lexmount/moli.git
synced 2026-11-22 00:02:21 +00:00
fix(fetch): limit idle waiting to initial page readiness
This commit is contained in:
@@ -16,8 +16,6 @@ pub enum FetchTimeoutPhase {
|
||||
WaitingForSubresourceResponse,
|
||||
WaitingForSelector,
|
||||
WaitingForScript,
|
||||
WaitingForPageDelay,
|
||||
EvaluatingScript,
|
||||
WaitingForIdle,
|
||||
}
|
||||
|
||||
@@ -57,8 +55,6 @@ impl fmt::Display for FetchTimeoutPhase {
|
||||
Self::WaitingForSubresourceResponse => "waiting for a subresource response",
|
||||
Self::WaitingForSelector => "waiting for a selector",
|
||||
Self::WaitingForScript => "waiting for a script to become truthy",
|
||||
Self::WaitingForPageDelay => "waiting for page delay",
|
||||
Self::EvaluatingScript => "evaluating JavaScript",
|
||||
Self::WaitingForIdle => "waiting for idle",
|
||||
})
|
||||
}
|
||||
@@ -139,10 +135,7 @@ impl FetchDeadline {
|
||||
self.at
|
||||
}
|
||||
|
||||
/// Enforces this absolute deadline while the supplied operation runs.
|
||||
/// Callers must still close or cancel work retained by another owner when
|
||||
/// the timed-out future is dropped.
|
||||
pub async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>
|
||||
pub(super) async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
|
||||
+31
-46
@@ -21,7 +21,7 @@ use anyhow::Result;
|
||||
use anyhow::{Context, anyhow};
|
||||
use clap::Parser;
|
||||
use moli_core::runtime::{
|
||||
Browser, FetchReadinessTimeout, FetchTimeoutPhase, FetchedDocument, NavigationRuntimeConfig,
|
||||
Browser, FetchReadinessTimeout, FetchedDocument, NavigationRuntimeConfig,
|
||||
RawDocumentFetchPolicy, storage_partition::StoragePartitionState,
|
||||
};
|
||||
use moli_fetch::{CookieHeaderPolicy, NetworkFetchFailureContext, Request};
|
||||
@@ -146,18 +146,14 @@ pub async fn run_cli_with_config<W: Write>(
|
||||
let readiness_result = async {
|
||||
readiness.wait_for_page(&browser, &mut page).await?;
|
||||
if args.delay_ms > 0 {
|
||||
readiness
|
||||
.within_idle_deadline(
|
||||
FetchTimeoutPhase::WaitingForPageDelay,
|
||||
browser.wait_for_page_delay(
|
||||
&mut page,
|
||||
std::time::Duration::from_millis(args.delay_ms),
|
||||
),
|
||||
browser
|
||||
.wait_for_page_delay(
|
||||
&mut page,
|
||||
std::time::Duration::from_millis(args.delay_ms),
|
||||
)
|
||||
.await
|
||||
.context("failed while waiting for page delay")?;
|
||||
}
|
||||
readiness.wait_for_idle(&browser, &mut page).await?;
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}
|
||||
.await;
|
||||
@@ -205,49 +201,38 @@ pub async fn run_cli_with_config<W: Write>(
|
||||
let http_failure = http_status_failure(&args, page_status, final_url.as_str());
|
||||
let with_output_context =
|
||||
|error| with_http_status_context(error, &args, page_status, final_url.as_str());
|
||||
let output_result = async {
|
||||
if !args.fail || http_failure.is_none() {
|
||||
let rendered = if let Some(expression) = eval_expression.as_deref() {
|
||||
let value = readiness
|
||||
.within_idle_deadline(FetchTimeoutPhase::EvaluatingScript, async {
|
||||
if config.browser.layout_policy().uses_real_layout() {
|
||||
page.publish_layout_async()
|
||||
.await
|
||||
.context("failed to publish layout before evaluating JavaScript")?;
|
||||
}
|
||||
if args.dump.is_some() {
|
||||
eval_output::execute(&mut page, expression).await?;
|
||||
Ok(None)
|
||||
} else {
|
||||
eval_output::evaluate(&mut page, expression).await.map(Some)
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
readiness.wait_for_idle(&browser, &mut page).await?;
|
||||
match value {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
fetch_dump::render_page_output_async(&mut page, &config.fetch).await?
|
||||
}
|
||||
}
|
||||
if !args.fail || http_failure.is_none() {
|
||||
let rendered = if let Some(expression) = eval_expression.as_deref() {
|
||||
if config.browser.layout_policy().uses_real_layout() {
|
||||
page.publish_layout_async()
|
||||
.await
|
||||
.context("failed to publish layout before evaluating JavaScript")
|
||||
.map_err(with_output_context)?;
|
||||
}
|
||||
if args.dump.is_some() {
|
||||
eval_output::execute(&mut page, expression)
|
||||
.await
|
||||
.map_err(with_output_context)?;
|
||||
fetch_dump::render_page_output_async(&mut page, &config.fetch).await
|
||||
} else {
|
||||
fetch_dump::render_page_output_async(&mut page, &config.fetch).await?
|
||||
};
|
||||
stdout
|
||||
.write_all(&rendered)
|
||||
.context("failed to write fetch output")?;
|
||||
stdout
|
||||
.flush()
|
||||
.context("failed to flush fetch output")?;
|
||||
eval_output::evaluate(&mut page, expression).await
|
||||
}
|
||||
} else {
|
||||
fetch_dump::render_page_output_async(&mut page, &config.fetch).await
|
||||
}
|
||||
Ok::<(), anyhow::Error>(())
|
||||
.map_err(with_output_context)?;
|
||||
stdout
|
||||
.write_all(&rendered)
|
||||
.context("failed to write fetch output")
|
||||
.map_err(with_output_context)?;
|
||||
stdout
|
||||
.flush()
|
||||
.context("failed to flush fetch output")
|
||||
.map_err(with_output_context)?;
|
||||
}
|
||||
.await
|
||||
.map_err(with_output_context);
|
||||
if let Err(error) = page.close_async().await {
|
||||
tracing::warn!(error = %error, "failed to close fetched page before browser shutdown");
|
||||
}
|
||||
output_result?;
|
||||
if let Some(error) = http_failure {
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
@@ -13,12 +13,11 @@ use anyhow::{Context, Result, anyhow, bail};
|
||||
use moli_core::{
|
||||
page::{Page, SubresourceResponseWaitCriteria},
|
||||
runtime::{
|
||||
Browser, FetchDeadline, FetchTimeoutPhase, FetchedDocument, RawDocumentFetchPolicy,
|
||||
RenderedDomWaitUntil,
|
||||
Browser, FetchDeadline, FetchedDocument, RawDocumentFetchPolicy, RenderedDomWaitUntil,
|
||||
},
|
||||
};
|
||||
use moli_fetch::Request;
|
||||
use std::{future::Future, path::Path, time::Duration};
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct ReadinessPlan {
|
||||
@@ -110,31 +109,6 @@ impl ReadinessPlan {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Include work scheduled by automation predicates, delays and --eval in
|
||||
/// the explicit idle lifecycle, sharing the original readiness deadline.
|
||||
pub(super) async fn wait_for_idle(&self, browser: &Browser, page: &mut Page) -> Result<()> {
|
||||
if self.wait_until == RenderedDomWaitUntil::Idle {
|
||||
browser
|
||||
.wait_for_page_readiness_with_deadline(page, self.wait_until, self.deadline)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Idle bounds execution as well as the subsequent resource observation.
|
||||
/// Other modes retain their existing delay and evaluation policy.
|
||||
pub(super) async fn within_idle_deadline<T>(
|
||||
&self,
|
||||
phase: FetchTimeoutPhase,
|
||||
operation: impl Future<Output = Result<T>>,
|
||||
) -> Result<T> {
|
||||
if self.wait_until == RenderedDomWaitUntil::Idle {
|
||||
self.deadline.wait(phase, operation).await
|
||||
} else {
|
||||
operation.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rendered_wait_until(wait_until: FetchWaitUntil) -> RenderedDomWaitUntil {
|
||||
|
||||
+6
-6
@@ -274,10 +274,9 @@ pub struct FetchArgs {
|
||||
|
||||
/// Maximum total readiness time in milliseconds. Initial and HTTP-error
|
||||
/// replacement navigations, the selected lifecycle stage, response match,
|
||||
/// selector, and script waits share one absolute deadline. Idle also bounds
|
||||
/// --delay-ms and --eval/--eval-file. Network-idle and DOM-stable return the
|
||||
/// current page when it expires; idle fails with a timeout error. Pass
|
||||
/// `--log-level warn` to see timeout warnings.
|
||||
/// selector, and script waits share one absolute deadline. Network-idle and
|
||||
/// DOM-stable return the current page when it expires; idle fails with a
|
||||
/// timeout error. Pass `--log-level warn` to see timeout warnings.
|
||||
#[arg(
|
||||
short,
|
||||
long = "timeout-ms",
|
||||
@@ -766,8 +765,9 @@ pub enum FetchWaitUntil {
|
||||
Load,
|
||||
NetworkIdle,
|
||||
DomStable,
|
||||
/// Wait for user async work and callbacks to finish. Active intervals,
|
||||
/// Workers, WebSockets and receiving MessagePorts keep the wait pending.
|
||||
/// Wait for initial page async work and callbacks to finish. Active
|
||||
/// intervals, Workers, WebSockets and receiving MessagePorts keep the wait
|
||||
/// pending. Later script and delay options do not repeat this wait.
|
||||
Idle,
|
||||
Done,
|
||||
}
|
||||
|
||||
+58
-102
@@ -7,7 +7,7 @@ use axum::{
|
||||
routing::get,
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use std::{process::Command, time::Duration};
|
||||
use std::time::Duration;
|
||||
|
||||
fn document(script: &str) -> String {
|
||||
format!(
|
||||
@@ -423,30 +423,29 @@ fn idle_has_a_strict_timeout_for_live_workers_and_websockets() -> Result<()> {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_waits_for_work_started_by_eval_before_exporting() -> Result<()> {
|
||||
let script =
|
||||
"setTimeout(() => document.querySelector('main').textContent = 'done', 100); 'value'";
|
||||
fn idle_does_not_repeat_after_eval() -> Result<()> {
|
||||
let script = "setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'; 'value'";
|
||||
assert_done(run(
|
||||
&document(""),
|
||||
"3000",
|
||||
"1000",
|
||||
&["--dump", "html", "--eval", script],
|
||||
)?);
|
||||
let output = run(&document(""), "3000", &["--eval", script])?;
|
||||
let output = run(&document(""), "1000", &["--eval", script])?;
|
||||
assert!(output.status.success(), "{}", clean_output(&output.stderr));
|
||||
assert_eq!(clean_output(&output.stdout), "value\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_waits_for_work_started_by_eval_file() -> Result<()> {
|
||||
fn idle_does_not_repeat_after_eval_file() -> Result<()> {
|
||||
let path = unique_temp_file_path("wait-idle", "eval.js")?;
|
||||
std::fs::write(
|
||||
&path,
|
||||
"setTimeout(() => document.querySelector('main').textContent = 'done', 100)",
|
||||
"setInterval(() => {}, 60000); document.querySelector('main').textContent = 'done'",
|
||||
)?;
|
||||
let output = run(
|
||||
&document(""),
|
||||
"3000",
|
||||
"1000",
|
||||
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
|
||||
);
|
||||
std::fs::remove_file(path)?;
|
||||
@@ -455,15 +454,15 @@ fn idle_waits_for_work_started_by_eval_file() -> Result<()> {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_also_drains_work_created_by_the_readiness_script() -> Result<()> {
|
||||
fn idle_does_not_repeat_after_readiness_script() -> Result<()> {
|
||||
assert_done(run(
|
||||
&document(""),
|
||||
"3000",
|
||||
"1000",
|
||||
&[
|
||||
"--dump",
|
||||
"html",
|
||||
"--wait-script",
|
||||
"(setTimeout(() => document.querySelector('main').textContent='done', 100), true)",
|
||||
"(setInterval(() => {}, 60000), document.querySelector('main').textContent='done', true)",
|
||||
],
|
||||
)?);
|
||||
Ok(())
|
||||
@@ -488,106 +487,63 @@ fn idle_waits_for_child_window_timers_and_releases_removed_windows() -> Result<(
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_keeps_one_deadline_for_work_started_after_initial_readiness() -> Result<()> {
|
||||
let output = run(
|
||||
&document("setTimeout(() => {}, 300)"),
|
||||
"1000",
|
||||
&["--dump", "html", "--eval", "setTimeout(() => {}, 800)"],
|
||||
)?;
|
||||
assert!(!output.status.success());
|
||||
assert!(output.stdout.is_empty());
|
||||
assert!(
|
||||
clean_output(&output.stderr).contains("timed out after 1000 ms while waiting for idle"),
|
||||
"{}",
|
||||
clean_output(&output.stderr)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_idle_execution_times_out(extra: &[&str], phase: &str) -> Result<()> {
|
||||
let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));
|
||||
command
|
||||
.args([
|
||||
"fetch",
|
||||
"--wait",
|
||||
"idle",
|
||||
"--timeout-ms",
|
||||
"1000",
|
||||
"--http-no-proxy",
|
||||
"*",
|
||||
"--log-level",
|
||||
"error",
|
||||
])
|
||||
.args(extra)
|
||||
.arg(document(""));
|
||||
// Also bound teardown: dropping only the caller future would leave an
|
||||
// awaited Promise in the renderer and hang browser shutdown.
|
||||
let output =
|
||||
super::subprocess::run_command_with_process_timeout(command, Duration::from_secs(5))?;
|
||||
let stderr = clean_output(&output.stderr);
|
||||
assert!(
|
||||
!output.status.success(),
|
||||
"stdout={}",
|
||||
clean_output(&output.stdout)
|
||||
);
|
||||
assert!(output.stdout.is_empty());
|
||||
assert!(
|
||||
stderr.contains(&format!("timed out after 1000 ms while {phase}")),
|
||||
"{stderr}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_deadline_interrupts_async_eval_and_acknowledges_page_cleanup() -> Result<()> {
|
||||
for expression in [
|
||||
"new Promise(resolve => setTimeout(() => resolve('late'), 10000))",
|
||||
"new Promise(() => {})",
|
||||
] {
|
||||
assert_idle_execution_times_out(&["--eval", expression], "evaluating JavaScript")?;
|
||||
assert_idle_execution_times_out(
|
||||
&["--dump", "html", "--eval", expression],
|
||||
"evaluating JavaScript",
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_deadline_interrupts_eval_file() -> Result<()> {
|
||||
let path = unique_temp_file_path("idle-eval-timeout", "eval.js")?;
|
||||
std::fs::write(&path, "new Promise(resolve => setTimeout(resolve, 10000))")?;
|
||||
let result = assert_idle_execution_times_out(
|
||||
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
|
||||
"evaluating JavaScript",
|
||||
);
|
||||
std::fs::remove_file(path)?;
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_deadline_interrupts_page_delay() -> Result<()> {
|
||||
assert_idle_execution_times_out(
|
||||
&["--dump", "html", "--delay-ms", "10000"],
|
||||
"waiting for page delay",
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_waits_for_async_eval_and_followup_work_that_fit_the_deadline() -> Result<()> {
|
||||
fn idle_does_not_repeat_after_page_delay() -> Result<()> {
|
||||
assert_done(run(
|
||||
&document(""),
|
||||
"3000",
|
||||
"1000",
|
||||
&[
|
||||
"--dump",
|
||||
"html",
|
||||
"--eval",
|
||||
"new Promise(resolve => setTimeout(() => { setTimeout(() => document.querySelector('main').textContent = 'done', 100); resolve('value'); }, 100))",
|
||||
"--wait-script",
|
||||
"(setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; }, 25), true)",
|
||||
"--delay-ms",
|
||||
"100",
|
||||
],
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_keeps_async_eval_outside_readiness_deadline() -> Result<()> {
|
||||
let expression = "new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve('value'); }, 1200))";
|
||||
assert_done(run(
|
||||
&document(""),
|
||||
"1000",
|
||||
&["--dump", "html", "--eval", expression],
|
||||
)?);
|
||||
let output = run(&document(""), "1000", &["--eval", expression])?;
|
||||
assert!(output.status.success(), "{}", clean_output(&output.stderr));
|
||||
assert_eq!(clean_output(&output.stdout), "value\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_keeps_async_eval_file_outside_readiness_deadline() -> Result<()> {
|
||||
let path = unique_temp_file_path("idle-async-eval", "eval.js")?;
|
||||
std::fs::write(
|
||||
&path,
|
||||
"new Promise(resolve => setTimeout(() => { setInterval(() => {}, 60000); document.querySelector('main').textContent='done'; resolve(); }, 1200))",
|
||||
)?;
|
||||
let output = run(
|
||||
&document(""),
|
||||
"1000",
|
||||
&["--dump", "html", "--eval-file", path.to_str().unwrap()],
|
||||
);
|
||||
std::fs::remove_file(path)?;
|
||||
assert_done(output?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_keeps_page_delay_outside_readiness_deadline() -> Result<()> {
|
||||
assert_done(run(
|
||||
&document("document.querySelector('main').textContent='done'"),
|
||||
"1000",
|
||||
&["--dump", "html", "--delay-ms", "1200"],
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_follows_navigation_and_keeps_waiting_on_the_replacement_document() -> Result<()> {
|
||||
let server = Server::new("setTimeout(() => location.href = '/next', 40)")?;
|
||||
|
||||
Reference in New Issue
Block a user