diff --git a/moli-benchmark/moli_benchmark/wpt_cross/runner.py b/moli-benchmark/moli_benchmark/wpt_cross/runner.py index 1d84c7e9ab..151937dd2f 100644 --- a/moli-benchmark/moli_benchmark/wpt_cross/runner.py +++ b/moli-benchmark/moli_benchmark/wpt_cross/runner.py @@ -610,7 +610,11 @@ async def _run_one_case( ) response, eval_seen = await client.recv_until_id( eval_id, - timeout=max(0.01, min(5.0, deadline - time.perf_counter())), + # A synchronous test can keep the renderer busy for longer than + # an ordinary CDP command timeout. Once the probe is in flight, + # let it use the case's remaining budget instead of turning a + # progressing test into an infrastructure error after 5 seconds. + timeout=max(0.01, deadline - time.perf_counter()), ) seen_messages.extend(eval_seen) except (RawCdpError, asyncio.TimeoutError) as error: diff --git a/moli-benchmark/tests/test_wpt_cross.py b/moli-benchmark/tests/test_wpt_cross.py index 59279a577a..594dc05d76 100644 --- a/moli-benchmark/tests/test_wpt_cross.py +++ b/moli-benchmark/tests/test_wpt_cross.py @@ -442,6 +442,94 @@ class WptCrossTests(unittest.TestCase): 2, ) + def test_run_one_case_gives_in_flight_probe_the_remaining_case_budget(self) -> None: + class FakeClient: + def __init__(self) -> None: + self.commands: list[tuple[str, dict | None, str | None]] = [] + self.probe_timeout: float | None = None + + async def send( + self, + method: str, + params: dict | None = None, + *, + session_id: str | None = None, + ) -> int: + self.commands.append((method, params, session_id)) + return len(self.commands) + + async def recv_until_id( + self, command_id: int, *, timeout: float + ) -> tuple[dict, list[dict]]: + method = self.commands[command_id - 1][0] + if method == "Page.navigate": + return ( + { + "sessionId": "SESSION-1", + "result": { + "frameId": "FRAME-1", + "loaderId": "LOADER-1", + }, + }, + [ + { + "method": "Page.frameNavigated", + "sessionId": "SESSION-1", + "params": { + "frame": { + "id": "FRAME-1", + "loaderId": "LOADER-1", + "url": "http://localhost/slow.html", + } + }, + } + ], + ) + + self.probe_timeout = timeout + # Model a renderer-bound command that needs more than the old + # five-second receive cap without making this test actually wait. + if timeout <= 5.0: + raise asyncio.TimeoutError() + return ( + { + "result": { + "result": { + "value": { + "href": "http://localhost/slow.html", + "casePath": "/slow.html", + "bridgeInstalled": True, + "payload": { + "case_path": "/slow.html", + "source": "completion-callback", + "harness": {"status": 0}, + "tests": [ + {"name": "slow result", "status": 0} + ], + }, + } + } + } + }, + [], + ) + + client = FakeClient() + result = asyncio.run( + _run_one_case( + client=client, # type: ignore[arg-type] + session_id="SESSION-1", + case_path="slow.html", + url="http://localhost/slow.html", + timeout_seconds=30.0, + ) + ) + + self.assertEqual(result.status, "pass") + self.assertEqual(result.subtests_pass, 1) + self.assertIsNotNone(client.probe_timeout) + self.assertGreater(client.probe_timeout or 0.0, 25.0) + def test_close_page_disposes_context_with_all_auxiliary_targets(self) -> None: class FakeClient: def __init__(self) -> None: