From 9c202716ee0f6fe83f0f270c14be61459ca4e43f Mon Sep 17 00:00:00 2001 From: ldm0 Date: Mon, 7 Sep 2026 16:46:46 +0800 Subject: [PATCH] fix(ci): stabilize frontend reference observations --- .../scripts/render-ci-regression-comment.cjs | 21 +- .../render-ci-regression-comment.test.cjs | 15 +- .github/workflows/ci.yml | 1 + moli-frontend-smoke/README.md | 5 + .../moli_frontend_smoke/analysis.py | 47 ++++- .../moli_frontend_smoke/cdp.py | 74 +++++-- .../moli_frontend_smoke/models.py | 13 ++ .../moli_frontend_smoke/runner.py | 171 +++++++++++++--- moli-frontend-smoke/tests/test_analysis.py | 35 ++++ moli-frontend-smoke/tests/test_cdp.py | 118 +++++++++++ moli-frontend-smoke/tests/test_runner.py | 190 ++++++++++++++++++ 11 files changed, 639 insertions(+), 51 deletions(-) diff --git a/.github/scripts/render-ci-regression-comment.cjs b/.github/scripts/render-ci-regression-comment.cjs index c6a12d7b4e..5e06dcf1cb 100644 --- a/.github/scripts/render-ci-regression-comment.cjs +++ b/.github/scripts/render-ci-regression-comment.cjs @@ -355,9 +355,10 @@ function frontendOverview(section) { const total = summary.results.length; const matches = (count(summary.counts.match) ?? 0) + (count(summary.counts.reference_ok) ?? 0); const problems = Math.max(0, total - matches); + const recoveries = count(objectAt(summary, 'referenceGate').recoveredCases) ?? 0; return { status: summary.ok === true && problems === 0 ? 'โœ…' : 'โŒ', - signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues`, + signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues${recoveries ? `; ${formatInteger(recoveries)} Chromium reference recoveries` : ''}`, }; } @@ -375,6 +376,24 @@ function renderFrontend(section) { '| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', `| ${formatInteger(count(summary.counts.match) ?? 0)} | ${formatInteger(count(summary.counts.dom_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.diagnostic_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.moli_error) ?? 0)} | ${formatInteger(count(summary.counts.reference_error) ?? 0)} | ${formatInteger(count(summary.counts.infrastructure_error) ?? 0)} | ${formatInteger(timeline.mismatchedFrames)} | ${formatDurationMilliseconds(summary.durationMs)} |` ); + const recoveredReferences = summary.results + .filter((result) => { + const chromium = objectAt(result, 'chromium'); + return Array.isArray(chromium.previous_failures) && chromium.previous_failures.length !== 0; + }) + .slice(0, MAX_DETAIL_ROWS); + if (recoveredReferences.length !== 0) { + lines.push('', '**Recovered Chromium reference infrastructure failures**', ''); + for (const result of recoveredReferences) { + const chromium = objectAt(result, 'chromium'); + const failures = chromium.previous_failures; + const first = isObject(failures[0]) ? failures[0] : {}; + const error = typeof first.error === 'string' ? first.error : 'unknown'; + lines.push( + `- ${code(result.id, 160)} โ€” ${formatInteger(failures.length)} failed attempt(s) before recovery ยท ${code(error, 160)}` + ); + } + } const issues = summary.results .filter((result) => isObject(result) && !['match', 'reference_ok'].includes(result.status)) .slice(0, MAX_DETAIL_ROWS); diff --git a/.github/scripts/render-ci-regression-comment.test.cjs b/.github/scripts/render-ci-regression-comment.test.cjs index a24adc11b7..4ef1765c35 100644 --- a/.github/scripts/render-ci-regression-comment.test.cjs +++ b/.github/scripts/render-ci-regression-comment.test.cjs @@ -116,11 +116,21 @@ function createArtifacts(root) { const frontendRoot = path.join(root, 'frontend'); writeJson(frontendRoot, 'summary.json', { ok: false, + referenceGate: { recoveredCases: 1 }, counts: { match: 1, dom_mismatch: 1 }, durationMs: 1_250, timeline: { chromiumFrames: 5, moliFrames: 5, mismatchedFrames: 1 }, results: [ - { id: 'safe', status: 'match', mismatchedFrames: [] }, + { + id: 'safe', + status: 'match', + mismatchedFrames: [], + chromium: { + previous_failures: [ + { error: 'Target.createBrowserContext failed: unknown' }, + ], + }, + }, { id: 'bad|case<', status: 'dom_mismatch', @@ -197,6 +207,9 @@ test('renders all five trusted artifact sections into one bounded report', (t) = assert.match(report, /artifacts: `5\/5`/); assert.match(report, /Raw binary \| 100 B \| 110 B \| \+10 B \| \+10\.000000%/); assert.match(report, /Frontend differential/); + assert.match(report, /1 Chromium reference recoveries/); + assert.match(report, /Recovered Chromium reference infrastructure failures/); + assert.match(report, /Target\.createBrowserContext failed: unknown/); assert.match(report, /`bad\\\|case<`/); assert.match(report, /first difference `\$\.body\\\|<`/); assert.match(report, /frames `ready\\\|<`/); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bd7cfc65a..199b927567 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -446,6 +446,7 @@ jobs: --chromium-bin "${{ steps.chromium.outputs.binary }}" --moli-bin "$PWD/target/ci-bin/moli-release-head/moli" --timeout-ms 30000 + --reference-infrastructure-retries 1 --output target/frontend-smoke - name: Validate frontend artifacts id: analysis diff --git a/moli-frontend-smoke/README.md b/moli-frontend-smoke/README.md index 1159941045..91fcc141a5 100644 --- a/moli-frontend-smoke/README.md +++ b/moli-frontend-smoke/README.md @@ -95,6 +95,11 @@ an explicit concurrency-boundary probe. Chromium-only baseline calibration can u concurrency, but a release conclusion should also repeat the reference run and compare every frame hash across scheduling modes. +Each worker keeps one browser-level CDP connection and creates an isolated browser context per +case. CI may pass `--reference-infrastructure-retries 1` to re-observe a case after a Chromium +connection or protocol failure. The retry does not apply to fixture errors, normalization errors, +Moli failures, or DOM differences, and the original failure remains recorded in `summary.json`. + Run Chromium as a reference-only fixture gate: ```bash diff --git a/moli-frontend-smoke/moli_frontend_smoke/analysis.py b/moli-frontend-smoke/moli_frontend_smoke/analysis.py index 41093b2157..bf19470c95 100644 --- a/moli-frontend-smoke/moli_frontend_smoke/analysis.py +++ b/moli-frontend-smoke/moli_frontend_smoke/analysis.py @@ -1003,6 +1003,28 @@ def _validate_observation_contract( errors: list[str] = [] if not isinstance(observation, dict) or observation.get("ok") is not True: return [f"{prefix}: {engine} observation is not successful"], 0 + attempt_count = observation.get("attempt_count", 1) + previous_failures = observation.get("previous_failures", []) + if ( + not isinstance(attempt_count, int) + or isinstance(attempt_count, bool) + or attempt_count < 1 + ): + errors.append(f"{prefix}: attempt count is invalid") + if not isinstance(previous_failures, list): + errors.append(f"{prefix}: previous failures is not a list") + elif isinstance(attempt_count, int) and not isinstance(attempt_count, bool): + if attempt_count != len(previous_failures) + 1: + errors.append(f"{prefix}: attempt count does not match previous failures") + for index, failure in enumerate(previous_failures, start=1): + if ( + not isinstance(failure, dict) + or failure.get("attempt") != index + or failure.get("kind") != "infrastructure" + ): + errors.append( + f"{prefix}: previous failure[{index - 1}] is invalid" + ) if not _is_nonempty_string(observation.get("dom_hash")): errors.append(f"{prefix}: final DOM hash is missing") if not _is_node_count(observation.get("node_count")): @@ -1133,6 +1155,8 @@ def _validate_reference_summary( ids: set[str] = set() frame_total = 0 + retried_cases = 0 + recovered_cases = 0 for position, result in enumerate(results): case_id = result.get("id") prefix = f"{label}: result[{position}]" @@ -1151,15 +1175,36 @@ def _validate_reference_summary( None, }: errors.append(f"{prefix}: status invalidates the Chromium reference phase") + chromium_observation = result.get("chromium") + if isinstance(chromium_observation, dict): + attempt_count = chromium_observation.get("attempt_count", 1) + if ( + isinstance(attempt_count, int) + and not isinstance(attempt_count, bool) + and attempt_count > 1 + ): + retried_cases += 1 + previous_failures = chromium_observation.get("previous_failures", []) + if ( + chromium_observation.get("ok") is True + and isinstance(previous_failures, list) + and previous_failures + ): + recovered_cases += 1 observation_errors, observation_frames = _validate_observation_contract( case_id=case_id, - observation=result.get("chromium"), + observation=chromium_observation, prefix=prefix, engine="Chromium", ) errors.extend(observation_errors) frame_total += observation_frames + if "retriedCases" in gate and gate.get("retriedCases") != retried_cases: + errors.append(f"{label}: retried case count does not match results") + if "recoveredCases" in gate and gate.get("recoveredCases") != recovered_cases: + errors.append(f"{label}: recovered case count does not match results") + counts = summary.get("counts") if mode == "reference" and counts != {"reference_ok": len(results)}: errors.append(f"{label}: strict counts do not match reference results") diff --git a/moli-frontend-smoke/moli_frontend_smoke/cdp.py b/moli-frontend-smoke/moli_frontend_smoke/cdp.py index ba43c3e325..e5492dcd3d 100644 --- a/moli-frontend-smoke/moli_frontend_smoke/cdp.py +++ b/moli-frontend-smoke/moli_frontend_smoke/cdp.py @@ -17,6 +17,10 @@ class CdpError(RuntimeError): pass +class FixtureObservationError(CdpError): + """The page or fixture violated its deterministic observation contract.""" + + class CdpCommandError(CdpError): def __init__(self, method: str, error: dict[str, Any]) -> None: super().__init__(f"{method} failed: {error}") @@ -227,13 +231,13 @@ async def _capture_document( timeout=timeout, ) if barrier.get("exceptionDetails"): - raise CdpError( + raise FixtureObservationError( "observable-tree materialization threw: " f"{barrier['exceptionDetails']}" ) materialized = (barrier.get("result") or {}).get("value") if not isinstance(materialized, int) or materialized < 1: - raise CdpError( + raise FixtureObservationError( "observable-tree materialization returned invalid element count: " f"{materialized!r}" ) @@ -325,17 +329,19 @@ def _reconcile_expected_diagnostics( ) -> None: expected_container = ready_state.get("expectedDiagnostics") or {} if not isinstance(expected_container, dict): - raise CdpError("fixture expectedDiagnostics is not an object") + raise FixtureObservationError("fixture expectedDiagnostics is not an object") expected = expected_container.get("networkFailures") or [] if not isinstance(expected, list): - raise CdpError("fixture expected network failures are not a list") + raise FixtureObservationError("fixture expected network failures are not a list") unmatched = list(diagnostics.get("networkFailures") or []) matched: list[dict[str, Any]] = [] missing: list[dict[str, Any]] = [] labels: set[str] = set() for position, item in enumerate(expected): if not isinstance(item, dict): - raise CdpError(f"expected network failure {position} is not an object") + raise FixtureObservationError( + f"expected network failure {position} is not an object" + ) label = item.get("label") url = item.get("url") resource_type = item.get("type") @@ -350,7 +356,9 @@ def _reconcile_expected_diagnostics( or not resource_type or not isinstance(canceled, bool) ): - raise CdpError(f"invalid expected network failure {position}: {item!r}") + raise FixtureObservationError( + f"invalid expected network failure {position}: {item!r}" + ) labels.add(label) match_index = next( ( @@ -407,9 +415,10 @@ async def observe_case( case: SmokeCase, url: str, timeout_ms: int, + client: RawCdpClient | None = None, ) -> EngineObservation: started = time.perf_counter() - client: RawCdpClient | None = None + owns_client = client is None context_id: str | None = None session_id: str | None = None ready_state: dict[str, Any] | None = None @@ -417,7 +426,10 @@ async def observe_case( final_root: dict[str, Any] | None = None diagnostics: dict[str, Any] = {} try: - client = await connect(endpoint) + if client is None: + client = await connect(endpoint) + else: + client.events.clear() context = await client.command("Target.createBrowserContext") context_id = context.get("browserContextId") if not isinstance(context_id, str) or not context_id: @@ -471,29 +483,37 @@ async def observe_case( timeout=command_timeout, ) if evaluation.get("exceptionDetails"): - raise CdpError( + raise FixtureObservationError( f"state Runtime.evaluate threw: {evaluation['exceptionDetails']}" ) state_value = (evaluation.get("result") or {}).get("value") if not isinstance(state_value, dict): - raise CdpError(f"fixture returned invalid state: {state_value!r}") + raise FixtureObservationError( + f"fixture returned invalid state: {state_value!r}" + ) ready_state = state_value if ready_state.get("id") != case.id: - raise CdpError(f"fixture state id mismatch: {ready_state!r}") + raise FixtureObservationError( + f"fixture state id mismatch: {ready_state!r}" + ) phase = ready_state.get("phase") if phase == "checkpoint": pending = ready_state.get("pendingFrame") if not isinstance(pending, dict): - raise CdpError(f"checkpoint has no pending frame: {ready_state!r}") + raise FixtureObservationError( + f"checkpoint has no pending frame: {ready_state!r}" + ) index = pending.get("index") name = pending.get("name") token = pending.get("token") if index != len(frames) or not isinstance(name, str) or not isinstance(token, str): - raise CdpError( + raise FixtureObservationError( f"invalid checkpoint sequence at frame {len(frames)}: {pending!r}" ) if len(frames) >= 32: - raise CdpError("fixture exceeded the 32-frame safety limit") + raise FixtureObservationError( + "fixture exceeded the 32-frame safety limit" + ) root = await _capture_document( client, session_id=session_id, @@ -512,21 +532,27 @@ async def observe_case( timeout=command_timeout, ) if resumed.get("exceptionDetails"): - raise CdpError( + raise FixtureObservationError( f"frame resume Runtime.evaluate threw: {resumed['exceptionDetails']}" ) if (resumed.get("result") or {}).get("value") is not True: - raise CdpError(f"fixture refused frame resume token {token!r}") + raise FixtureObservationError( + f"fixture refused frame resume token {token!r}" + ) after_token = token continue if phase != "ready": - raise CdpError(f"fixture did not become ready: {ready_state!r}") + raise FixtureObservationError( + f"fixture did not become ready: {ready_state!r}" + ) if ready_state.get("errors"): - raise CdpError(f"fixture reported errors: {ready_state!r}") + raise FixtureObservationError( + f"fixture reported errors: {ready_state!r}" + ) reported_frames = ready_state.get("frames") observed_names = [frame.name for frame in frames] if reported_frames != observed_names: - raise CdpError( + raise FixtureObservationError( "fixture frame history mismatch: " f"reported {reported_frames!r}, observed {observed_names!r}" ) @@ -548,7 +574,7 @@ async def observe_case( diagnostics = _diagnostics(client.events, session_id) _reconcile_expected_diagnostics(diagnostics, ready_state) if _diagnostics_have_errors(diagnostics): - raise CdpError( + raise FixtureObservationError( "fixture emitted browser diagnostics: " + json.dumps(diagnostics, ensure_ascii=False, separators=(",", ":")) ) @@ -574,6 +600,11 @@ async def observe_case( diagnostics=diagnostics, error_type=type(error).__name__, error=str(error), + failure_kind=( + "fixture" + if isinstance(error, FixtureObservationError) + else "infrastructure" + ), ) finally: if client is not None: @@ -586,4 +617,5 @@ async def observe_case( ) except Exception: pass - await client.websocket.close() + if owns_client: + await client.websocket.close() diff --git a/moli-frontend-smoke/moli_frontend_smoke/models.py b/moli-frontend-smoke/moli_frontend_smoke/models.py index 38a449a68e..fa0ca1bcc6 100644 --- a/moli-frontend-smoke/moli_frontend_smoke/models.py +++ b/moli-frontend-smoke/moli_frontend_smoke/models.py @@ -6,6 +6,7 @@ from typing import Any, Literal FrameworkName = Literal["react", "vue", "angular"] Complexity = Literal["simple", "medium", "complex"] +ObservationFailureKind = Literal["infrastructure", "fixture", "normalization"] CaseStatus = Literal[ "match", "reference_ok", @@ -61,6 +62,15 @@ class DomFrameObservation: return value +@dataclass(frozen=True) +class ObservationFailure: + attempt: int + kind: ObservationFailureKind + duration_ms: float + error_type: str | None + error: str | None + + @dataclass class EngineObservation: engine: str @@ -74,6 +84,9 @@ class EngineObservation: diagnostics: dict[str, Any] = field(default_factory=dict) error_type: str | None = None error: str | None = None + failure_kind: ObservationFailureKind | None = None + attempt_count: int = 1 + previous_failures: list[ObservationFailure] = field(default_factory=list) def summary_json(self) -> dict[str, Any]: value = asdict(self) diff --git a/moli-frontend-smoke/moli_frontend_smoke/runner.py b/moli-frontend-smoke/moli_frontend_smoke/runner.py index 9b8dc8afa6..b9221d3582 100644 --- a/moli-frontend-smoke/moli_frontend_smoke/runner.py +++ b/moli-frontend-smoke/moli_frontend_smoke/runner.py @@ -21,7 +21,7 @@ from .browsers import ( start_moli, wait_for_cdp_endpoint, ) -from .cdp import observe_case +from .cdp import RawCdpClient, connect, observe_case from .config import ( PROJECT_ROOT, REPO_ROOT, @@ -33,7 +33,7 @@ from .config import ( from .dom import dom_hash, first_difference, iter_nodes, normalize_dom_node, unified_dom_diff from .fixture_server import FixtureServer from .manifest import SmokeManifest, load_manifest, select_cases -from .models import CaseResult, EngineObservation, SmokeCase +from .models import CaseResult, EngineObservation, ObservationFailure, SmokeCase def _split_values(values: Iterable[str]) -> set[str]: @@ -257,6 +257,25 @@ def _write_failure_artifact( return str(case_dir.relative_to(output_dir)) +async def _close_cdp_client(client: RawCdpClient | None) -> None: + if client is None: + return + try: + await client.websocket.close() + except Exception: + pass + + +def _attempt_failure(attempt: int, observation: EngineObservation) -> ObservationFailure: + return ObservationFailure( + attempt=attempt, + kind=observation.failure_kind or "infrastructure", + duration_ms=observation.duration_ms, + error_type=observation.error_type, + error=observation.error, + ) + + async def _observe_phase_case( *, engine: str, @@ -264,33 +283,90 @@ async def _observe_phase_case( case: SmokeCase, fixture_url: str, timeout_ms: int, - semaphore: asyncio.Semaphore, + client: RawCdpClient | None, + infrastructure_retries: int, progress: dict[str, int], total: int, -) -> EngineObservation: - async with semaphore: - url = fixture_url.rstrip("/") + case.path - observation = await observe_case( - engine=engine, - endpoint=endpoint, - case=case, - url=url, - timeout_ms=timeout_ms, - ) +) -> tuple[EngineObservation, RawCdpClient | None]: + url = fixture_url.rstrip("/") + case.path + previous_failures: list[ObservationFailure] = [] + attempt = 0 + while True: + attempt += 1 + if client is None: + connect_started = time.perf_counter() + try: + client = await connect(endpoint) + except Exception as error: + observation = EngineObservation( + engine=engine, + ok=False, + duration_ms=(time.perf_counter() - connect_started) * 1000, + error_type=type(error).__name__, + error=f"CDP connection failed: {error}", + failure_kind="infrastructure", + ) + else: + observation = await observe_case( + engine=engine, + endpoint=endpoint, + case=case, + url=url, + timeout_ms=timeout_ms, + client=client, + ) + else: + observation = await observe_case( + engine=engine, + endpoint=endpoint, + case=case, + url=url, + timeout_ms=timeout_ms, + client=client, + ) try: _normalize_observation(observation) except Exception as error: observation.ok = False observation.error_type = type(error).__name__ observation.error = f"DOM normalization failed: {error}" - progress["completed"] += 1 - state = "ok" if observation.ok else "error" + observation.failure_kind = "normalization" + + if observation.failure_kind == "infrastructure": + await _close_cdp_client(client) + client = None + + should_retry = ( + engine == "chromium" + and observation.failure_kind == "infrastructure" + and attempt <= infrastructure_retries + ) + if not should_retry: + break + previous_failures.append(_attempt_failure(attempt, observation)) + error = (observation.error or observation.error_type or "unknown").replace( + "\n", " " + ) print( - f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}", + f"[{engine} retry {attempt}/{infrastructure_retries}] " + f"{case.id}: {error[:240]}", file=sys.stderr, flush=True, ) - return observation + + observation.duration_ms += sum( + failure.duration_ms for failure in previous_failures + ) + observation.attempt_count = attempt + observation.previous_failures = previous_failures + progress["completed"] += 1 + state = "ok" if observation.ok else "error" + print( + f"[{engine} {progress['completed']:>3}/{total}] {state:<5} {case.id}", + file=sys.stderr, + flush=True, + ) + return observation, client async def _observe_phase( @@ -301,27 +377,43 @@ async def _observe_phase( fixture_url: str, timeout_ms: int, jobs: int, + infrastructure_retries: int = 0, ) -> dict[str, EngineObservation]: - semaphore = asyncio.Semaphore(jobs) + if not cases: + return {} + queue: asyncio.Queue[SmokeCase] = asyncio.Queue() + for case in cases: + queue.put_nowait(case) progress = {"completed": 0} - observations = await asyncio.gather( - *[ - asyncio.create_task( - _observe_phase_case( + observations: dict[str, EngineObservation] = {} + + async def worker() -> None: + client: RawCdpClient | None = None + try: + while True: + try: + case = queue.get_nowait() + except asyncio.QueueEmpty: + return + observation, client = await _observe_phase_case( engine=engine, endpoint=endpoint, case=case, fixture_url=fixture_url, timeout_ms=timeout_ms, - semaphore=semaphore, + client=client, + infrastructure_retries=infrastructure_retries, progress=progress, total=len(cases), ) - ) - for case in cases - ] + observations[case.id] = observation + finally: + await _close_cdp_client(client) + + await asyncio.gather( + *(asyncio.create_task(worker()) for _ in range(min(jobs, len(cases)))) ) - return {case.id: observation for case, observation in zip(cases, observations, strict=True)} + return {case.id: observations[case.id] for case in cases} def _case_result( @@ -402,6 +494,9 @@ def _default_output_dir() -> Path: async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + reference_infrastructure_retries = getattr( + args, "reference_infrastructure_retries", 0 + ) manifest_path = Path(args.manifest).expanduser().resolve() manifest = load_manifest(manifest_path) if not args.allow_partial_manifest: @@ -452,6 +547,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: fixture_url=fixture.url, timeout_ms=args.timeout_ms, jobs=args.jobs, + infrastructure_retries=reference_infrastructure_retries, ) reference_gate_ok = all( observation.ok for observation in chromium_observations.values() @@ -479,6 +575,7 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: fixture_url=fixture.url, timeout_ms=args.timeout_ms, jobs=args.jobs, + infrastructure_retries=0, ) effective_reference_only = args.reference_only or not reference_gate_ok @@ -531,6 +628,15 @@ async def run(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: 1 for observation in chromium_observations.values() if not observation.ok ), "moliPhaseStarted": bool(moli_observations), + "infrastructureRetryLimit": reference_infrastructure_retries, + "retriedCases": sum( + observation.attempt_count > 1 + for observation in chromium_observations.values() + ), + "recoveredCases": sum( + observation.ok and bool(observation.previous_failures) + for observation in chromium_observations.values() + ), }, "fixture": fixture.url, "engines": { @@ -622,6 +728,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=15_000, help="Per-page fixture ready timeout.", ) + parser.add_argument( + "--reference-infrastructure-retries", + type=int, + default=0, + help=( + "Retry only Chromium CDP/infrastructure observation failures; fixture, " + "normalization, and Moli failures remain single-attempt gates." + ), + ) parser.add_argument("--output", help="Artifact output directory.") parser.add_argument("--chromium-bin", help="Chromium executable.") parser.add_argument("--moli-bin", help="Moli executable.") @@ -643,6 +758,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--jobs must be between 1 and 32") if args.timeout_ms < 100 or args.timeout_ms > 120_000: parser.error("--timeout-ms must be between 100 and 120000") + if not 0 <= args.reference_infrastructure_retries <= 3: + parser.error("--reference-infrastructure-retries must be between 0 and 3") if args.reference_only and args.moli_endpoint: parser.error("--reference-only cannot be combined with --moli-endpoint") return args diff --git a/moli-frontend-smoke/tests/test_analysis.py b/moli-frontend-smoke/tests/test_analysis.py index 526b2ab73d..d67e86d424 100644 --- a/moli-frontend-smoke/tests/test_analysis.py +++ b/moli-frontend-smoke/tests/test_analysis.py @@ -138,6 +138,41 @@ def test_reference_stability_ignores_endpoint_but_requires_every_frame_hash() -> ] +def test_reference_contract_preserves_recovered_infrastructure_failures() -> None: + first = _reference_summary() + first["referenceGate"].update( # type: ignore[union-attr] + { + "infrastructureRetryLimit": 1, + "retriedCases": 1, + "recoveredCases": 1, + } + ) + first["results"][0]["chromium"].update( # type: ignore[index,union-attr] + { + "attempt_count": 2, + "previous_failures": [ + { + "attempt": 1, + "kind": "infrastructure", + "duration_ms": 1.0, + "error_type": "CdpCommandError", + "error": "Target.createBrowserContext failed: unknown", + } + ], + } + ) + second = deepcopy(first) + + assert compare_reference_data(first, second)["ok"] # type: ignore[arg-type] + + second["referenceGate"]["recoveredCases"] = 0 # type: ignore[index] + result = compare_reference_data(first, second) # type: ignore[arg-type] + assert not result["ok"] + assert result["secondValidationErrors"] == [ + "second: recovered case count does not match results" + ] + + def test_projection_is_explicitly_non_gating_and_can_isolate_ua_edges() -> None: case = {"id": "react/family/case", "framework": "react"} chromium = { diff --git a/moli-frontend-smoke/tests/test_cdp.py b/moli-frontend-smoke/tests/test_cdp.py index 796186103d..a55a133c34 100644 --- a/moli-frontend-smoke/tests/test_cdp.py +++ b/moli-frontend-smoke/tests/test_cdp.py @@ -2,7 +2,11 @@ from __future__ import annotations import asyncio +import pytest + from moli_frontend_smoke.cdp import ( + CdpCommandError, + FixtureObservationError, _DOM_ENABLE_PARAMS, _OBSERVABLE_TREE_BARRIER_EXPRESSION, _capture_document, @@ -11,7 +15,106 @@ from moli_frontend_smoke.cdp import ( _reconcile_expected_diagnostics, _resume_expression, _state_expression, + observe_case, ) +from moli_frontend_smoke.models import SmokeCase + + +def _case() -> SmokeCase: + return SmokeCase( + id="vue/web-platform-integration/formdata-searchparams-pipeline", + framework="vue", + family="web-platform-integration", + complexity="complex", + slug="formdata-searchparams-pipeline", + title="FormData and URLSearchParams pipeline", + variant=0, + seed=1, + size=4, + path="/cases/vue/web-platform-integration/formdata-searchparams-pipeline/index.html", + ) + + +class _FakeWebSocket: + def __init__(self) -> None: + self.close_count = 0 + + async def close(self) -> None: + self.close_count += 1 + + +class _FixtureFailureClient: + def __init__(self) -> None: + self.websocket = _FakeWebSocket() + self.events: list[dict[str, object]] = [] + self.methods: list[str] = [] + + async def command(self, method, params=None, **_kwargs): + self.methods.append(method) + if method == "Target.createBrowserContext": + return {"browserContextId": "context"} + if method == "Target.createTarget": + return {"targetId": "target"} + if method == "Target.attachToTarget": + return {"sessionId": "session"} + if method == "Runtime.evaluate": + return {"exceptionDetails": {"text": "fixture failed"}} + return {} + + async def wait_event(self, *_args, **_kwargs): + return {} + + +def test_observe_case_distinguishes_fixture_failures_from_infrastructure() -> None: + client = _FixtureFailureClient() + + observation = asyncio.run( + observe_case( + engine="chromium", + endpoint="http://127.0.0.1:9222", + case=_case(), + url="http://127.0.0.1:3000/case", + timeout_ms=1000, + client=client, # type: ignore[arg-type] + ) + ) + + assert not observation.ok + assert observation.failure_kind == "fixture" + assert observation.error_type == "FixtureObservationError" + assert "state Runtime.evaluate threw" in (observation.error or "") + assert client.methods[-1] == "Target.disposeBrowserContext" + assert client.websocket.close_count == 0 + + +class _InfrastructureFailureClient: + def __init__(self) -> None: + self.websocket = _FakeWebSocket() + self.events: list[dict[str, object]] = [] + + async def command(self, method, _params=None, **_kwargs): + raise CdpCommandError(method, {"code": -32000, "message": "unknown"}) + + +def test_observe_case_marks_cdp_command_errors_as_infrastructure() -> None: + client = _InfrastructureFailureClient() + + observation = asyncio.run( + observe_case( + engine="chromium", + endpoint="http://127.0.0.1:9222", + case=_case(), + url="http://127.0.0.1:3000/case", + timeout_ms=1000, + client=client, # type: ignore[arg-type] + ) + ) + + assert not observation.ok + assert observation.failure_kind == "infrastructure" + assert observation.error_type == "CdpCommandError" + assert "unknown" in (observation.error or "") + assert client.websocket.close_count == 0 def test_diagnostics_preserve_runtime_console_and_network_failures() -> None: @@ -222,6 +325,21 @@ def test_missing_expected_network_failure_remains_a_diagnostic_error() -> None: assert _diagnostics_have_errors(diagnostics) +def test_invalid_expected_diagnostics_is_a_fixture_failure() -> None: + diagnostics = { + "exceptions": [], + "consoleErrors": [], + "networkFailures": [], + "httpErrors": [], + } + + with pytest.raises(FixtureObservationError): + _reconcile_expected_diagnostics( + diagnostics, + {"expectedDiagnostics": {"networkFailures": "invalid"}}, + ) + + def test_state_expression_waits_for_a_new_checkpoint_or_terminal_state() -> None: expression = _state_expression("react/family/case", 12_345, "old-token") diff --git a/moli-frontend-smoke/tests/test_runner.py b/moli-frontend-smoke/tests/test_runner.py index 9473e95c01..49db3f1632 100644 --- a/moli-frontend-smoke/tests/test_runner.py +++ b/moli-frontend-smoke/tests/test_runner.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import json from collections import deque +from dataclasses import replace from types import SimpleNamespace import moli_frontend_smoke.runner as runner @@ -14,6 +15,7 @@ from moli_frontend_smoke.models import ( from moli_frontend_smoke.runner import ( _case_result, _normalize_observation, + _observe_phase, parse_args, ) @@ -77,6 +79,191 @@ def test_default_differential_is_serial() -> None: assert parse_args([]).jobs == 1 +def test_reference_infrastructure_retry_is_opt_in() -> None: + assert parse_args([]).reference_infrastructure_retries == 0 + assert ( + parse_args(["--reference-infrastructure-retries", "1"]) + .reference_infrastructure_retries + == 1 + ) + + +class _FakeWebSocket: + def __init__(self) -> None: + self.close_count = 0 + + async def close(self) -> None: + self.close_count += 1 + + +class _FakeClient: + def __init__(self) -> None: + self.websocket = _FakeWebSocket() + self.events: list[dict[str, object]] = [] + + +def test_observation_worker_reuses_one_cdp_connection(monkeypatch) -> None: + clients: list[_FakeClient] = [] + observed_clients: list[_FakeClient] = [] + + async def fake_connect(_endpoint): + client = _FakeClient() + clients.append(client) + return client + + async def fake_observe_case(*, engine, client, **_kwargs): + observed_clients.append(client) + return _observation(engine, {"nodeType": 9, "nodeName": "#document"}) + + monkeypatch.setattr(runner, "connect", fake_connect) + monkeypatch.setattr(runner, "observe_case", fake_observe_case) + first = _case() + second = replace(first, id="react/family/second", slug="second") + + observations = asyncio.run( + _observe_phase( + engine="chromium", + endpoint="http://127.0.0.1:9222", + cases=(first, second), + fixture_url="http://127.0.0.1:3000", + timeout_ms=1000, + jobs=1, + ) + ) + + assert list(observations) == [first.id, second.id] + assert len(clients) == 1 + assert observed_clients == [clients[0], clients[0]] + assert clients[0].websocket.close_count == 1 + + +def test_chromium_retries_only_infrastructure_failures(monkeypatch) -> None: + clients: list[_FakeClient] = [] + attempts = 0 + + async def fake_connect(_endpoint): + client = _FakeClient() + clients.append(client) + return client + + async def fake_observe_case(*, engine, **_kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + return EngineObservation( + engine=engine, + ok=False, + duration_ms=2.0, + error_type="CdpCommandError", + error="Target.createBrowserContext failed: unknown", + failure_kind="infrastructure", + ) + return _observation(engine, {"nodeType": 9, "nodeName": "#document"}) + + monkeypatch.setattr(runner, "connect", fake_connect) + monkeypatch.setattr(runner, "observe_case", fake_observe_case) + + observation = asyncio.run( + _observe_phase( + engine="chromium", + endpoint="http://127.0.0.1:9222", + cases=(_case(),), + fixture_url="http://127.0.0.1:3000", + timeout_ms=1000, + jobs=1, + infrastructure_retries=1, + ) + )[_case().id] + + assert observation.ok + assert observation.attempt_count == 2 + assert observation.duration_ms == 3.0 + assert [failure.error for failure in observation.previous_failures] == [ + "Target.createBrowserContext failed: unknown" + ] + assert len(clients) == 2 + assert all(client.websocket.close_count == 1 for client in clients) + + +def test_fixture_failure_is_never_retried(monkeypatch) -> None: + attempts = 0 + + async def fake_connect(_endpoint): + return _FakeClient() + + async def fake_observe_case(*, engine, **_kwargs): + nonlocal attempts + attempts += 1 + return EngineObservation( + engine=engine, + ok=False, + duration_ms=1.0, + error_type="FixtureObservationError", + error="fixture assertion failed", + failure_kind="fixture", + ) + + monkeypatch.setattr(runner, "connect", fake_connect) + monkeypatch.setattr(runner, "observe_case", fake_observe_case) + + observation = asyncio.run( + _observe_phase( + engine="chromium", + endpoint="http://127.0.0.1:9222", + cases=(_case(),), + fixture_url="http://127.0.0.1:3000", + timeout_ms=1000, + jobs=1, + infrastructure_retries=1, + ) + )[_case().id] + + assert not observation.ok + assert observation.failure_kind == "fixture" + assert observation.attempt_count == 1 + assert observation.previous_failures == [] + assert attempts == 1 + + +def test_moli_failure_is_never_retried(monkeypatch) -> None: + attempts = 0 + + async def fake_connect(_endpoint): + return _FakeClient() + + async def fake_observe_case(*, engine, **_kwargs): + nonlocal attempts + attempts += 1 + return EngineObservation( + engine=engine, + ok=False, + duration_ms=1.0, + error_type="CdpCommandError", + error="candidate connection failed", + failure_kind="infrastructure", + ) + + monkeypatch.setattr(runner, "connect", fake_connect) + monkeypatch.setattr(runner, "observe_case", fake_observe_case) + + observation = asyncio.run( + _observe_phase( + engine="moli", + endpoint="http://127.0.0.1:9222", + cases=(_case(),), + fixture_url="http://127.0.0.1:3000", + timeout_ms=1000, + jobs=1, + infrastructure_retries=1, + ) + )[_case().id] + + assert not observation.ok + assert observation.attempt_count == 1 + assert observation.previous_failures == [] + assert attempts == 1 + + def test_any_moli_tree_difference_is_a_failure(tmp_path) -> None: chromium = _observation("chromium", {"nodeType": 3, "nodeValue": "reference"}) moli = _observation("moli", {"nodeType": 3, "nodeValue": "different"}) @@ -378,4 +565,7 @@ def test_failed_reference_gate_never_starts_moli(monkeypatch, tmp_path) -> None: "cases": 1, "errors": 1, "moliPhaseStarted": False, + "infrastructureRetryLimit": 0, + "retriedCases": 0, + "recoveredCases": 0, }