from __future__ import annotations import hashlib import json import signal import ssl import sys import tempfile import unittest from dataclasses import replace from pathlib import Path from unittest.mock import patch from urllib.error import HTTPError from urllib.request import HTTPSHandler, ProxyHandler, build_opener from moli_benchmark import webmainbench as bench from moli_benchmark.process import ProcessResult, run_process PAGE = bench.Case( "11111111-1111-4111-8111-111111111111", "https://example.org/", "

Page

" ) REDIRECT = bench.Case( bench.REDIRECT_CASE_ID, "http://langrensha.163.com/", "", ) DNS_ERROR = ( b"Error: failed to fetch `https://127.0.0.1:1234/case/page.html`\n" b"Reason: failed to resolve langrensha.163.com:443: failed to lookup address information: " b"Temporary failure in name resolution\n" ) def process_result(**changes: object) -> ProcessResult: return replace( ProcessResult( command=[], returncode=0, elapsed_ms=1, stdout=b"# Page\n", stderr=b"", timed_out=False, resources={}, ), **changes, ) class WebMainBenchTests(unittest.TestCase): def test_expected_redirect_is_matched_by_case_and_error(self) -> None: result = process_result(returncode=1, stdout=b"", stderr=DNS_ERROR) self.assertEqual(bench.classify(REDIRECT, result), "expected_failure") self.assertEqual(bench.classify(PAGE, result), "failure") self.assertEqual( bench.classify( REDIRECT, replace(result, stderr=b"Reason: unrelated error\n") ), "failure", ) self.assertEqual(bench.classify(REDIRECT, process_result()), "success") def test_exception_never_hides_panic_crash_or_timeout(self) -> None: base = process_result(returncode=1, stdout=b"", stderr=DNS_ERROR) for result, expected in [ ( replace( base, stderr=b"thread 'render_runtime' panicked at tokenizer/mod.rs:254:9:\n" + DNS_ERROR, ), "panic", ), (replace(base, returncode=-signal.SIGSEGV), "crash"), (replace(base, timed_out=True), "timeout"), ( replace( base, stderr=b"Reason: fetch readiness timed out after 45000 ms while waiting for Done\n", ), "timeout", ), ]: with self.subTest(expected=expected): self.assertEqual(bench.classify(REDIRECT, result), expected) self.assertEqual( bench.classify( PAGE, process_result( stderr=b"thread '' panicked at parser.rs:12:3:\n" ), ), "panic", ) def test_success_requires_nonempty_markdown(self) -> None: self.assertEqual( bench.classify(PAGE, process_result(stdout=b" \n\t")), "empty_output" ) self.assertEqual(bench.classify(PAGE, process_result()), "success") def test_gate_requires_each_case_not_just_a_success_count(self) -> None: cases = [PAGE, REDIRECT] valid = [ {"track_id": PAGE.track_id, "status": "success"}, {"track_id": REDIRECT.track_id, "status": "expected_failure"}, ] with patch.object(bench, "CASE_COUNT", 2): self.assertEqual(bench.health_issues(cases, valid), []) self.assertTrue(bench.health_issues(cases, valid[:1])) self.assertTrue(bench.health_issues(cases, [valid[0], valid[0]])) changed_failure = [ {"track_id": PAGE.track_id, "status": "failure"}, {"track_id": REDIRECT.track_id, "status": "success"}, ] self.assertTrue(bench.health_issues(cases, changed_failure)) def test_dataset_drift_is_rejected_even_when_a_file_is_cached(self) -> None: with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "dataset.jsonl" path.write_text("{}\n", encoding="utf-8") with patch.object(bench, "urlopen") as download: with self.assertRaisesRegex(ValueError, "SHA-256 mismatch"): bench.download_dataset(path) download.assert_not_called() def test_dataset_requires_complete_unique_ids(self) -> None: with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "dataset.jsonl" rows = [ {"track_id": case.track_id, "url": case.url, "html": case.html} for case in [PAGE, REDIRECT] ] for data, valid in [ (rows, True), (rows[:1], False), ([rows[0], rows[0]], False), ]: path.write_text( "".join(json.dumps(row) + "\n" for row in data), encoding="utf-8" ) with ( patch.object(bench, "DATASET_SHA256", bench.file_sha256(path)), patch.object(bench, "CASE_COUNT", 2), ): if valid: self.assertEqual(bench.load_cases(path), [PAGE, REDIRECT]) else: with self.assertRaisesRegex(ValueError, "unique cases"): bench.load_cases(path) def test_runner_refuses_host_network_before_changing_interfaces(self) -> None: links = json.dumps([{"ifname": "lo"}, {"ifname": "eth0"}]).encode() with ( patch.object(bench.subprocess, "check_output", return_value=links), patch.object(bench.subprocess, "run") as run, ): with self.assertRaisesRegex(RuntimeError, "network namespace"): bench.require_loopback_namespace() run.assert_not_called() def test_https_fixture_preserves_malformed_html_and_inline_scripts(self) -> None: page = replace( PAGE, html='\n', ) with bench.fixture_server([page]) as (origin, cert): context = ssl.create_default_context(cafile=str(cert)) client = build_opener(ProxyHandler({}), HTTPSHandler(context=context)) with client.open( f"{origin}/case/{page.track_id}.html?query=1", timeout=5 ) as response: self.assertEqual(response.status, 200) self.assertEqual(response.read(), page.html.encode("utf-8")) for path in ["/missing.js", f"/elsewhere/{page.track_id}.html"]: with self.subTest(path=path): with self.assertRaises(HTTPError) as error: client.open(origin + path, timeout=5) self.assertEqual(error.exception.code, 404) error.exception.close() self.assertFalse(cert.exists()) def test_process_watchdog_fails_even_if_partial_output_exists(self) -> None: with tempfile.TemporaryDirectory() as temporary: result = run_process( [ sys.executable, "-c", 'import time; print("# Partial output", flush=True); time.sleep(30)', ], cwd=Path(temporary), timeout_seconds=1, sample_resources=False, ) self.assertTrue(result.timed_out) self.assertIn(b"Partial output", result.stdout) self.assertEqual(bench.classify(PAGE, result), "timeout") def test_failed_page_retains_evidence_and_does_not_skip_remaining_cases( self, ) -> None: with tempfile.TemporaryDirectory() as temporary: output = Path(temporary) / "run" with ( patch.object(bench, "load_cases", return_value=[PAGE, REDIRECT]), patch.object(bench, "CASE_COUNT", 2), patch.object(bench, "require_loopback_namespace"), patch.object( bench, "run_process", side_effect=[ process_result( returncode=1, stdout=b"", stderr=b"Reason: regression\n" ), process_result(), ], ), patch("builtins.print"), ): passed = bench.replay( Path(sys.executable), Path("unused"), output, "a" * 40 ) self.assertFalse(passed) report = json.loads((output / "summary.json").read_text()) self.assertEqual(report["completed"], 2) self.assertEqual(report["counts"], {"failure": 1, "success": 1}) records = [ json.loads(line) for line in (output / "results.jsonl").read_text().splitlines() ] self.assertEqual( [row["track_id"] for row in records], [PAGE.track_id, REDIRECT.track_id] ) self.assertEqual( records[0]["html_sha256"], hashlib.sha256(PAGE.html.encode()).hexdigest(), ) self.assertEqual( (output / f"{PAGE.track_id}.stderr").read_bytes(), b"Reason: regression\n", ) self.assertEqual( (output / f"{REDIRECT.track_id}.md").read_bytes(), b"# Page\n" ) self.assertIn("**FAIL**", (output / "summary.md").read_text()) self.assertEqual(list(output.glob("*.pem")), []) def test_interrupted_suite_keeps_partial_results_and_fails(self) -> None: with tempfile.TemporaryDirectory() as temporary: output = Path(temporary) / "run" with ( patch.object(bench, "load_cases", return_value=[PAGE, REDIRECT]), patch.object(bench, "CASE_COUNT", 2), patch.object(bench, "require_loopback_namespace"), patch.object( bench, "run_process", side_effect=[process_result(), OSError("cannot launch process")], ), patch("builtins.print"), self.assertLogs(bench.LOGGER, level="ERROR"), ): passed = bench.replay( Path(sys.executable), Path("unused"), output, "a" * 40 ) self.assertFalse(passed) report = json.loads((output / "summary.json").read_text()) self.assertEqual(report["completed"], 1) self.assertIn( "Infrastructure error: OSError: cannot launch process", report["issues"] ) self.assertEqual( len((output / "results.jsonl").read_text().splitlines()), 1 ) self.assertIn("**FAIL**", (output / "summary.md").read_text()) if __name__ == "__main__": unittest.main()