from __future__ import annotations import json import tempfile import unittest from pathlib import Path from unittest.mock import patch from moli_benchmark.wpt import ( REPORT_PREFIX, _by_tag_rows, _collect_reports, _run_command, _wpt_case_diff, run_wpt_suite, ) class WptReportTests(unittest.TestCase): def test_collect_reports_records_tag_rates(self) -> None: with tempfile.TemporaryDirectory() as target_temp, tempfile.TemporaryDirectory() as suite_temp: target_dir = Path(target_temp) suite_dir = Path(suite_temp) (target_dir / f"{REPORT_PREFIX}sample.json").write_text( json.dumps( { "cases": [ {"id": "a", "expected": "PASS", "actual": "PASS", "category": "pass", "tags": ["dom", "events"]}, { "id": "b", "expected": "PASS", "actual": "FAIL", "category": "unexpected-fail", "tags": ["dom"], "failures": ["assertion failed"], }, {"id": "c", "expected": "FAIL", "actual": "FAIL", "category": "known-fail", "tags": ["network"]}, {"id": "d", "expected": "PASS", "actual": "SKIP", "category": "skip", "tags": []}, ] } ), encoding="utf-8", ) summary, cases = _collect_reports(target_dir, suite_dir) self.assertEqual(len(cases), 4) self.assertEqual(summary["total"], 4) self.assertEqual(summary["pass"], 1) self.assertEqual(summary["unexpected_fail"], 1) self.assertEqual(summary["known_fail"], 1) self.assertEqual(summary["skip"], 1) self.assertEqual(summary["pass_rate_percent"], 25.0) self.assertEqual(summary["unexpected_fail_rate_percent"], 25.0) self.assertEqual(summary["skip_rate_percent"], 25.0) self.assertEqual(summary["by_tag"]["dom"]["total"], 2) self.assertEqual(summary["by_tag"]["dom"]["pass_rate_percent"], 50.0) self.assertEqual(summary["by_tag"]["events"]["pass_rate_percent"], 100.0) def test_by_tag_rows_are_stable_and_csv_ready(self) -> None: rows = _by_tag_rows( { "by_tag": { "network": {"total": 2, "pass": 1, "pass_rate_percent": 50.0}, "dom": {"total": 1, "pass": 1, "pass_rate_percent": 100.0}, } } ) self.assertEqual([row["tag"] for row in rows], ["dom", "network"]) self.assertEqual(rows[0]["pass_rate_percent"], 100.0) def test_wpt_case_diff_tracks_added_removed_expectation_and_category_changes(self) -> None: summary, rows = _wpt_case_diff( current_cases=[ {"id": "kept", "expected": "FAIL", "actual": "PASS", "category": "unexpected-pass", "tags": "dom"}, {"id": "added", "expected": "PASS", "actual": "PASS", "category": "pass", "tags": "events"}, ], baseline_cases=[ {"id": "kept", "expected": "PASS", "actual": "FAIL", "category": "unexpected-fail", "tags": "dom"}, {"id": "removed", "expected": "PASS", "actual": "PASS", "category": "pass", "tags": "network"}, ], ) self.assertEqual(summary["added"], 1) self.assertEqual(summary["removed"], 1) self.assertEqual(summary["expectation_changes"], 1) self.assertEqual(summary["category_changes"], 1) self.assertEqual(summary["total_changes"], 4) self.assertEqual([row["kind"] for row in rows], ["added", "removed", "expectation-change", "category-change"]) def test_wpt_runner_uses_release_nextest_only(self) -> None: command = _run_command("nextest") self.assertEqual(command[:3], ["cargo", "nextest", "run"]) self.assertIn("--release", command) self.assertNotIn("test", command[:2]) with self.assertRaisesRegex(RuntimeError, "release nextest runner"): _run_command("cargo") def test_run_wpt_suite_writes_baseline_diff_artifacts(self) -> None: with tempfile.TemporaryDirectory() as target_temp, tempfile.TemporaryDirectory() as output_temp, tempfile.TemporaryDirectory() as baseline_temp: target_dir = Path(target_temp) output_dir = Path(output_temp) baseline = Path(baseline_temp) / "baseline.json" (target_dir / f"{REPORT_PREFIX}sample.json").write_text( json.dumps( { "cases": [ {"id": "kept", "expected": "FAIL", "actual": "PASS", "category": "unexpected-pass", "tags": ["dom"]}, {"id": "added", "expected": "PASS", "actual": "PASS", "category": "pass", "tags": ["events"]}, ] } ), encoding="utf-8", ) baseline.write_text( json.dumps( { "cases": [ {"id": "kept", "expected": "PASS", "actual": "FAIL", "category": "unexpected-fail", "tags": ["dom"]}, {"id": "removed", "expected": "PASS", "actual": "PASS", "category": "pass", "tags": ["network"]}, ] } ), encoding="utf-8", ) with patch.dict("os.environ", {"CARGO_TARGET_DIR": str(target_dir)}): summary = run_wpt_suite( output_dir=output_dir, timeout_seconds=1.0, runner="nextest", compat=None, case_filter=None, tag_filter=None, no_run=True, baseline=baseline, ) self.assertEqual(summary["diff"]["total_changes"], 4) self.assertTrue((output_dir / "wpt" / "diff.json").exists()) self.assertTrue((output_dir / "wpt" / "diff.csv").exists()) if __name__ == "__main__": unittest.main()