mirror of
https://github.com/lexmount/moli.git
synced 2026-09-23 16:01:25 +00:00
ldm0
c461cb1799
Move public-web seeds into CSV fixtures and share scheduling, command construction, classification, evidence, and artifact handling across top-sites and wild-web suites. Bind Chromium DCL and response evidence to the current navigation loader, surface navigation errors promptly, rotate target-first order across repeated runs, and normalize HTTP and timeout outcomes so cross-engine results remain comparable.
269 lines
11 KiB
Python
269 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from tempfile import TemporaryDirectory
|
|
from pathlib import Path
|
|
|
|
from moli_benchmark.artifacts import write_json
|
|
from moli_benchmark.html_report import _artifact_paths_by_suite, _chartjs_document, _report_payload
|
|
|
|
|
|
class HtmlReportTests(unittest.TestCase):
|
|
def test_artifact_index_omits_absent_conditional_artifacts(self) -> None:
|
|
paths = _artifact_paths_by_suite(
|
|
[
|
|
{"suite": "startup", "cache_artifacts": []},
|
|
{"suite": "cdp-session", "total_failures": 0, "total_trace_events": 0},
|
|
{"suite": "wild-web", "total_failures": 0, "replay_artifacts": 0},
|
|
{"suite": "wpt", "diff": None},
|
|
]
|
|
)
|
|
|
|
self.assertNotIn("startup/cache/", paths["startup"])
|
|
self.assertNotIn("cdp-session/traces/", paths["cdp-session"])
|
|
self.assertNotIn("wild-web/failures/", paths["wild-web"])
|
|
self.assertNotIn("wild-web/replay/manifest.json", paths["wild-web"])
|
|
self.assertNotIn("wpt/diff.json", paths["wpt"])
|
|
self.assertNotIn("wpt/diff.csv", paths["wpt"])
|
|
|
|
def test_artifact_index_includes_present_conditional_artifacts(self) -> None:
|
|
paths = _artifact_paths_by_suite(
|
|
[
|
|
{"suite": "startup", "cache_artifacts": ["startup/cache/serve-ready-run-1/drop-caches.txt"]},
|
|
{"suite": "cdp-session", "total_failures": 0, "total_trace_events": 2},
|
|
{"suite": "wild-web", "total_failures": 1, "replay_artifacts": 1},
|
|
{"suite": "wpt", "diff": {"total_changes": 1}},
|
|
]
|
|
)
|
|
|
|
self.assertIn("startup/cache/", paths["startup"])
|
|
self.assertIn("cdp-session/traces/", paths["cdp-session"])
|
|
self.assertIn("wild-web/failures/", paths["wild-web"])
|
|
self.assertIn("wild-web/replay/manifest.json", paths["wild-web"])
|
|
self.assertIn("wpt/diff.json", paths["wpt"])
|
|
self.assertIn("wpt/diff.csv", paths["wpt"])
|
|
|
|
def test_artifact_index_includes_render_compare_details(self) -> None:
|
|
paths = _artifact_paths_by_suite([{"suite": "render-compare", "total_failures": 0}])
|
|
|
|
self.assertIn("render-compare/runs.json", paths["render-compare"])
|
|
self.assertIn("render-compare/baseline-sites.csv", paths["render-compare"])
|
|
|
|
def test_artifact_index_includes_top_sites_site_outcomes(self) -> None:
|
|
paths = _artifact_paths_by_suite(
|
|
[{"suite": "top-sites", "total_failures": 0}]
|
|
)
|
|
|
|
self.assertIn("top-sites/site-outcomes.json", paths["top-sites"])
|
|
|
|
def test_cdp_session_trace_link_uses_nested_summary_fallback(self) -> None:
|
|
paths = _artifact_paths_by_suite(
|
|
[
|
|
{
|
|
"suite": "cdp-session",
|
|
"total_failures": 0,
|
|
"targets": {"moli": {"console_errors": 0, "js_exceptions": 1, "network_failures": 0}},
|
|
}
|
|
]
|
|
)
|
|
|
|
self.assertIn("cdp-session/traces/", paths["cdp-session"])
|
|
|
|
def test_report_payload_adds_combined_web_scraping_variant_view(self) -> None:
|
|
payload = _report_payload(
|
|
output_dir=Path("/tmp/report"),
|
|
versions={},
|
|
publish_readiness=None,
|
|
report_diff=None,
|
|
summaries=[
|
|
{
|
|
"suite": "synthetic-compare",
|
|
"gate_target": "moli",
|
|
"gate_failures": 0,
|
|
"cases": ["static-html"],
|
|
"targets": {
|
|
"moli": {
|
|
"engine": "moli",
|
|
"driver": "fetch",
|
|
"label": "moli / fetch",
|
|
"failures": 0,
|
|
"cases": {"static-html": {"elapsed_ms": {"p50": 10.0, "count": 1}, "failures": 0}},
|
|
}
|
|
},
|
|
},
|
|
{
|
|
"suite": "cdp-session",
|
|
"gate_target": "moli-cdp",
|
|
"gate_failures": 0,
|
|
"cases": ["static-html"],
|
|
"targets": {
|
|
"moli-cdp": {
|
|
"engine": "moli",
|
|
"driver": "cdp",
|
|
"label": "moli / cdp",
|
|
"failures": 0,
|
|
"cases": {"static-html": {"elapsed_ms": {"p50": 20.0, "count": 1}, "failures": 0}},
|
|
}
|
|
},
|
|
},
|
|
],
|
|
)
|
|
|
|
combined = payload["horizontal_comparisons"][0]
|
|
self.assertEqual(combined["suite"], "web-scraping-variants")
|
|
self.assertEqual(list(combined["targets"]), ["moli", "moli-cdp"])
|
|
self.assertEqual(combined["targets"]["moli-cdp"]["label"], "moli / cdp")
|
|
|
|
def test_report_payload_embeds_render_compare_page_rows(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
output_dir = Path(temp_dir)
|
|
write_json(
|
|
output_dir / "render-compare" / "runs.json",
|
|
[
|
|
{
|
|
"target": "moli",
|
|
"engine": "moli",
|
|
"rank": 1,
|
|
"domain": "https://example.test/page",
|
|
"url": "https://example.test/page",
|
|
"category": "render-match",
|
|
"ok": True,
|
|
"elapsed_ms": 12.0,
|
|
"peak_rss_bytes": 4096,
|
|
"render_quality_score": 99.0,
|
|
"key_phrases": ["large internal detail should be dropped"],
|
|
}
|
|
],
|
|
)
|
|
write_json(
|
|
output_dir / "render-compare" / "baseline-sites.json",
|
|
[
|
|
{
|
|
"rank": 1,
|
|
"domain": "https://example.test/page",
|
|
"url": "https://example.test/page",
|
|
"category": "baseline-usable",
|
|
"usable": True,
|
|
"baseline_elapsed_ms": 10.0,
|
|
}
|
|
],
|
|
)
|
|
|
|
payload = _report_payload(
|
|
output_dir=output_dir,
|
|
versions={},
|
|
publish_readiness=None,
|
|
report_diff=None,
|
|
summaries=[{"suite": "render-compare", "targets": {}}],
|
|
)
|
|
|
|
self.assertEqual(payload["render_compare"]["run_count"], 1)
|
|
self.assertEqual(payload["render_compare"]["baseline_site_count"], 1)
|
|
self.assertEqual(payload["render_compare"]["runs"][0]["render_quality_score"], 99.0)
|
|
self.assertEqual(payload["render_compare"]["runs"][0]["peak_rss_bytes"], 4096)
|
|
self.assertNotIn("key_phrases", payload["render_compare"]["runs"][0])
|
|
|
|
def test_chartjs_report_uses_horizontal_case_metric_bars(self) -> None:
|
|
document = _chartjs_document(
|
|
{
|
|
"output_dir": "/tmp/report",
|
|
"versions": {},
|
|
"publish_readiness": {},
|
|
"summaries": [
|
|
{
|
|
"suite": "synthetic-compare",
|
|
"cases": ["static-html"],
|
|
"targets": {
|
|
"moli": {
|
|
"engine": "moli",
|
|
"driver": "fetch",
|
|
"label": "moli / fetch",
|
|
"failures": 0,
|
|
"cases": {"static-html": {"elapsed_ms": {"p50": 1.0, "count": 1}, "failures": 0}},
|
|
}
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertIn("indexAxis: 'y'", document)
|
|
self.assertIn("document.getElementById('latencyChart').parentElement.style.height", document)
|
|
self.assertIn("document.getElementById('rssChart').parentElement.style.height", document)
|
|
self.assertIn("Memory RSS P50 by Case", document)
|
|
self.assertIn("Web Page Request Scores", document)
|
|
self.assertIn("function renderWebPages()", document)
|
|
self.assertIn("render_quality_score", document)
|
|
self.assertIn("peak_rss_bytes", document)
|
|
self.assertIn('class="panel wide"', document)
|
|
self.assertIn("'peak_rss_bytes'", document)
|
|
self.assertIn("x: { beginAtZero: true, grid: { color: '#e6ebf1' }, title: { display: true, text: label } }", document)
|
|
self.assertIn("y: { grid: { display: false }, ticks: { autoSkip: false } }", document)
|
|
|
|
def test_chartjs_report_keeps_failure_reason_in_tooltip(self) -> None:
|
|
document = _chartjs_document({"output_dir": "/tmp/report", "versions": {}, "summaries": []})
|
|
|
|
self.assertIn("function statusBadge(status, tooltip)", document)
|
|
self.assertIn('class="status-tooltip"', document)
|
|
self.assertIn("data-tooltip=", document)
|
|
self.assertNotIn('<span class="muted">reason</span>', document)
|
|
|
|
def test_chartjs_report_aggregate_pass_rate_prefers_attempts_and_passes(self) -> None:
|
|
document = _chartjs_document(
|
|
{
|
|
"output_dir": "/tmp/report",
|
|
"versions": {},
|
|
"summaries": [
|
|
{
|
|
"suite": "crawler",
|
|
"targets": {
|
|
"moli": {
|
|
"label": "moli",
|
|
"passes": 3,
|
|
"failures": 2,
|
|
"runs": 1,
|
|
}
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertIn("const attempts = passes + failures;", document)
|
|
self.assertIn("result.pages ?? result.seeds ?? result.sites ?? result.runs", document)
|
|
self.assertIn("result.successes ?? result.categories?.success ?? result.passes", document)
|
|
self.assertIn("counted pass rate", document)
|
|
self.assertIn("raw pass rate", document)
|
|
self.assertIn("successful p50 / p90 / p95 ms", document)
|
|
self.assertIn("common p50 / p90 / p95 ms", document)
|
|
self.assertIn("result?.successful_elapsed_ms || result?.elapsed_ms", document)
|
|
self.assertIn("summary?.common_success?.targets?.[target]", document)
|
|
self.assertIn("site stability", document)
|
|
self.assertIn("HTTP status observed", document)
|
|
self.assertIn("not repeat-validated", document)
|
|
self.assertIn("Pairwise site outcomes", document)
|
|
self.assertIn("SINGLE SAMPLE", document)
|
|
|
|
def test_chartjs_report_renders_every_aggregate_summary_and_prioritizes_top_sites(
|
|
self,
|
|
) -> None:
|
|
document = _chartjs_document(
|
|
{
|
|
"output_dir": "/tmp/report",
|
|
"versions": {},
|
|
"summaries": [
|
|
{"suite": "wild-web", "targets": {"moli": {}}},
|
|
{"suite": "top-sites", "targets": {"moli": {}}},
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertIn("function horizontalSummaries()", document)
|
|
self.assertIn("'top-sites', 'wild-web'", document)
|
|
self.assertIn("summaries.map(summary =>", document)
|
|
self.assertIn('class="comparison-summary"', document)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|