mirror of
https://github.com/daijro/camoufox.git
synced 2026-11-19 00:07:28 +00:00
150 lines
5.9 KiB
Python
150 lines
5.9 KiB
Python
"""Shared pytest plumbing for the suite runners.
|
|
|
|
Locating the built binary, running pytest against a chosen interpreter, and
|
|
reading back the JUnit XML it writes. `junit_test_id` is the one that matters
|
|
beyond this file: it produces the identity a test is known by, and that identity
|
|
has to survive a suite being re-fetched at a different tag, or run from a
|
|
different working directory, or sharded -- otherwise a run cannot be compared
|
|
with the one before it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
from ._util import BROWSER_ROOT, Result, read_upstream_sh, run
|
|
|
|
|
|
def source_dir(version: Optional[str] = None, release: Optional[str] = None) -> Path:
|
|
"""The generated Firefox tree, e.g. browser/camoufox-153.0.4-beta.32/."""
|
|
up = read_upstream_sh()
|
|
return BROWSER_ROOT / f"camoufox-{version or up['version']}-{release or up['release']}"
|
|
|
|
|
|
def built_binary(version: Optional[str] = None, release: Optional[str] = None) -> Path:
|
|
"""Path to the freshly built camoufox-bin. Not guaranteed to exist."""
|
|
override = os.environ.get("CAMOUFOX_BINARY")
|
|
if override:
|
|
return Path(override)
|
|
return source_dir(version, release) / "obj-x86_64-pc-linux-gnu" / "dist" / "bin" / "camoufox-bin"
|
|
|
|
|
|
def require_binary(version: Optional[str] = None, release: Optional[str] = None) -> Path:
|
|
path = built_binary(version, release)
|
|
if not path.exists():
|
|
raise FileNotFoundError(
|
|
f"no built binary at {path}. Run `make -C browser build` first, or set CAMOUFOX_BINARY."
|
|
)
|
|
return path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# pytest / junit
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# pytest junit escapes some characters; normalise so ids are stable across
|
|
# pytest versions and across checkouts rooted at different paths.
|
|
_NORM = re.compile(r"\s+")
|
|
|
|
|
|
def junit_test_id(classname: str, name: str) -> str:
|
|
"""A stable identity for one test, in pytest node-id form.
|
|
|
|
junit's `classname` is the dotted module path, and how many leading segments
|
|
it carries depends on where pytest was invoked from -- `tests.async.test_page`
|
|
from the checkout root, `async.test_page` from inside `tests/`. Trimming to
|
|
the last two segments makes a run comparable with any other run of the same
|
|
test, however it was launched.
|
|
|
|
For a test inside a class, pytest appends the class to that dotted path
|
|
(`async.test_page_clock.TestWhileRunning`), and the trailing segment is then
|
|
a class, not a module. Splitting on the last `test_*` segment tells the two
|
|
apart, so a class-based test comes back as
|
|
|
|
async/test_page_clock.py::TestWhileRunning::test_should_pause
|
|
|
|
-- a node id pytest will actually accept. Treating the class as a module
|
|
instead produced `test_page_clock/TestWhileRunning.py::test_should_pause`,
|
|
which names a directory that does not exist, so the id could not be fed back
|
|
to pytest and no skiplist entry could ever match it.
|
|
"""
|
|
parts = [p for p in classname.split(".") if p and p != "tests"]
|
|
# The last segment that looks like a test module is the file; anything after
|
|
# it is class nesting. Test files are `test_*.py` by pytest convention, and
|
|
# classes are `Test*` -- so this does not have to guess from case alone.
|
|
module_end = len(parts)
|
|
for i in range(len(parts) - 1, -1, -1):
|
|
if parts[i].startswith("test_"):
|
|
module_end = i + 1
|
|
break
|
|
module_parts, class_parts = parts[:module_end], parts[module_end:]
|
|
module = (
|
|
"/".join(module_parts[-2:])
|
|
if len(module_parts) >= 2
|
|
else (module_parts[-1] if module_parts else "")
|
|
)
|
|
suffix = "".join(f"::{c}" for c in class_parts)
|
|
return _NORM.sub(" ", f"{module}.py{suffix}::{name}").strip()
|
|
|
|
|
|
def parse_junit(path: Path) -> Dict[str, str]:
|
|
"""junit XML -> {test_id: pass|fail|error|skip}."""
|
|
if not path.exists():
|
|
return {}
|
|
outcomes: Dict[str, str] = {}
|
|
tree = ET.parse(path)
|
|
for case in tree.getroot().iter("testcase"):
|
|
tid = junit_test_id(case.get("classname", ""), case.get("name", ""))
|
|
if not tid:
|
|
continue
|
|
if case.find("error") is not None:
|
|
outcome = "error"
|
|
elif case.find("failure") is not None:
|
|
outcome = "fail"
|
|
elif case.find("skipped") is not None:
|
|
outcome = "skip"
|
|
else:
|
|
outcome = "pass"
|
|
# With --count / reruns the same id appears twice; a pass on any
|
|
# attempt wins, matching evidence.GateResult.record().
|
|
if outcomes.get(tid) == "pass":
|
|
continue
|
|
outcomes[tid] = outcome
|
|
return outcomes
|
|
|
|
|
|
def _has_plugin(python: Path, module: str) -> bool:
|
|
return run([str(python), "-c", f"import {module}"]).ok
|
|
|
|
|
|
def run_pytest(
|
|
*,
|
|
cwd: Path,
|
|
python: Path,
|
|
args: List[str],
|
|
junit: Path,
|
|
env: Optional[Dict[str, str]] = None,
|
|
timeout: int = 7200,
|
|
per_test_timeout: Optional[int] = 180,
|
|
) -> Result:
|
|
"""Run pytest and write junit XML.
|
|
|
|
`per_test_timeout` guards against a hung browser wedging the whole job, but
|
|
it needs pytest-timeout. Passing the flag without the plugin makes pytest
|
|
exit 4 on an unrecognised argument -- which looks exactly like "the suite
|
|
did not run", because it did not. So the flag is only added when the plugin
|
|
is actually importable in that interpreter.
|
|
"""
|
|
junit.parent.mkdir(parents=True, exist_ok=True)
|
|
# A run that dies before writing junit must not leave the last run's.
|
|
junit.unlink(missing_ok=True)
|
|
cmd = [str(python), "-m", "pytest", f"--junitxml={junit}", "-p", "no:randomly"]
|
|
if per_test_timeout and _has_plugin(python, "pytest_timeout"):
|
|
cmd.append(f"--timeout={per_test_timeout}")
|
|
cmd.extend(args)
|
|
return run(cmd, cwd=cwd, env=env, timeout=timeout, tee=True, capture=False)
|