diff --git a/AGENTS.md b/AGENTS.md index 7de19f59..3edbc0a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -231,7 +231,7 @@ this section in step with the launcher options. - **Leave the viewport alone.** When the window size is spoofed, new pages default to `no_viewport=True`; forcing a `viewport` asks the window to resize to something the identity does not claim. - **`is_mobile` is ignored** (with a warning). Camoufox has desktop identities only. - **Persistent contexts** need `persistent_context=True` and `user_data_dir`, and cannot be served by `launch_server`. -- **`headless="virtual"`** needs Linux and `Xvfb` (`CannotFindXvfb`, `VirtualDisplayNotSupported` otherwise). +- **`headless="virtual"`** needs Linux, `Xvfb` and `setpriv` from util-linux (`CannotFindXvfb`, `VirtualDisplayNotSupported` otherwise). - **`os` values are lowercase** `windows`, `macos`, `linux`; anything else raises `InvalidOS`. - **`geoip` in Python needs the extra**: `pip install "camoufox[geoip]"`, else `NotInstalledGeoIPExtra`. - **`InvalidIP`** is raised when the public-IP or proxy exit-IP lookup fails, or when `webrtc_ip` is not a valid IPv4/IPv6 address. Pass `webrtc_ip` and `timezone_id` to `NewContext` to skip the lookup. diff --git a/browser/tests/native/test_crash_recovery.py b/browser/tests/native/test_crash_recovery.py index b06090cf..95b3476e 100644 --- a/browser/tests/native/test_crash_recovery.py +++ b/browser/tests/native/test_crash_recovery.py @@ -27,6 +27,9 @@ from __future__ import annotations import asyncio import enum +import os +import subprocess # nosec +import sys from pathlib import Path import pytest @@ -307,6 +310,79 @@ async def test_browser_crash_under_a_virtual_display_does_not_leak_xvfb(binary, leak_check.assert_clean() +def _on_display(psutil_mod, display: str, xvfb: int) -> list: + """The display's Xvfb and every process on the machine using it, ours or not. + + Xvfb inherits its launcher's environment, so it is matched by pid. + """ + found = [] + for proc in psutil_mod.process_iter(): + try: + running_xvfb = proc.pid == xvfb and proc.status() != psutil_mod.STATUS_ZOMBIE + if running_xvfb or proc.environ().get("DISPLAY") == display: + found.append(f"{proc.name()}({proc.pid})") + except (psutil_mod.NoSuchProcess, psutil_mod.AccessDenied, psutil_mod.ZombieProcess): + continue + return found + + +async def test_launcher_sigkill_leaves_nothing_on_its_display(binary, psutil_mod): + """The process that called Camoufox() is killed, so no close path runs (#868). + + Every other test here keeps the launcher alive to tear down. When the + launcher itself is SIGKILLed -- by `timeout`, a supervisor, a cancelled CI + job -- what it started is reparented away from it, and out of leak_check's + sight, which only follows our descendants. So this looks machine-wide: the + Xvfb, the browser, and any session helpers they autolaunched must all go. + """ + if not _virtual_display_supported(): + pytest.fail("Xvfb is not installed") + + import camoufox + + child = subprocess.Popen( # nosec + [ + sys.executable, + "-c", + "import time\n" + "from camoufox.sync_api import Camoufox\n" + f"with Camoufox(executable_path={str(binary)!r}, headless='virtual',\n" + " i_know_what_im_doing=True) as browser:\n" + " browser.new_page().goto('about:blank')\n" + " print('ready', flush=True)\n" + " time.sleep(120)\n", + ], + stdout=subprocess.PIPE, + text=True, + env={**os.environ, "PYTHONPATH": str(Path(camoufox.__file__).parents[1])}, + ) + try: + ready = await bounded( + asyncio.to_thread(lambda: any(line.strip() == "ready" for line in child.stdout)), + 60, + "launch in the child", + ) + assert ready, "the child exited without launching a browser" + started = {p.name(): p for p in psutil_mod.Process(child.pid).children(recursive=True)} + display = started[BROWSER].environ()["DISPLAY"] + xvfb = started[XVFB].pid + finally: + child.kill() + child.wait() + + survivors = _on_display(psutil_mod, display, xvfb) + deadline = asyncio.get_running_loop().time() + 15 + while survivors and asyncio.get_running_loop().time() < deadline: + await asyncio.sleep(0.25) + survivors = _on_display(psutil_mod, display, xvfb) + assert not survivors, ( + f"{survivors} outlived the SIGKILLed process that launched them on {display}. " + "Every killed scraper strands another set until the host runs out of memory." + ) + for path in x11_artifacts_for(display): + assert not Path(path).exists(), f"{path} outlived the launcher that claimed {display}" + + async def test_a_stale_x11_lock_does_not_block_the_next_display(psutil_mod): """Recovery from someone else's mess. diff --git a/browser/tests/native/test_tribal_rules.py b/browser/tests/native/test_tribal_rules.py index b7d8807a..a7ceec19 100644 --- a/browser/tests/native/test_tribal_rules.py +++ b/browser/tests/native/test_tribal_rules.py @@ -145,14 +145,17 @@ def test_composite_can_still_be_turned_on(monkeypatch): ) -def test_displays_are_claimed_with_displayfd_not_a_lockfile_scan(): +def _xvfb_launch() -> Dict[str, Any]: + """The argv and Popen kwargs Xvfb is launched with. + + Asserted against what is actually launched, not against the source text -- + a first draft of the displayfd test passed on the *comment* above the call, + which is exactly the kind of vacuous assertion that makes a rule file worse + than nothing. + """ from camoufox import virtdisplay - # Asserted against the argv Xvfb is actually launched with, not against the - # source text -- a first draft of this test passed on the *comment* above - # the call, which is exactly the kind of vacuous assertion that makes a rule - # file worse than nothing. - captured = {} + captured: Dict[str, Any] = {} class FakePopen: def __init__(self, cmd, **kwargs): @@ -167,6 +170,7 @@ def test_displays_are_claimed_with_displayfd_not_a_lockfile_scan(): monkey = pytest.MonkeyPatch() try: monkey.setattr(type(display), "xvfb_path", property(lambda self: "/usr/bin/Xvfb")) + monkey.setattr(type(display), "setpriv_path", property(lambda self: "/usr/bin/setpriv")) monkey.setattr(virtdisplay.subprocess, "Popen", FakePopen) # get() blocks reading the display number back from a pipe the fake # never writes to; the bounded read is what makes that safe to do here. @@ -177,7 +181,11 @@ def test_displays_are_claimed_with_displayfd_not_a_lockfile_scan(): pass finally: monkey.undo() + return captured + +def test_displays_are_claimed_with_displayfd_not_a_lockfile_scan(): + captured = _xvfb_launch() cmd = captured.get("cmd") or [] assert "-displayfd" in cmd, ( "Xvfb is no longer launched with -displayfd, so display allocation has gone back " @@ -189,6 +197,15 @@ def test_displays_are_claimed_with_displayfd_not_a_lockfile_scan(): ) +def test_xvfb_carries_a_parent_death_signal(): + cmd = _xvfb_launch().get("cmd") or [] + assert cmd[:4] == ["/usr/bin/setpriv", "--pdeathsig", "TERM", "/usr/bin/Xvfb"], ( + "Xvfb is no longer launched under setpriv --pdeathsig TERM, so a launcher " + f"killed by a signal strands it. argv was: {cmd}" + + explain("xvfb-dies-with-its-launcher") + ) + + def test_the_displayfd_read_is_bounded(): from camoufox import virtdisplay diff --git a/ci/tribal-rules.yml b/ci/tribal-rules.yml index 20eb1ac8..4d56956b 100644 --- a/ci/tribal-rules.yml +++ b/ci/tribal-rules.yml @@ -131,6 +131,24 @@ rules: later display allocation. Both halves matter: SIGKILL then wait() so nothing is reaped later, then unlink both files. + - id: xvfb-dies-with-its-launcher + title: Xvfb gets a parent-death signal, so it never outlives its launcher + check: automated + evidence: + - "issue #868: 113 orphaned Xvfb processes after SIGTERMed launchers" + - "browser/tests/native/test_crash_recovery.py::test_launcher_sigkill_leaves_nothing_on_its_display" + - "dbus-launch(1): --autolaunch implies --exit-with-session, whose babysitter kills the bus when it loses its X connection" + rationale: >- + kill() runs on the close path, which a launcher killed by a signal never + reaches, and start_new_session keeps Xvfb out of a process-group kill. So + Xvfb is exec'd through `setpriv --pdeathsig TERM`: the kernel signals it + when its launcher dies, even by SIGKILL, and SIGTERM lets Xvfb remove its + own lock and socket. The session bus Firefox autolaunches on that display + dies with the X connection, taking its helpers with it. setpriv rather + than prctl in a preexec_fn: it is the same in both packages, and + preexec_fn is unsafe in a threaded parent. The signal fires when the + thread that spawned Xvfb exits, which is the thread that owns the browser. + # ------------------------------------------------------------------------- # Dependencies # ------------------------------------------------------------------------- diff --git a/python/README.md b/python/README.md index 107f5b86..ec22fb2f 100644 --- a/python/README.md +++ b/python/README.md @@ -167,7 +167,7 @@ host's only when the identity claims the host's GPU. Otherwise |---|---|---| | `False` (default) | A normal window | Needs a display. | | `True` | Firefox headless | Camoufox patches the headless tells it knows about. | -| `"virtual"` | A headed browser on a private Xvfb display | Linux only, needs `Xvfb`. Handled by `Camoufox`, `AsyncCamoufox`, `NewBrowser` and `launch_server`. The display is 1x1 unless `CAMOUFOX_VIRTUAL_DISPLAY_SIZE` (e.g. `1920x1080x24`) says otherwise. | +| `"virtual"` | A headed browser on a private Xvfb display | Linux only, needs `Xvfb` and `setpriv` (util-linux). Handled by `Camoufox`, `AsyncCamoufox`, `NewBrowser` and `launch_server`. The display is 1x1 unless `CAMOUFOX_VIRTUAL_DISPLAY_SIZE` (e.g. `1920x1080x24`) says otherwise. | ### Remote server diff --git a/python/src/camoufox/virtdisplay.py b/python/src/camoufox/virtdisplay.py index 4131a7ba..7aff1c28 100644 --- a/python/src/camoufox/virtdisplay.py +++ b/python/src/camoufox/virtdisplay.py @@ -87,6 +87,13 @@ class VirtualDisplay: raise CannotExecuteXvfb(f"I do not have permission to execute Xvfb: {path}") return path + @property + def setpriv_path(self) -> str: + path = which("setpriv") + if not path: + raise CannotFindXvfb("Please install util-linux (for setpriv) to use headless mode.") + return path + def get(self) -> str: self._assert_linux() @@ -97,7 +104,14 @@ class VirtualDisplay: # the child (unlike Node's `stdio: [..., 'pipe']` which renumbers # to 3), so we tell Xvfb that exact number. read_fd, write_fd = os.pipe() - cmd = [self.xvfb_path, "-displayfd", str(write_fd), *self.xvfb_args] + # setpriv's parent-death signal stops Xvfb when this process dies + # without reaching kill(), even by SIGKILL; SIGTERM lets Xvfb remove + # its own lock and socket. See xvfb-dies-with-its-launcher in + # ci/tribal-rules.yml. + cmd = [ + self.setpriv_path, "--pdeathsig", "TERM", + self.xvfb_path, "-displayfd", str(write_fd), *self.xvfb_args, + ] if self.debug: print("Starting virtual display:", " ".join(cmd)) self.proc = subprocess.Popen( # nosec diff --git a/python/tests/test_virtdisplay.py b/python/tests/test_virtdisplay.py index 8903bed7..26d4c69e 100644 --- a/python/tests/test_virtdisplay.py +++ b/python/tests/test_virtdisplay.py @@ -14,6 +14,8 @@ takes one X11 socket plus our -displayfd pipe. import os import re +import signal +import subprocess # nosec import sys import time from typing import List, Set @@ -128,3 +130,47 @@ def test_released_display_numbers_can_be_reused_on_the_next_launch(tracked): # Sanity: a_display was a valid form too. assert DISPLAY_RE.match(a_display), a_display + + +def _alive(pid: int) -> bool: + """True while `pid` runs; a zombie waiting for its new parent counts as dead.""" + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().rsplit(")", 1)[1].split()[0] != "Z" + except FileNotFoundError: + return False + + +def test_xvfb_dies_with_the_process_that_started_it(): + """A SIGKILLed launcher never runs kill(), so the kernel must stop Xvfb (#868). + + Supervisors, `timeout` and CI cancellation end the Python process with a + signal that skips every `finally`. Xvfb then has to go on its own, and + remove its lock and socket on the way out. + """ + child = subprocess.Popen( # nosec + [ + sys.executable, + "-c", + "import sys, time\n" + "from camoufox.virtdisplay import VirtualDisplay\n" + "vd = VirtualDisplay()\n" + "print(vd.get()[1:], vd.proc.pid, flush=True)\n" + "time.sleep(60)\n", + ], + stdout=subprocess.PIPE, + text=True, + ) + display, pid = child.stdout.readline().split() + xvfb = int(pid) + assert _alive(xvfb) + + child.send_signal(signal.SIGKILL) + child.wait() + + deadline = time.monotonic() + 5 + while _alive(xvfb) and time.monotonic() < deadline: + time.sleep(0.025) + assert not _alive(xvfb), f"Xvfb {xvfb} outlived the process that started it" + assert not os.path.exists(f"/tmp/.X{display}-lock") + assert not os.path.exists(f"/tmp/.X11-unix/X{display}") diff --git a/typescript/README.md b/typescript/README.md index d54363d3..a055d7e7 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -154,7 +154,7 @@ host's only when the identity claims the host's GPU. Otherwise |---|---|---| | `false` (default) | A normal window | Needs a display. | | `true` | Firefox headless | Camoufox patches the headless tells it knows about. | -| `"virtual"` | A headed browser on a private Xvfb display | Linux only, needs `Xvfb`. Handled by `Camoufox()`, `NewBrowser()` and `launchServer()`. The display is 1x1 unless `CAMOUFOX_VIRTUAL_DISPLAY_SIZE` (e.g. `1920x1080x24`) says otherwise. | +| `"virtual"` | A headed browser on a private Xvfb display | Linux only, needs `Xvfb` and `setpriv` (util-linux). Handled by `Camoufox()`, `NewBrowser()` and `launchServer()`. The display is 1x1 unless `CAMOUFOX_VIRTUAL_DISPLAY_SIZE` (e.g. `1920x1080x24`) says otherwise. | ### Remote server diff --git a/typescript/src/virtdisplay.ts b/typescript/src/virtdisplay.ts index 4437f754..0f0e7d7c 100644 --- a/typescript/src/virtdisplay.ts +++ b/typescript/src/virtdisplay.ts @@ -112,6 +112,19 @@ export class VirtualDisplay { return resolved; } + get setprivPath(): string { + try { + return execFileSync("which", ["setpriv"], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + throw new CannotFindXvfb( + "Please install util-linux (for setpriv) to use headless mode.", + ); + } + } + /** * Spawn Xvfb (if it isn't already running) and return its ":N" display. * @@ -124,7 +137,19 @@ export class VirtualDisplay { VirtualDisplay.assertLinux(); if (this.proc === null) { - const cmd = [this.xvfbPath, "-displayfd", "3", ...this.xvfbArgs]; + // setpriv's parent-death signal stops Xvfb when this process dies + // without reaching kill(), even by SIGKILL; SIGTERM lets Xvfb remove + // its own lock and socket. See xvfb-dies-with-its-launcher in + // ci/tribal-rules.yml. + const cmd = [ + this.setprivPath, + "--pdeathsig", + "TERM", + this.xvfbPath, + "-displayfd", + "3", + ...this.xvfbArgs, + ]; if (this.debug) { console.log("Starting virtual display:", cmd.join(" ")); } diff --git a/typescript/tests/virtdisplay.test.ts b/typescript/tests/virtdisplay.test.ts index aef2bc6a..f10d7fbe 100644 --- a/typescript/tests/virtdisplay.test.ts +++ b/typescript/tests/virtdisplay.test.ts @@ -5,10 +5,11 @@ * * VIRTDISPLAY_TEST_N controls the concurrent-launch count (default 50). */ -import { execFileSync } from "node:child_process"; +import { execFileSync, spawn } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; +import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; import { CannotExecuteXvfb, @@ -109,6 +110,16 @@ function hasXvfb(): boolean { } } +/** True while `pid` runs; a zombie waiting for its new parent counts as dead. */ +function alive(pid: number): boolean { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8"); + return stat.slice(stat.lastIndexOf(")") + 2)[0] !== "Z"; + } catch { + return false; + } +} + const DISPLAY_RE = /^:\d+$/; const N = Number.parseInt(process.env.VIRTDISPLAY_TEST_N ?? "50", 10); @@ -186,6 +197,44 @@ describe.skipIf( expect(aDisplay).toMatch(DISPLAY_RE); }); + it("Xvfb dies with the process that started it (#868)", async () => { + // A SIGKILLed launcher never runs kill(), so the kernel must stop Xvfb, + // and SIGTERM lets Xvfb remove its own lock and socket. The child loads + // the source through Vite's module runner, as vitest does. + const src = fileURLToPath( + new URL("../src/virtdisplay.ts", import.meta.url), + ); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { runnerImport } from "vite"; + const { module: m } = await runnerImport(${JSON.stringify(src)}); + const vd = new m.VirtualDisplay(); + console.log((await vd.get()).slice(1), vd.proc.pid); + setInterval(() => {}, 60_000);`, + ], + { + cwd: path.dirname(path.dirname(src)), + stdio: ["ignore", "pipe", "inherit"], + }, + ); + const [display, pid] = await new Promise((resolve) => + child.stdout.once("data", (d) => resolve(String(d).trim().split(" "))), + ); + expect(alive(Number(pid))).toBe(true); + + child.kill("SIGKILL"); + const deadline = Date.now() + 5000; + while (alive(Number(pid)) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 25)); + } + expect(alive(Number(pid)), `Xvfb ${pid} outlived its launcher`).toBe(false); + expect(fs.existsSync(`/tmp/.X${display}-lock`)).toBe(false); + expect(fs.existsSync(`/tmp/.X11-unix/X${display}`)).toBe(false); + }, 30_000); + it("kill() removes the lock and socket even when Xvfb already died", async () => { const vd = track(new VirtualDisplay()); const display = (await vd.get()).slice(1);