diff --git a/python/src/camoufox/addons.py b/python/src/camoufox/addons.py index 32f36bbe..7ee83c20 100644 --- a/python/src/camoufox/addons.py +++ b/python/src/camoufox/addons.py @@ -1,7 +1,8 @@ import os import shutil +import tempfile from enum import Enum -from multiprocessing import Lock +from threading import Lock from typing import List, Optional from .exceptions import InvalidAddonPath @@ -10,6 +11,11 @@ from .pkgman import INSTALL_DIR, unzip, webdl # Addons are stored in a shared folder, not per-browser version ADDONS_DIR = INSTALL_DIR / "addons" +# One lock for the module: `with Lock():` on a fresh multiprocessing.Lock per call +# guarded nothing, so concurrent launches in one process raced the same +# check-then-extract. +_EXTRACT_LOCK = Lock() + class DefaultAddons(Enum): """ @@ -44,17 +50,35 @@ def add_default_addons( addons = [addon for addon in DefaultAddons if addon not in exclude_list] - with Lock(): + with _EXTRACT_LOCK: maybe_download_addons(addons, addons_list) def download_and_extract(url: str, extract_path: str, name: str) -> None: """ - Downloads and extracts an addon from a given URL to a specified path + Downloads and extracts an addon from a given URL to a specified path. + + Extracts into a sibling temp dir and renames it into place last, so a + concurrent launch (another process, which the lock cannot see) never finds + a manifest.json in a directory whose other files are still being written. """ - # Create a temporary file to store the downloaded zip buffer = webdl(url, desc=f"Downloading addon ({name})", bar=False) - unzip(buffer, extract_path, f"Extracting addon ({name})", bar=False) + parent = os.path.dirname(extract_path) + tmp_dir = tempfile.mkdtemp(prefix=f".{name}.tmp-", dir=parent) + try: + unzip(buffer, tmp_dir, f"Extracting addon ({name})", bar=False) + try: + os.replace(tmp_dir, extract_path) + except OSError: + # The target is not empty (or exists at all, on Windows). A complete + # copy came from another process and may be loading in its browser, + # so keep it; only an incomplete one is replaced. + if os.path.exists(os.path.join(extract_path, 'manifest.json')): + return + shutil.rmtree(extract_path) + os.replace(tmp_dir, extract_path) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) def get_addon_path(addon_name: str) -> str: @@ -92,6 +116,8 @@ def maybe_download_addons( if addons_list is not None: addons_list.append(addon_path) except Exception as e: - # Drop the partial directory so the next run re-downloads. - shutil.rmtree(addon_path, ignore_errors=True) + # Drop the partial directory so the next run re-downloads. A complete + # one came from another process and may be loading in its browser. + if not os.path.exists(os.path.join(addon_path, 'manifest.json')): + shutil.rmtree(addon_path, ignore_errors=True) print(f"Failed to download and extract {addon.name}: {e}") diff --git a/python/tests/test_addons.py b/python/tests/test_addons.py index d448e899..b5f6b484 100644 --- a/python/tests/test_addons.py +++ b/python/tests/test_addons.py @@ -12,11 +12,13 @@ Run with: """ import os +import threading +import time import pytest from camoufox import addons as addons_mod -from camoufox.addons import DefaultAddons, maybe_download_addons +from camoufox.addons import DefaultAddons, add_default_addons, maybe_download_addons UBO = DefaultAddons.UBO.name @@ -89,3 +91,89 @@ def test_failed_download_removes_partial_dir(addons_dir, monkeypatch): # trusting an addon that has no manifest.json. assert not path.exists() assert out == [] + + +def test_concurrent_launches_download_once(addons_dir, monkeypatch): + # Two launches in one process share the check-then-extract. A lock created + # per call guarded nothing, so both saw no manifest and both downloaded. + calls = [] + + def slow(url, extract_path, name): + calls.append(name) + time.sleep(0.2) + _write_manifest(url, extract_path, name) + + monkeypatch.setattr(addons_mod, "download_and_extract", slow) + + outs = [[], []] + threads = [threading.Thread(target=add_default_addons, args=(out,)) for out in outs] + for t in threads: + t.start() + for t in threads: + t.join() + + assert calls == [UBO] + assert outs == [[str(addons_dir / UBO)]] * 2 + + +def test_extraction_appears_complete_or_not_at_all(tmp_path, monkeypatch): + # Another process checks for manifest.json without the lock, so the addon + # directory must never hold a manifest while its other files are missing. + target = tmp_path / UBO + target.mkdir() + seen = [] + + def fake_unzip(buffer, extract_path, desc=None, bar=True): + seen.append((target / "manifest.json").exists()) + _write_manifest(None, extract_path, UBO) + with open(os.path.join(extract_path, "background.js"), "w") as f: + f.write("") + seen.append((target / "manifest.json").exists()) + + monkeypatch.setattr(addons_mod, "webdl", lambda *a, **k: b"") + monkeypatch.setattr(addons_mod, "unzip", fake_unzip) + + addons_mod.download_and_extract("url", str(target), UBO) + + assert seen == [False, False] + assert sorted(os.listdir(target)) == ["background.js", "manifest.json"] + assert os.listdir(tmp_path) == [UBO] + + +def test_complete_addon_installed_meanwhile_is_kept(tmp_path, monkeypatch): + # Another process can finish installing the addon while this one extracts. + # Its copy may already be loading in a browser, so it must not be deleted. + target = tmp_path / UBO + target.mkdir() + + def fake_unzip(buffer, extract_path, desc=None, bar=True): + _write_manifest(None, extract_path, UBO) + _write_manifest(None, str(target), UBO) + with open(target / "theirs.js", "w") as f: + f.write("") + + monkeypatch.setattr(addons_mod, "webdl", lambda *a, **k: b"") + monkeypatch.setattr(addons_mod, "unzip", fake_unzip) + + addons_mod.download_and_extract("url", str(target), UBO) + + assert sorted(os.listdir(target)) == ["manifest.json", "theirs.js"] + assert os.listdir(tmp_path) == [UBO] + + +def test_failed_download_keeps_complete_addon(addons_dir, monkeypatch): + # This launch's download fails after another process installed the addon. + # That copy may already be loading in a browser, so it must not be deleted. + path = addons_dir / UBO + + def fail(url, extract_path, name): + _write_manifest(url, str(path), name) + raise RuntimeError("network died mid-download") + + monkeypatch.setattr(addons_mod, "download_and_extract", fail) + + out = [] + maybe_download_addons([DefaultAddons.UBO], out) + + assert (path / "manifest.json").exists() + assert out == [] diff --git a/typescript/src/addons.ts b/typescript/src/addons.ts index 15cd2740..63d9de51 100644 --- a/typescript/src/addons.ts +++ b/typescript/src/addons.ts @@ -21,6 +21,10 @@ export type DefaultAddon = keyof typeof DefaultAddons; // Addons are stored in a shared folder, not per-browser version const ADDONS_DIR: string = path.join(INSTALL_DIR, "addons"); +// One chain for the module: concurrent launches in one process would otherwise +// interleave the same check-then-extract at its awaits. +let extractChain: Promise = Promise.resolve(); + /** * Confirms that the addon paths are valid. */ @@ -50,11 +54,17 @@ export async function addDefaultAddons( addons[name] = url; } } - await maybeDownloadAddons(addons, addonsList); + const run = extractChain.then(() => maybeDownloadAddons(addons, addonsList)); + extractChain = run.catch(() => undefined); + await run; } /** * Downloads and extracts an addon from a given URL to a specified path. + * + * Extracts into a sibling temp dir and renames it into place last, so a + * concurrent launch (another process, which the chain cannot see) never finds + * a manifest.json in a directory whose other files are still being written. */ export async function downloadAndExtract( url: string, @@ -62,7 +72,24 @@ export async function downloadAndExtract( name: string, ): Promise { const buffer = await webdl(url, `Downloading addon (${name})`, false); - unzip(buffer, extractPath, `Extracting addon (${name})`, false); + const tmpDir = fs.mkdtempSync( + path.join(path.dirname(extractPath), `.${name}.tmp-`), + ); + try { + unzip(buffer, tmpDir, `Extracting addon (${name})`, false); + try { + fs.renameSync(tmpDir, extractPath); + } catch { + // The target is not empty (or exists at all, on Windows). A complete + // copy came from another process and may be loading in its browser, + // so keep it; only an incomplete one is replaced. + if (fs.existsSync(path.join(extractPath, "manifest.json"))) return; + fs.rmSync(extractPath, { recursive: true }); + fs.renameSync(tmpDir, extractPath); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } } /** @@ -102,8 +129,11 @@ export async function maybeDownloadAddons( await addonsDeps.downloadAndExtract(url, addonPath, addonName); addonsList?.push(addonPath); } catch (e) { - // Drop the partial directory so the next run re-downloads. - fs.rmSync(addonPath, { recursive: true, force: true }); + // Drop the partial directory so the next run re-downloads. A complete + // one came from another process and may be loading in its browser. + if (!fs.existsSync(path.join(addonPath, "manifest.json"))) { + fs.rmSync(addonPath, { recursive: true, force: true }); + } console.log(`Failed to download and extract ${addonName}: ${e}`); } } diff --git a/typescript/tests/addons.test.ts b/typescript/tests/addons.test.ts index 3d23746c..56ac715a 100644 --- a/typescript/tests/addons.test.ts +++ b/typescript/tests/addons.test.ts @@ -81,3 +81,88 @@ it("removes the partial directory when a download fails", async () => { ); log.mockRestore(); }); + +it("downloads once for concurrent launches", async () => { + // Two launches in one process share the check-then-extract; unserialized, + // both saw no manifest at the await and both downloaded. + const calls: string[] = []; + addons.addonsDeps.downloadAndExtract = async (_url, extractPath, name) => { + calls.push(name); + await new Promise((resolve) => setTimeout(resolve, 50)); + writeManifest(extractPath); + }; + + const outs: string[][] = [[], []]; + await Promise.all(outs.map((out) => addons.addDefaultAddons(out))); + + expect(calls).toEqual(["UBO"]); + expect(outs).toEqual([[ubo], [ubo]]); +}); + +/** Re-imports addons with webdl stubbed and unzip replaced by `fakeUnzip`. */ +async function withUnzip(fakeUnzip: (extractPath: string) => void) { + vi.resetModules(); + vi.doMock("../src/pkgman.js", async (importOriginal) => ({ + ...(await importOriginal()), + webdl: async () => Buffer.alloc(0), + unzip: (_buffer: Buffer, extractPath: string) => fakeUnzip(extractPath), + })); + const mod: Addons = await import("../src/addons.js"); + vi.doUnmock("../src/pkgman.js"); + return mod; +} + +it("extracts so the addon appears complete or not at all", async () => { + // Another process checks for manifest.json without the chain, so the addon + // directory must never hold a manifest while its other files are missing. + fs.mkdirSync(ubo, { recursive: true }); + const seen: boolean[] = []; + const mod = await withUnzip((extractPath) => { + seen.push(fs.existsSync(path.join(ubo, "manifest.json"))); + writeManifest(extractPath); + fs.writeFileSync(path.join(extractPath, "background.js"), ""); + seen.push(fs.existsSync(path.join(ubo, "manifest.json"))); + }); + + await mod.downloadAndExtract("url", ubo, "UBO"); + + expect(seen).toEqual([false, false]); + expect(fs.readdirSync(ubo).sort()).toEqual([ + "background.js", + "manifest.json", + ]); + expect(fs.readdirSync(path.dirname(ubo))).toEqual(["UBO"]); +}); + +it("keeps a complete addon installed meanwhile", async () => { + // Another process can finish installing the addon while this one extracts. + // Its copy may already be loading in a browser, so it must not be deleted. + fs.mkdirSync(ubo, { recursive: true }); + const mod = await withUnzip((extractPath) => { + writeManifest(extractPath); + writeManifest(ubo); + fs.writeFileSync(path.join(ubo, "theirs.js"), ""); + }); + + await mod.downloadAndExtract("url", ubo, "UBO"); + + expect(fs.readdirSync(ubo).sort()).toEqual(["manifest.json", "theirs.js"]); + expect(fs.readdirSync(path.dirname(ubo))).toEqual(["UBO"]); +}); + +it("keeps a complete addon when a download fails", async () => { + // This launch's download fails after another process installed the addon. + // That copy may already be loading in a browser, so it must not be deleted. + addons.addonsDeps.downloadAndExtract = async () => { + writeManifest(ubo); + throw new Error("network died mid-download"); + }; + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + const out: string[] = []; + await addons.maybeDownloadAddons({ ...addons.DefaultAddons }, out); + + expect(fs.existsSync(path.join(ubo, "manifest.json"))).toBe(true); + expect(out).toEqual([]); + log.mockRestore(); +});