From f36390a19ebe21a5b082cf2f7817ef1f988464e0 Mon Sep 17 00:00:00 2001 From: Jake Writer Date: Wed, 30 Sep 2026 20:02:52 +0000 Subject: [PATCH] feat(geoip): make GeoIP AIO the default source, deprecate GeoLite2 (#820) The default GeoIP source was MaxMind GeoLite2 via sapics/ip-location-db, whose URLs kept serving the 2026-06-17 build after that project moved to GitHub Releases (found in #815). GeoIP AIO (daijro/geoip-all-in-one) resolves timezones more accurately on real proxy IPs and is rebuilt weekly. - repos.yml: AIO is the default; GeoLite2 is `deprecated: true`, with the Releases URLs from #815 so it still works when picked by name. - A cache holding a deprecated source it was not explicitly given (`camoufox set --geoip` or the GUI) moves to the default and drops the old database. An explicit choice is kept, with a FutureWarning. - needs_update() reads the database's build date instead of the file age: refresh once the build is over 8 days old, re-checking at most daily, and warn when a fresh download is over 30 days old (a frozen source). - get_geolocation(geoip_db=...) now reads that source's own database rather than the active one's, and no longer makes it the active one. - tests/test_geoip_sources.py (from #815) downloads every non-deprecated source and fails when its build is stale; tests.yml installs the geoip extra so it runs, and so gates every release. - TypeScript twin updated to match; goldens answer in both layouts. Co-authored-by: lp177 <57773165+lp177@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .github/workflows/tests.yml | 4 +- pythonlib/README.md | 6 +- pythonlib/camoufox/__main__.py | 14 +- pythonlib/camoufox/geolocation.py | 134 ++++++++++++++---- pythonlib/camoufox/gui/backend.py | 2 +- pythonlib/camoufox/repos.yml | 44 +++--- pythonlib/camoufox/utils.py | 4 +- pythonlib/camoufox/warnings.yml | 10 ++ pythonlib/tests/test_geoip_config.py | 98 +++++++++++++ pythonlib/tests/test_geoip_sources.py | 43 ++++++ typescript/scripts/golden/launch_golden.py | 11 +- typescript/src/__main__.ts | 8 +- typescript/src/geolocation.ts | 151 ++++++++++++++++++--- typescript/src/utils.ts | 2 +- typescript/tests/geoip-config.test.ts | 115 ++++++++++++++++ typescript/tests/launch-golden.test.ts | 1 + 16 files changed, 573 insertions(+), 74 deletions(-) create mode 100644 pythonlib/tests/test_geoip_config.py create mode 100644 pythonlib/tests/test_geoip_sources.py create mode 100644 typescript/tests/geoip-config.test.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0dd3ee..323e43a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -303,7 +303,9 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - run: | - pip install -r ci/requirements.txt pytest -e pythonlib + # The geoip extra: tests/test_geoip_sources.py downloads every GeoIP + # source and fails when one stops publishing, which gates releases. + pip install -r ci/requirements.txt pytest -e 'pythonlib[geoip]' # fpgen downloads its model on first import with TLS verification # OFF and no checksum, and its release picker can only ever reach the # April-2025 model. Install the pinned one first: see diff --git a/pythonlib/README.md b/pythonlib/README.md index 6684281..f1c24ea 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -31,6 +31,8 @@ pip install -U camoufox[geoip] The `geoip` parameter is optional, but heavily recommended if you are using proxies. It will download an extra dataset to determine the user's longitude, latitude, timezone, country, & locale. +The dataset is [GeoIP All-in-One](https://github.com/daijro/geoip-all-in-one), which merges several IP databases and is rebuilt weekly. Camoufox fetches the newest build and refreshes it once it is a week old. + Next, download the Camoufox browser: ```bash @@ -266,12 +268,12 @@ Browser Latest in official/stable? Yes Last Sync 2026-03-07 00:23 GeoIP - Database MaxMind GeoLite2 + Database GeoIP AIO by daijro Updated 2026-03-07 00:24 Storage Install path /home/name/.cache/camoufox Browser(s) directory size 1.2 GB - GeoIP database size 40.7 MB + GeoIP database size 116.4 MB Config file /home/name/.cache/camoufox/config.json Repo cache /home/name/.cache/camoufox/repo_cache.json ``` diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index 9b47059..047df93 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -20,6 +20,7 @@ from .geolocation import ( get_mmdb_path, load_geoip_config, save_geoip_config, + warn_if_deprecated, ) from .multiversion import ( BROWSERS_DIR, @@ -666,14 +667,23 @@ def _select_geoip_source(): return current = load_geoip_config().get("name", "") - choices = [(r["name"] + (" [active]" if r.get("name") == current else ""), r) for r in repos] + choices = [ + ( + r["name"] + + (" (deprecated)" if r.get("deprecated") else "") + + (" [active]" if r.get("name") == current else ""), + r, + ) + for r in repos + ] selected = _inquirer_select(choices, "Select GeoIP source") if not selected: return - save_geoip_config(selected) + save_geoip_config(selected, explicit=True) rprint(f"GeoIP source: {selected['name']}", fg="green") + warn_if_deprecated(selected) @cli.command(name="list") diff --git a/pythonlib/camoufox/geolocation.py b/pythonlib/camoufox/geolocation.py index d4c1eca..9b71663 100644 --- a/pythonlib/camoufox/geolocation.py +++ b/pythonlib/camoufox/geolocation.py @@ -2,8 +2,10 @@ Helpers to fetch geolocation, timezone, and locale data given an IP """ +import os import shutil import tempfile +import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, cast @@ -12,6 +14,7 @@ from yaml import CDumper, CLoader from yaml import dump as yaml_dump from yaml import load as yaml_load +from ._warnings import WARNINGS_DATA, _warn_from_caller from .exceptions import NotInstalledGeoIPExtra, UnknownIPLocation from .ip import validate_ip from .locales import SELECTOR, Geolocation @@ -29,6 +32,15 @@ GEOIP_DIR = Path(user_cache_dir("camoufox")) / "geoip" MMDB_DIR = GEOIP_DIR / "mmdb" GEOIP_CONFIG = GEOIP_DIR / "config.yml" +# A database whose data was built longer ago than this is refreshed. The +# default source publishes weekly, so a week and a day catches every release. +UPDATE_DAYS = 8 +# ...but at most once a day, so a source that stops publishing costs one +# download a day rather than one per launch. +RECHECK_DAYS = 1 +# A freshly downloaded build older than this means its source is frozen. +FROZEN_DAYS = 30 + def _find_in(data: Dict, key: str) -> Any: """ @@ -50,7 +62,7 @@ def _load_geoip_repos() -> Tuple[List[Dict], str]: with open(LOCAL_DATA / 'repos.yml', 'r') as f: data = yaml_load(f, Loader=CLoader) geoip_repos = data.get('geoip', []) - default_name = data.get('default', {}).get('geoip', 'GeoLite2') + default_name = data.get('default', {}).get('geoip', 'GeoIP AIO by daijro') return geoip_repos, default_name @@ -81,27 +93,49 @@ def _get_geoip_config_by_name(name: Optional[str] = None) -> Dict: raise ValueError("No GeoIP repos configured in repos.yml") +def warn_if_deprecated(config: Dict) -> None: + """ + Warn that a GeoIP source is deprecated and name the default to use instead + """ + if config.get('deprecated'): + _, default_name = _load_geoip_repos() + _warn_from_caller( + WARNINGS_DATA['geoip_deprecated'].format(name=config['name'], default=default_name), + FutureWarning, + ) + + def load_geoip_config() -> Dict: """ - Load active GeoIP config from disk, falling back to repos.yml default + Load active GeoIP config from disk, falling back to repos.yml default. + + A saved deprecated source that the user did not pick explicitly (every + cache written before the default changed) resolves to the default. """ if GEOIP_CONFIG.exists(): with open(GEOIP_CONFIG, 'r') as f: - saved = yaml_load(f, Loader=CLoader) + saved = yaml_load(f, Loader=CLoader) or {} try: - return _get_geoip_config_by_name(saved.get('name')) + config = _get_geoip_config_by_name(saved.get('name')) except (ValueError, KeyError): return saved + if config.get('deprecated') and not saved.get('explicit'): + return _get_geoip_config_by_name(None) + return config return _get_geoip_config_by_name(None) -def save_geoip_config(config: Dict) -> None: +def save_geoip_config(config: Dict, explicit: bool = False) -> None: """ - Save active GeoIP source name to disk + Save active GeoIP source name to disk. `explicit` records that the user + chose it, so a later default change does not move them off it. """ GEOIP_DIR.mkdir(parents=True, exist_ok=True) + saved: Dict[str, Any] = {'name': config['name']} + if explicit: + saved['explicit'] = True with open(GEOIP_CONFIG, 'w') as f: - yaml_dump({'name': config['name']}, f, Dumper=CDumper, default_flow_style=False) + yaml_dump(saved, f, Dumper=CDumper, default_flow_style=False) def get_mmdb_path(ip_version: str = 'ipv4', config: Optional[Dict] = None) -> Path: @@ -130,9 +164,11 @@ def geoip_allowed() -> None: def download_mmdb( source: Optional[str] = None, progress_callback: Optional[callable] = None, + activate: bool = True, ) -> None: """ - Downloads the GeoIP database(s) to geoip/mmdb/ + Downloads the GeoIP database(s) to geoip/mmdb/. A named `source` becomes + the user's explicit choice unless `activate` is False. """ geoip_allowed() @@ -180,6 +216,8 @@ def download_mmdb( tmp.seek(0) with open(mmdb_path, 'wb') as dst: shutil.copyfileobj(tmp, dst) + # The mtime is when we last checked, which needs_update() throttles on + os.utime(mmdb_path) break except Exception as e: last_error = e @@ -187,7 +225,40 @@ def download_mmdb( else: raise last_error or Exception(f"Failed to download {ip_ver}") - save_geoip_config(config) + age = _build_age_days(mmdb_path) + if age is not None and age > FROZEN_DAYS: + _warn_from_caller( + WARNINGS_DATA['geoip_frozen'].format(name=config['name'], days=int(age)), + RuntimeWarning, + ) + + _remove_deprecated_databases(keep=config) + if activate: + # A refresh of the active source keeps whether the user chose it + save_geoip_config(config, explicit=bool(source) or _chosen_explicitly(config)) + + +def _chosen_explicitly(config: Dict) -> bool: + """ + Whether the saved config names this source as the user's explicit choice + """ + if not GEOIP_CONFIG.exists(): + return False + with open(GEOIP_CONFIG, 'r') as f: + saved = yaml_load(f, Loader=CLoader) or {} + return bool(saved.get('explicit')) and saved.get('name') == config['name'] + + +def _remove_deprecated_databases(keep: Dict) -> None: + """ + Delete downloaded databases of deprecated sources other than `keep` + """ + repos, _ = _load_geoip_repos() + for repo in repos: + if not repo.get('deprecated') or repo['name'] == keep['name']: + continue + for path in MMDB_DIR.glob(f"{repo['name'].lower()}-*.mmdb"): + path.unlink(missing_ok=True) def remove_mmdb() -> None: @@ -202,24 +273,39 @@ def remove_mmdb() -> None: rprint("GeoIP database removed.") +def _build_age_days(mmdb_path: Path) -> Optional[float]: + """ + Days since the database's data was built, from its metadata + """ + import maxminddb + + try: + with maxminddb.open_database(str(mmdb_path)) as reader: + return (time.time() - reader.metadata().build_epoch) / 86400 + except Exception: + return None + + def needs_update(config: Optional[Dict] = None) -> bool: """ - Check if the GeoIP database needs an update (older than 30 days) - """ - from datetime import datetime, timedelta + Check if the GeoIP database needs an update: its data was built over + UPDATE_DAYS ago and it was last downloaded over RECHECK_DAYS ago. + This reads the build date rather than the file's age, so a source that + keeps serving an old build is noticed. + """ if config is None: config = load_geoip_config() - update_days = 30 - ipv4_path = get_mmdb_path('ipv4', config) if not ipv4_path.exists(): return True - mtime = datetime.fromtimestamp(ipv4_path.stat().st_mtime) - age = datetime.now() - mtime - return age > timedelta(days=update_days) + checked_days = (time.time() - ipv4_path.stat().st_mtime) / 86400 + if checked_days < RECHECK_DAYS: + return False + build_days = _build_age_days(ipv4_path) + return build_days is None or build_days > UPDATE_DAYS def get_geolocation(ip: str, geoip_db: Optional[str] = None) -> Geolocation: @@ -230,16 +316,14 @@ def get_geolocation(ip: str, geoip_db: Optional[str] = None) -> Geolocation: validate_ip(ip) ip_version = 'ipv6' if ':' in ip else 'ipv4' - mmdb_path = get_mmdb_path(ip_version) - if not mmdb_path.exists() or needs_update(): - download_mmdb() - mmdb_path = get_mmdb_path(ip_version) + # A per-call geoip_db reads its own database without becoming the active one + config = _get_geoip_config_by_name(geoip_db) if geoip_db else load_geoip_config() + warn_if_deprecated(config) + mmdb_path = get_mmdb_path(ip_version, config) - if geoip_db: - config = _get_geoip_config_by_name(geoip_db) - else: - config = load_geoip_config() + if not mmdb_path.exists() or needs_update(config): + download_mmdb(geoip_db, activate=not geoip_db) paths = config['paths'] with maxminddb.open_database(str(mmdb_path)) as reader: diff --git a/pythonlib/camoufox/gui/backend.py b/pythonlib/camoufox/gui/backend.py index 7f2cfe4..794ad8a 100644 --- a/pythonlib/camoufox/gui/backend.py +++ b/pythonlib/camoufox/gui/backend.py @@ -861,7 +861,7 @@ class Backend(QObject): if source not in self._geoip_downloaded: return - save_geoip_config(_get_geoip_config_by_name(source)) + save_geoip_config(_get_geoip_config_by_name(source), explicit=True) self._load_geoip() @Slot(int) diff --git a/pythonlib/camoufox/repos.yml b/pythonlib/camoufox/repos.yml index 2f75360..7ede658 100644 --- a/pythonlib/camoufox/repos.yml +++ b/pythonlib/camoufox/repos.yml @@ -1,7 +1,7 @@ # Default configurations default: browser: Official - geoip: MaxMind GeoLite2 + geoip: GeoIP AIO by daijro # Browser repositories browsers: @@ -39,23 +39,18 @@ browsers: # Assume all browsers # GeoIP database repositories +# +# Every URL points at the source's newest build, never a pinned one: the +# launcher re-downloads once the build it has is a week old (see +# needs_update() in geolocation.py), and tests/test_geoip_sources.py fails CI +# and the release when a source stops publishing. +# +# `deprecated: true` sources still work when chosen by name, with a warning. +# A cache that holds one without an explicit `camoufox set --geoip` choice +# (every install that predates the switch to AIO) moves to the default. geoip: - # GeoLite2 City - Full city-level data with timezone - - name: MaxMind GeoLite2 - urls: - ipv4: - - https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-city-mmdb/geolite2-city-ipv4.mmdb - - https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/geolite2-city-mmdb/geolite2-city-ipv4.mmdb - ipv6: - - https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-city-mmdb/geolite2-city-ipv6.mmdb - - https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/geolite2-city-mmdb/geolite2-city-ipv6.mmdb - paths: - iso_code: country_code - longitude: longitude - latitude: latitude - timezone: timezone - - # GeoIP All-in-One - Combined IPv4/IPv6 with all fields + # GeoIP All-in-One - Combined IPv4/IPv6 with all fields. Merges several + # databases per range; rebuilt weekly by daijro/geoip-all-in-one. - name: GeoIP AIO by daijro extract: true urls: @@ -66,3 +61,18 @@ geoip: longitude: location.longitude latitude: location.latitude timezone: location.time_zone + + # GeoLite2 City via sapics/ip-location-db, which publishes to GitHub + # Releases since 2026-06-18 (URLs from daijro/camoufox#815). + - name: MaxMind GeoLite2 + deprecated: true + urls: + ipv4: + - https://github.com/sapics/ip-location-db/releases/download/latest/geolite2-city-ipv4.mmdb + ipv6: + - https://github.com/sapics/ip-location-db/releases/download/latest/geolite2-city-ipv6.mmdb + paths: + iso_code: country_code + longitude: longitude + latitude: latitude + timezone: timezone diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index 3574dca..967bde4 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -878,8 +878,8 @@ def launch_options( Calculate longitude, latitude, timezone, country, & locale based on the IP address. Pass the target IP address to use, or `True` to find the IP address automatically. geoip_db (Optional[str]): - Name of the GeoIP database to use (e.g., "MaxMind"). - If not specified, uses the configured default. + Name of the GeoIP database to use (e.g., "GeoIP AIO by daijro"). + If not specified, uses the one chosen with `camoufox set --geoip`, or the default. humanize (Optional[Union[bool, float]]): Humanize the cursor movement. Takes either `True`, or the MAX duration in seconds of the cursor movement. diff --git a/pythonlib/camoufox/warnings.yml b/pythonlib/camoufox/warnings.yml index 91d9e73..c24d154 100644 --- a/pythonlib/camoufox/warnings.yml +++ b/pythonlib/camoufox/warnings.yml @@ -75,3 +75,13 @@ fallback: |- Please report this at https://github.com/daijro/camoufox/issues/new and include: {report} + +geoip_deprecated: >- + The GeoIP source "{name}" is deprecated and will be removed in a future release. + "{default}" is the default and resolves locations more accurately. + Run `camoufox set --geoip` to switch, or stop passing `geoip_db`. + +geoip_frozen: >- + The newest "{name}" GeoIP database was built {days} days ago, so its source may have + stopped publishing and locations may be out of date. Please report this at + https://github.com/daijro/camoufox/issues. diff --git a/pythonlib/tests/test_geoip_config.py b/pythonlib/tests/test_geoip_config.py new file mode 100644 index 0000000..bdaf038 --- /dev/null +++ b/pythonlib/tests/test_geoip_config.py @@ -0,0 +1,98 @@ +"""Which GeoIP source is active, and when its database is refreshed.""" + +import os +import time +import warnings + +import pytest +from yaml import safe_dump + +from camoufox import geolocation + +DEFAULT = "GeoIP AIO by daijro" +DEPRECATED = "MaxMind GeoLite2" + + +@pytest.fixture(autouse=True) +def cache(tmp_path, monkeypatch): + monkeypatch.setattr(geolocation, "GEOIP_DIR", tmp_path) + monkeypatch.setattr(geolocation, "MMDB_DIR", tmp_path / "mmdb") + monkeypatch.setattr(geolocation, "GEOIP_CONFIG", tmp_path / "config.yml") + return tmp_path + + +def _saved(cache, **fields): + (cache / "config.yml").write_text(safe_dump(fields)) + + +def test_default_is_aio(): + assert geolocation.load_geoip_config()["name"] == DEFAULT + repos, _ = geolocation._load_geoip_repos() + assert [r["name"] for r in repos if r.get("deprecated")] == [DEPRECATED] + + +def test_implicit_deprecated_source_moves_to_default(cache): + # What every cache written before the default changed holds + _saved(cache, name=DEPRECATED) + assert geolocation.load_geoip_config()["name"] == DEFAULT + + +def test_explicit_deprecated_source_is_kept(cache): + _saved(cache, name=DEPRECATED, explicit=True) + assert geolocation.load_geoip_config()["name"] == DEPRECATED + + +def test_save_records_explicit_choice(cache): + config = geolocation._get_geoip_config_by_name(DEPRECATED) + geolocation.save_geoip_config(config, explicit=True) + assert geolocation.load_geoip_config()["name"] == DEPRECATED + geolocation.save_geoip_config(config) + assert geolocation.load_geoip_config()["name"] == DEFAULT + + +def test_refresh_keeps_an_explicit_choice(cache, monkeypatch): + # `camoufox fetch` and the weekly refresh download without naming a source + _saved(cache, name=DEPRECATED, explicit=True) + monkeypatch.setattr(geolocation, "webdl", lambda url, buffer, **kw: buffer.write(b"x")) + monkeypatch.setattr(geolocation, "_build_age_days", lambda path: 1) + geolocation.download_mmdb() + assert geolocation.load_geoip_config()["name"] == DEPRECATED + + +def test_deprecated_source_warns(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + geolocation.warn_if_deprecated(geolocation._get_geoip_config_by_name(DEPRECATED)) + geolocation.warn_if_deprecated(geolocation._get_geoip_config_by_name(DEFAULT)) + assert [w.category for w in caught] == [FutureWarning] + assert DEFAULT in str(caught[0].message) + + +def _database(cache, checked_days_ago): + config = geolocation.load_geoip_config() + path = geolocation.get_mmdb_path("ipv4", config) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"") + stamp = time.time() - checked_days_ago * 86400 + os.utime(path, (stamp, stamp)) + return config + + +@pytest.mark.parametrize( + "checked, built, stale", + [ + (0.5, 400, False), # checked today: wait, even for an old build + (2, 3, False), # this week's build + (2, 9, True), # a release has been missed + (40, 3, False), # an old file holding a new build is not stale + (2, None, True), # unreadable database + ], +) +def test_needs_update_reads_the_build_date(cache, monkeypatch, checked, built, stale): + config = _database(cache, checked) + monkeypatch.setattr(geolocation, "_build_age_days", lambda path: built) + assert geolocation.needs_update(config) is stale + + +def test_missing_database_needs_update(): + assert geolocation.needs_update() is True diff --git a/pythonlib/tests/test_geoip_sources.py b/pythonlib/tests/test_geoip_sources.py new file mode 100644 index 0000000..655d416 --- /dev/null +++ b/pythonlib/tests/test_geoip_sources.py @@ -0,0 +1,43 @@ +"""Every GeoIP source camoufox offers must be serving a current build. + +The launcher always downloads a source's newest build, so a source that stops +publishing leaves every user on stale data with no error: GeoLite2's URLs +served the 2026-06-17 build for months after sapics/ip-location-db moved to +GitHub Releases (found in daijro/camoufox#815, which this test comes from). +This suite runs in tests.yml, which gates every release. +""" + +import time + +import pytest + +from camoufox import geolocation + +maxminddb = pytest.importorskip("maxminddb") + +MAX_AGE_DAYS = 30 + +SOURCES = [repo["name"] for repo in geolocation._load_geoip_repos()[0] if not repo.get("deprecated")] + + +@pytest.mark.parametrize("name", SOURCES) +def test_source_serves_a_current_build(name, tmp_path, monkeypatch): + monkeypatch.setattr(geolocation, "GEOIP_DIR", tmp_path) + monkeypatch.setattr(geolocation, "MMDB_DIR", tmp_path / "mmdb") + monkeypatch.setattr(geolocation, "GEOIP_CONFIG", tmp_path / "config.yml") + + geolocation.download_mmdb(name) + + databases = sorted((tmp_path / "mmdb").glob("*.mmdb")) + assert databases, f"{name} downloaded no database" + for database in databases: + with maxminddb.open_database(str(database)) as reader: + age_days = (time.time() - reader.metadata().build_epoch) / 86400 + record = reader.get("8.8.8.8") + assert age_days < MAX_AGE_DAYS, ( + f"{name}: {database.name} was built {age_days:.0f} days ago, its source is no longer updated" + ) + config = geolocation._get_geoip_config_by_name(name) + assert geolocation._find_in(record, config["paths"]["iso_code"]) == "US", ( + f"{name}: the configured paths do not read this database" + ) diff --git a/typescript/scripts/golden/launch_golden.py b/typescript/scripts/golden/launch_golden.py index c81c6a6..5b32660 100644 --- a/typescript/scripts/golden/launch_golden.py +++ b/typescript/scripts/golden/launch_golden.py @@ -56,6 +56,15 @@ GEO_TABLE = { '2a01:4f8::1': {'country_code': 'DE', 'longitude': 9.491, 'latitude': 51.2993, 'timezone': 'Europe/Berlin'}, '203.0.113.7': {'country_code': 'JP', 'longitude': 139.6899, 'latitude': 35.6893, 'timezone': 'Asia/Tokyo'}, } +# Each record answers in both layouts: GeoLite2's flat fields and the nested +# ones the default source (GeoIP AIO) is read through. +for _rec in GEO_TABLE.values(): + _rec['country'] = {'iso_code': _rec['country_code']} + _rec['location'] = { + 'longitude': _rec['longitude'], + 'latitude': _rec['latitude'], + 'time_zone': _rec['timezone'], + } _fake_mmdb = types.ModuleType('maxminddb') @@ -88,7 +97,7 @@ from camoufox.utils import launch_options # noqa: E402 assert str(utils.INSTALL_DIR) == str(CACHE), (utils.INSTALL_DIR, CACHE) # The mmdb files only have to exist and be fresh; the fake reader answers. -for name in ('maxmind geolite2-ipv4.mmdb', 'maxmind geolite2-ipv6.mmdb'): +for name in ('geoip aio by daijro-combined.mmdb', 'maxmind geolite2-ipv4.mmdb', 'maxmind geolite2-ipv6.mmdb'): p = geolocation.MMDB_DIR / name p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(b'') diff --git a/typescript/src/__main__.ts b/typescript/src/__main__.ts index 38d4e6f..49cb2a0 100644 --- a/typescript/src/__main__.ts +++ b/typescript/src/__main__.ts @@ -25,6 +25,7 @@ import { getMmdbPath, loadGeoipConfig, saveGeoipConfig, + warnIfDeprecated, } from "./geolocation.js"; import { BROWSERS_DIR, @@ -886,15 +887,18 @@ async function selectGeoIPSource(): Promise { // unreadable config: nothing is marked active } const choices: Array<[string, GeoIPRepo]> = repos.map((r) => [ - r.name + (r.name === current ? " [active]" : ""), + r.name + + (r.deprecated ? " (deprecated)" : "") + + (r.name === current ? " [active]" : ""), r, ]); const selected = await select(choices, "Select GeoIP source"); if (!selected) return; - saveGeoipConfig(selected); + saveGeoipConfig(selected, true); rprint(`GeoIP source: ${selected.name}`, "green"); + warnIfDeprecated(selected); } program diff --git a/typescript/src/geolocation.ts b/typescript/src/geolocation.ts index 47083f1..806e1d9 100644 --- a/typescript/src/geolocation.ts +++ b/typescript/src/geolocation.ts @@ -14,22 +14,36 @@ import { NotInstalledGeoIPExtra, UnknownIPLocation } from "./exceptions.js"; import { validateIP } from "./ip.js"; import { Geolocation, SELECTOR } from "./locales.js"; import { INSTALL_DIR, LOCAL_DATA } from "./paths.js"; +import { loadWarnings, warn } from "./warnings.js"; export const GEOIP_DIR: string = path.join(INSTALL_DIR, "geoip"); export const MMDB_DIR: string = path.join(GEOIP_DIR, "mmdb"); export const GEOIP_CONFIG: string = path.join(GEOIP_DIR, "config.yml"); +/** A database whose data was built longer ago than this is refreshed. The + * default source publishes weekly, so a week and a day catches every release. */ +export const UPDATE_DAYS = 8; +/** ...but at most once a day, so a source that stops publishing costs one + * download a day rather than one per launch. */ +export const RECHECK_DAYS = 1; +/** A freshly downloaded build older than this means its source is frozen. */ +export const FROZEN_DAYS = 30; + +const DAY_MS = 24 * 60 * 60 * 1000; + export interface GeoIPRepo { name: string; urls: Record; paths: Record; extract?: boolean; + deprecated?: boolean; [key: string]: any; } -/** A reader over an mmdb file: maxminddb.Reader's `get`. */ +/** A reader over an mmdb file: maxminddb.Reader's `get` and `metadata`. */ export interface MmdbReader { get(ip: string): any; + metadata?: { buildEpoch: Date }; close?(): void; } @@ -58,7 +72,8 @@ export const geoipDeps = { const buffer = fs.readFileSync(mmdbPath); return new maxmind.Reader(buffer); }, - downloadMmdb: (source?: string) => downloadMmdb(source), + downloadMmdb: (source?: string) => + downloadMmdb(source, undefined, source === undefined), }; /** @@ -86,7 +101,7 @@ function loadGeoipRepos(): [GeoIPRepo[], string] { fs.readFileSync(path.join(LOCAL_DATA, "repos.yml"), "utf-8"), ) as Record) ?? {}; const geoipRepos: GeoIPRepo[] = data.geoip ?? []; - const defaultName: string = data.default?.geoip ?? "GeoLite2"; + const defaultName: string = data.default?.geoip ?? "GeoIP AIO by daijro"; return [geoipRepos, defaultName]; } @@ -127,8 +142,26 @@ export function getGeoipConfigByName(name?: string | null): GeoIPRepo { throw new Error("No GeoIP repos configured in repos.yml"); } +/** + * Warn that a GeoIP source is deprecated and name the default to use instead. + */ +export function warnIfDeprecated(config: GeoIPRepo): void { + if (config.deprecated) { + const [, defaultName] = loadGeoipRepos(); + warn( + loadWarnings() + .geoip_deprecated.replace("{name}", config.name) + .replace("{default}", defaultName), + "FutureWarning", + ); + } +} + /** * Load the active GeoIP config from disk, falling back to the repos.yml default. + * + * A saved deprecated source that the user did not pick explicitly (every + * cache written before the default changed) resolves to the default. */ export function loadGeoipConfig(): GeoIPRepo { if (fs.existsSync(GEOIP_CONFIG)) { @@ -137,21 +170,31 @@ export function loadGeoipConfig(): GeoIPRepo { string, any >) ?? {}; + let config: GeoIPRepo; try { - return getGeoipConfigByName(saved.name); + config = getGeoipConfigByName(saved.name); } catch { return saved as GeoIPRepo; } + if (config.deprecated && !saved.explicit) { + return getGeoipConfigByName(undefined); + } + return config; } return getGeoipConfigByName(undefined); } /** - * Save the active GeoIP source name to disk. + * Save the active GeoIP source name to disk. `explicit` records that the user + * chose it, so a later default change does not move them off it. */ -export function saveGeoipConfig(config: GeoIPRepo): void { +export function saveGeoipConfig(config: GeoIPRepo, explicit = false): void { fs.mkdirSync(GEOIP_DIR, { recursive: true }); - fs.writeFileSync(GEOIP_CONFIG, stringifyYaml({ name: config.name })); + const saved: Record = { name: config.name }; + if (explicit) { + saved.explicit = true; + } + fs.writeFileSync(GEOIP_CONFIG, stringifyYaml(saved)); } /** @@ -182,11 +225,13 @@ export function geoipAllowed(): void { } /** - * Downloads the GeoIP database(s) to geoip/mmdb/. + * Downloads the GeoIP database(s) to geoip/mmdb/. A named `source` becomes + * the user's explicit choice unless `activate` is false. */ export async function downloadMmdb( source?: string, progressCallback?: (downloaded: number, total: number) => void, + activate = true, ): Promise { geoipAllowed(); const { unzip, webdl } = await import("./pkgman.js"); @@ -233,6 +278,9 @@ export async function downloadMmdb( } else { fs.writeFileSync(mmdbPath, buffer); } + // The mtime is when we last checked, which needsUpdate() throttles on + const now = new Date(); + fs.utimesSync(mmdbPath, now, now); done = true; break; } catch (error) { @@ -244,9 +292,64 @@ export async function downloadMmdb( if (!done) { throw lastError ?? new Error(`Failed to download ${ipVer}`); } + + const age = await buildAgeDays(mmdbPath); + if (age !== null && age > FROZEN_DAYS) { + warn( + loadWarnings() + .geoip_frozen.replace("{name}", config.name) + .replace("{days}", String(Math.trunc(age))), + "RuntimeWarning", + ); + } } - saveGeoipConfig(config); + removeDeprecatedDatabases(config); + if (activate) { + // A refresh of the active source keeps whether the user chose it + saveGeoipConfig(config, Boolean(source) || chosenExplicitly(config)); + } +} + +/** Whether the saved config names this source as the user's explicit choice. */ +function chosenExplicitly(config: GeoIPRepo): boolean { + if (!fs.existsSync(GEOIP_CONFIG)) return false; + const saved = + (parseYaml(fs.readFileSync(GEOIP_CONFIG, "utf-8")) as Record< + string, + any + >) ?? {}; + return Boolean(saved.explicit) && saved.name === config.name; +} + +/** Delete downloaded databases of deprecated sources other than `keep`. */ +function removeDeprecatedDatabases(keep: GeoIPRepo): void { + const [repos] = loadGeoipRepos(); + if (!fs.existsSync(MMDB_DIR)) return; + for (const repo of repos) { + if (!repo.deprecated || repo.name === keep.name) continue; + const prefix = `${repo.name.toLowerCase()}-`; + for (const file of fs.readdirSync(MMDB_DIR)) { + if (file.startsWith(prefix) && file.endsWith(".mmdb")) { + fs.rmSync(path.join(MMDB_DIR, file), { force: true }); + } + } + } +} + +/** Days since the database's data was built, from its metadata. */ +async function buildAgeDays(mmdbPath: string): Promise { + try { + const reader = await geoipDeps.openDatabase(mmdbPath); + try { + const built = reader.metadata?.buildEpoch; + return built ? (Date.now() - built.getTime()) / DAY_MS : null; + } finally { + reader.close?.(); + } + } catch { + return null; + } } /** Path(tmpdir).rglob('*.mmdb')[0] */ @@ -261,18 +364,25 @@ function findFirstMmdb(dir: string): string | null { } /** - * Check if the GeoIP database needs an update (older than 30 days). + * Check if the GeoIP database needs an update: its data was built over + * UPDATE_DAYS ago and it was last downloaded over RECHECK_DAYS ago. + * + * This reads the build date rather than the file's age, so a source that + * keeps serving an old build is noticed. */ -export function needsUpdate(config?: GeoIPRepo): boolean { +export async function needsUpdate(config?: GeoIPRepo): Promise { const cfg = config ?? loadGeoipConfig(); - const updateDays = 30; const ipv4Path = getMmdbPath("ipv4", cfg); if (!fs.existsSync(ipv4Path)) { return true; } - const age = Date.now() - fs.statSync(ipv4Path).mtimeMs; - return age > updateDays * 24 * 60 * 60 * 1000; + const checkedDays = (Date.now() - fs.statSync(ipv4Path).mtimeMs) / DAY_MS; + if (checkedDays < RECHECK_DAYS) { + return false; + } + const buildDays = await buildAgeDays(ipv4Path); + return buildDays === null || buildDays > UPDATE_DAYS; } /** float(x) for a value read out of the database. */ @@ -298,14 +408,15 @@ export async function getGeolocation( ): Promise { validateIP(ip); const ipVersion = ip.includes(":") ? "ipv6" : "ipv4"; - let mmdbPath = getMmdbPath(ipVersion); - - if (!fs.existsSync(mmdbPath) || needsUpdate()) { - await geoipDeps.downloadMmdb(); - mmdbPath = getMmdbPath(ipVersion); - } + // A per-call geoipDb reads its own database without becoming the active one const config = geoipDb ? getGeoipConfigByName(geoipDb) : loadGeoipConfig(); + warnIfDeprecated(config); + const mmdbPath = getMmdbPath(ipVersion, config); + + if (!fs.existsSync(mmdbPath) || (await needsUpdate(config))) { + await geoipDeps.downloadMmdb(geoipDb); + } const paths = config.paths; const reader = await geoipDeps.openDatabase(mmdbPath); diff --git a/typescript/src/utils.ts b/typescript/src/utils.ts index 609f726..5497543 100644 --- a/typescript/src/utils.ts +++ b/typescript/src/utils.ts @@ -1148,7 +1148,7 @@ export interface LaunchOptions { /** Calculate longitude, latitude, timezone, country, & locale based on the IP * address. Pass the target IP address to use, or `true` to find it. */ geoip?: string | boolean; - /** Name of the GeoIP database to use (e.g. "MaxMind GeoLite2"). */ + /** Name of the GeoIP database to use (e.g. "GeoIP AIO by daijro"). */ geoip_db?: string; /** Humanize the cursor movement: `true`, or the MAX duration in seconds. */ humanize?: boolean | number; diff --git a/typescript/tests/geoip-config.test.ts b/typescript/tests/geoip-config.test.ts new file mode 100644 index 0000000..a9aded1 --- /dev/null +++ b/typescript/tests/geoip-config.test.ts @@ -0,0 +1,115 @@ +/** + * Mirrors pythonlib/tests/test_geoip_config.py: which GeoIP source is active, + * and when its database is refreshed. + */ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { stringify as stringifyYaml } from "yaml"; + +const DEFAULT = "GeoIP AIO by daijro"; +const DEPRECATED = "MaxMind GeoLite2"; +const DAY_MS = 24 * 60 * 60 * 1000; + +let tmp: string; +let g: typeof import("../src/geolocation.js"); +let w: typeof import("../src/warnings.js"); + +beforeEach(async () => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "camoufox-geoip-")); + vi.resetModules(); + // INSTALL_DIR is computed at import time + vi.doMock("../src/paths.js", async (importOriginal) => ({ + ...(await importOriginal()), + INSTALL_DIR: tmp, + })); + g = await import("../src/geolocation.js"); + w = await import("../src/warnings.js"); +}); + +afterEach(() => { + vi.doUnmock("../src/paths.js"); + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +function saved(fields: Record) { + fs.mkdirSync(g.GEOIP_DIR, { recursive: true }); + fs.writeFileSync(g.GEOIP_CONFIG, stringifyYaml(fields)); +} + +it("defaults to AIO", () => { + expect(g.loadGeoipConfig().name).toBe(DEFAULT); +}); + +it("moves an implicit deprecated source to the default", () => { + // What every cache written before the default changed holds + saved({ name: DEPRECATED }); + expect(g.loadGeoipConfig().name).toBe(DEFAULT); +}); + +it("keeps an explicit deprecated source", () => { + saved({ name: DEPRECATED, explicit: true }); + expect(g.loadGeoipConfig().name).toBe(DEPRECATED); +}); + +it("records an explicit choice", () => { + const config = g.getGeoipConfigByName(DEPRECATED); + g.saveGeoipConfig(config, true); + expect(g.loadGeoipConfig().name).toBe(DEPRECATED); + g.saveGeoipConfig(config); + expect(g.loadGeoipConfig().name).toBe(DEFAULT); +}); + +it("keeps an explicit choice through a refresh", async () => { + // `camoufox fetch` and the weekly refresh download without naming a source + saved({ name: DEPRECATED, explicit: true }); + vi.doMock("../src/pkgman.js", async (importOriginal) => ({ + ...(await importOriginal()), + webdl: async () => Buffer.from("x"), + })); + try { + await g.downloadMmdb(); + } finally { + vi.doUnmock("../src/pkgman.js"); + } + expect(g.loadGeoipConfig().name).toBe(DEPRECATED); + // The stubbed download, not a real one + expect(fs.readFileSync(g.getMmdbPath("ipv4"), "utf-8")).toBe("x"); +}); + +it("warns on a deprecated source", async () => { + const { warnings } = await w.recordWarnings(() => { + g.warnIfDeprecated(g.getGeoipConfigByName(DEPRECATED)); + g.warnIfDeprecated(g.getGeoipConfigByName(DEFAULT)); + }); + expect(warnings.map((x) => x.category)).toEqual(["FutureWarning"]); + expect(warnings[0].message).toContain(DEFAULT); +}); + +it.each([ + [0.5, 400, false], // checked today: wait, even for an old build + [2, 3, false], // this week's build + [2, 9, true], // a release has been missed + [40, 3, false], // an old file holding a new build is not stale + [2, null, true], // unreadable database +])("checked %s days ago, built %s days ago: stale=%s", async (checked, built, stale) => { + const config = g.loadGeoipConfig(); + const mmdb = g.getMmdbPath("ipv4", config); + fs.mkdirSync(path.dirname(mmdb), { recursive: true }); + fs.writeFileSync(mmdb, ""); + const stamp = new Date(Date.now() - checked * DAY_MS); + fs.utimesSync(mmdb, stamp, stamp); + g.geoipDeps.openDatabase = async () => { + if (built === null) throw new Error("unreadable"); + return { + get: () => null, + metadata: { buildEpoch: new Date(Date.now() - built * DAY_MS) }, + }; + }; + expect(await g.needsUpdate(config)).toBe(stale); +}); + +it("needs an update when the database is missing", async () => { + expect(await g.needsUpdate()).toBe(true); +}); diff --git a/typescript/tests/launch-golden.test.ts b/typescript/tests/launch-golden.test.ts index 9365e8e..8a522d8 100644 --- a/typescript/tests/launch-golden.test.ts +++ b/typescript/tests/launch-golden.test.ts @@ -137,6 +137,7 @@ beforeAll(async () => { // The mmdb files only have to exist and be fresh; the fake reader answers. fs.mkdirSync(mods.geolocation.MMDB_DIR, { recursive: true }); for (const name of [ + "geoip aio by daijro-combined.mmdb", "maxmind geolite2-ipv4.mmdb", "maxmind geolite2-ipv6.mmdb", ]) {