feat(geoip): make GeoIP AIO the default source, deprecate GeoLite2 (#820)

The default GeoIP source was MaxMind GeoLite2 via sapics/ip-location-db,
whose URLs kept serving the 2026-06-17 build after that project moved to
GitHub Releases (found in #815). GeoIP AIO (daijro/geoip-all-in-one)
resolves timezones more accurately on real proxy IPs and is rebuilt weekly.

- repos.yml: AIO is the default; GeoLite2 is `deprecated: true`, with the
  Releases URLs from #815 so it still works when picked by name.
- A cache holding a deprecated source it was not explicitly given
  (`camoufox set --geoip` or the GUI) moves to the default and drops the
  old database. An explicit choice is kept, with a FutureWarning.
- needs_update() reads the database's build date instead of the file age:
  refresh once the build is over 8 days old, re-checking at most daily,
  and warn when a fresh download is over 30 days old (a frozen source).
- get_geolocation(geoip_db=...) now reads that source's own database
  rather than the active one's, and no longer makes it the active one.
- tests/test_geoip_sources.py (from #815) downloads every non-deprecated
  source and fails when its build is stale; tests.yml installs the geoip
  extra so it runs, and so gates every release.
- TypeScript twin updated to match; goldens answer in both layouts.

Co-authored-by: lp177 <57773165+lp177@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jake Writer
2026-09-30 20:02:52 +00:00
committed by GitHub
co-authored by lp177 Claude Opus 5.5
parent 0db845fac0
commit f36390a19e
16 changed files with 573 additions and 74 deletions
+4 -2
View File
@@ -31,6 +31,8 @@ pip install -U camoufox[geoip]
The `geoip` parameter is optional, but heavily recommended if you are using proxies. It will download an extra dataset to determine the user's longitude, latitude, timezone, country, & locale.
The dataset is [GeoIP All-in-One](https://github.com/daijro/geoip-all-in-one), which merges several IP databases and is rebuilt weekly. Camoufox fetches the newest build and refreshes it once it is a week old.
Next, download the Camoufox browser:
```bash
@@ -266,12 +268,12 @@ Browser
Latest in official/stable? Yes
Last Sync 2026-03-07 00:23
GeoIP
Database MaxMind GeoLite2
Database GeoIP AIO by daijro
Updated 2026-03-07 00:24
Storage
Install path /home/name/.cache/camoufox
Browser(s) directory size 1.2 GB
GeoIP database size 40.7 MB
GeoIP database size 116.4 MB
Config file /home/name/.cache/camoufox/config.json
Repo cache /home/name/.cache/camoufox/repo_cache.json
```
+12 -2
View File
@@ -20,6 +20,7 @@ from .geolocation import (
get_mmdb_path,
load_geoip_config,
save_geoip_config,
warn_if_deprecated,
)
from .multiversion import (
BROWSERS_DIR,
@@ -666,14 +667,23 @@ def _select_geoip_source():
return
current = load_geoip_config().get("name", "")
choices = [(r["name"] + (" [active]" if r.get("name") == current else ""), r) for r in repos]
choices = [
(
r["name"]
+ (" (deprecated)" if r.get("deprecated") else "")
+ (" [active]" if r.get("name") == current else ""),
r,
)
for r in repos
]
selected = _inquirer_select(choices, "Select GeoIP source")
if not selected:
return
save_geoip_config(selected)
save_geoip_config(selected, explicit=True)
rprint(f"GeoIP source: {selected['name']}", fg="green")
warn_if_deprecated(selected)
@cli.command(name="list")
+109 -25
View File
@@ -2,8 +2,10 @@
Helpers to fetch geolocation, timezone, and locale data given an IP
"""
import os
import shutil
import tempfile
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, cast
@@ -12,6 +14,7 @@ from yaml import CDumper, CLoader
from yaml import dump as yaml_dump
from yaml import load as yaml_load
from ._warnings import WARNINGS_DATA, _warn_from_caller
from .exceptions import NotInstalledGeoIPExtra, UnknownIPLocation
from .ip import validate_ip
from .locales import SELECTOR, Geolocation
@@ -29,6 +32,15 @@ GEOIP_DIR = Path(user_cache_dir("camoufox")) / "geoip"
MMDB_DIR = GEOIP_DIR / "mmdb"
GEOIP_CONFIG = GEOIP_DIR / "config.yml"
# A database whose data was built longer ago than this is refreshed. The
# default source publishes weekly, so a week and a day catches every release.
UPDATE_DAYS = 8
# ...but at most once a day, so a source that stops publishing costs one
# download a day rather than one per launch.
RECHECK_DAYS = 1
# A freshly downloaded build older than this means its source is frozen.
FROZEN_DAYS = 30
def _find_in(data: Dict, key: str) -> Any:
"""
@@ -50,7 +62,7 @@ def _load_geoip_repos() -> Tuple[List[Dict], str]:
with open(LOCAL_DATA / 'repos.yml', 'r') as f:
data = yaml_load(f, Loader=CLoader)
geoip_repos = data.get('geoip', [])
default_name = data.get('default', {}).get('geoip', 'GeoLite2')
default_name = data.get('default', {}).get('geoip', 'GeoIP AIO by daijro')
return geoip_repos, default_name
@@ -81,27 +93,49 @@ def _get_geoip_config_by_name(name: Optional[str] = None) -> Dict:
raise ValueError("No GeoIP repos configured in repos.yml")
def warn_if_deprecated(config: Dict) -> None:
"""
Warn that a GeoIP source is deprecated and name the default to use instead
"""
if config.get('deprecated'):
_, default_name = _load_geoip_repos()
_warn_from_caller(
WARNINGS_DATA['geoip_deprecated'].format(name=config['name'], default=default_name),
FutureWarning,
)
def load_geoip_config() -> Dict:
"""
Load active GeoIP config from disk, falling back to repos.yml default
Load active GeoIP config from disk, falling back to repos.yml default.
A saved deprecated source that the user did not pick explicitly (every
cache written before the default changed) resolves to the default.
"""
if GEOIP_CONFIG.exists():
with open(GEOIP_CONFIG, 'r') as f:
saved = yaml_load(f, Loader=CLoader)
saved = yaml_load(f, Loader=CLoader) or {}
try:
return _get_geoip_config_by_name(saved.get('name'))
config = _get_geoip_config_by_name(saved.get('name'))
except (ValueError, KeyError):
return saved
if config.get('deprecated') and not saved.get('explicit'):
return _get_geoip_config_by_name(None)
return config
return _get_geoip_config_by_name(None)
def save_geoip_config(config: Dict) -> None:
def save_geoip_config(config: Dict, explicit: bool = False) -> None:
"""
Save active GeoIP source name to disk
Save active GeoIP source name to disk. `explicit` records that the user
chose it, so a later default change does not move them off it.
"""
GEOIP_DIR.mkdir(parents=True, exist_ok=True)
saved: Dict[str, Any] = {'name': config['name']}
if explicit:
saved['explicit'] = True
with open(GEOIP_CONFIG, 'w') as f:
yaml_dump({'name': config['name']}, f, Dumper=CDumper, default_flow_style=False)
yaml_dump(saved, f, Dumper=CDumper, default_flow_style=False)
def get_mmdb_path(ip_version: str = 'ipv4', config: Optional[Dict] = None) -> Path:
@@ -130,9 +164,11 @@ def geoip_allowed() -> None:
def download_mmdb(
source: Optional[str] = None,
progress_callback: Optional[callable] = None,
activate: bool = True,
) -> None:
"""
Downloads the GeoIP database(s) to geoip/mmdb/
Downloads the GeoIP database(s) to geoip/mmdb/. A named `source` becomes
the user's explicit choice unless `activate` is False.
"""
geoip_allowed()
@@ -180,6 +216,8 @@ def download_mmdb(
tmp.seek(0)
with open(mmdb_path, 'wb') as dst:
shutil.copyfileobj(tmp, dst)
# The mtime is when we last checked, which needs_update() throttles on
os.utime(mmdb_path)
break
except Exception as e:
last_error = e
@@ -187,7 +225,40 @@ def download_mmdb(
else:
raise last_error or Exception(f"Failed to download {ip_ver}")
save_geoip_config(config)
age = _build_age_days(mmdb_path)
if age is not None and age > FROZEN_DAYS:
_warn_from_caller(
WARNINGS_DATA['geoip_frozen'].format(name=config['name'], days=int(age)),
RuntimeWarning,
)
_remove_deprecated_databases(keep=config)
if activate:
# A refresh of the active source keeps whether the user chose it
save_geoip_config(config, explicit=bool(source) or _chosen_explicitly(config))
def _chosen_explicitly(config: Dict) -> bool:
"""
Whether the saved config names this source as the user's explicit choice
"""
if not GEOIP_CONFIG.exists():
return False
with open(GEOIP_CONFIG, 'r') as f:
saved = yaml_load(f, Loader=CLoader) or {}
return bool(saved.get('explicit')) and saved.get('name') == config['name']
def _remove_deprecated_databases(keep: Dict) -> None:
"""
Delete downloaded databases of deprecated sources other than `keep`
"""
repos, _ = _load_geoip_repos()
for repo in repos:
if not repo.get('deprecated') or repo['name'] == keep['name']:
continue
for path in MMDB_DIR.glob(f"{repo['name'].lower()}-*.mmdb"):
path.unlink(missing_ok=True)
def remove_mmdb() -> None:
@@ -202,24 +273,39 @@ def remove_mmdb() -> None:
rprint("GeoIP database removed.")
def _build_age_days(mmdb_path: Path) -> Optional[float]:
"""
Days since the database's data was built, from its metadata
"""
import maxminddb
try:
with maxminddb.open_database(str(mmdb_path)) as reader:
return (time.time() - reader.metadata().build_epoch) / 86400
except Exception:
return None
def needs_update(config: Optional[Dict] = None) -> bool:
"""
Check if the GeoIP database needs an update (older than 30 days)
"""
from datetime import datetime, timedelta
Check if the GeoIP database needs an update: its data was built over
UPDATE_DAYS ago and it was last downloaded over RECHECK_DAYS ago.
This reads the build date rather than the file's age, so a source that
keeps serving an old build is noticed.
"""
if config is None:
config = load_geoip_config()
update_days = 30
ipv4_path = get_mmdb_path('ipv4', config)
if not ipv4_path.exists():
return True
mtime = datetime.fromtimestamp(ipv4_path.stat().st_mtime)
age = datetime.now() - mtime
return age > timedelta(days=update_days)
checked_days = (time.time() - ipv4_path.stat().st_mtime) / 86400
if checked_days < RECHECK_DAYS:
return False
build_days = _build_age_days(ipv4_path)
return build_days is None or build_days > UPDATE_DAYS
def get_geolocation(ip: str, geoip_db: Optional[str] = None) -> Geolocation:
@@ -230,16 +316,14 @@ def get_geolocation(ip: str, geoip_db: Optional[str] = None) -> Geolocation:
validate_ip(ip)
ip_version = 'ipv6' if ':' in ip else 'ipv4'
mmdb_path = get_mmdb_path(ip_version)
if not mmdb_path.exists() or needs_update():
download_mmdb()
mmdb_path = get_mmdb_path(ip_version)
# A per-call geoip_db reads its own database without becoming the active one
config = _get_geoip_config_by_name(geoip_db) if geoip_db else load_geoip_config()
warn_if_deprecated(config)
mmdb_path = get_mmdb_path(ip_version, config)
if geoip_db:
config = _get_geoip_config_by_name(geoip_db)
else:
config = load_geoip_config()
if not mmdb_path.exists() or needs_update(config):
download_mmdb(geoip_db, activate=not geoip_db)
paths = config['paths']
with maxminddb.open_database(str(mmdb_path)) as reader:
+1 -1
View File
@@ -861,7 +861,7 @@ class Backend(QObject):
if source not in self._geoip_downloaded:
return
save_geoip_config(_get_geoip_config_by_name(source))
save_geoip_config(_get_geoip_config_by_name(source), explicit=True)
self._load_geoip()
@Slot(int)
+27 -17
View File
@@ -1,7 +1,7 @@
# Default configurations
default:
browser: Official
geoip: MaxMind GeoLite2
geoip: GeoIP AIO by daijro
# Browser repositories
browsers:
@@ -39,23 +39,18 @@ browsers:
# Assume all browsers
# GeoIP database repositories
#
# Every URL points at the source's newest build, never a pinned one: the
# launcher re-downloads once the build it has is a week old (see
# needs_update() in geolocation.py), and tests/test_geoip_sources.py fails CI
# and the release when a source stops publishing.
#
# `deprecated: true` sources still work when chosen by name, with a warning.
# A cache that holds one without an explicit `camoufox set --geoip` choice
# (every install that predates the switch to AIO) moves to the default.
geoip:
# GeoLite2 City - Full city-level data with timezone
- name: MaxMind GeoLite2
urls:
ipv4:
- https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-city-mmdb/geolite2-city-ipv4.mmdb
- https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/geolite2-city-mmdb/geolite2-city-ipv4.mmdb
ipv6:
- https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-city-mmdb/geolite2-city-ipv6.mmdb
- https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/geolite2-city-mmdb/geolite2-city-ipv6.mmdb
paths:
iso_code: country_code
longitude: longitude
latitude: latitude
timezone: timezone
# GeoIP All-in-One - Combined IPv4/IPv6 with all fields
# GeoIP All-in-One - Combined IPv4/IPv6 with all fields. Merges several
# databases per range; rebuilt weekly by daijro/geoip-all-in-one.
- name: GeoIP AIO by daijro
extract: true
urls:
@@ -66,3 +61,18 @@ geoip:
longitude: location.longitude
latitude: location.latitude
timezone: location.time_zone
# GeoLite2 City via sapics/ip-location-db, which publishes to GitHub
# Releases since 2026-06-18 (URLs from daijro/camoufox#815).
- name: MaxMind GeoLite2
deprecated: true
urls:
ipv4:
- https://github.com/sapics/ip-location-db/releases/download/latest/geolite2-city-ipv4.mmdb
ipv6:
- https://github.com/sapics/ip-location-db/releases/download/latest/geolite2-city-ipv6.mmdb
paths:
iso_code: country_code
longitude: longitude
latitude: latitude
timezone: timezone
+2 -2
View File
@@ -878,8 +878,8 @@ def launch_options(
Calculate longitude, latitude, timezone, country, & locale based on the IP address.
Pass the target IP address to use, or `True` to find the IP address automatically.
geoip_db (Optional[str]):
Name of the GeoIP database to use (e.g., "MaxMind").
If not specified, uses the configured default.
Name of the GeoIP database to use (e.g., "GeoIP AIO by daijro").
If not specified, uses the one chosen with `camoufox set --geoip`, or the default.
humanize (Optional[Union[bool, float]]):
Humanize the cursor movement.
Takes either `True`, or the MAX duration in seconds of the cursor movement.
+10
View File
@@ -75,3 +75,13 @@ fallback: |-
Please report this at https://github.com/daijro/camoufox/issues/new and include:
{report}
geoip_deprecated: >-
The GeoIP source "{name}" is deprecated and will be removed in a future release.
"{default}" is the default and resolves locations more accurately.
Run `camoufox set --geoip` to switch, or stop passing `geoip_db`.
geoip_frozen: >-
The newest "{name}" GeoIP database was built {days} days ago, so its source may have
stopped publishing and locations may be out of date. Please report this at
https://github.com/daijro/camoufox/issues.
+98
View File
@@ -0,0 +1,98 @@
"""Which GeoIP source is active, and when its database is refreshed."""
import os
import time
import warnings
import pytest
from yaml import safe_dump
from camoufox import geolocation
DEFAULT = "GeoIP AIO by daijro"
DEPRECATED = "MaxMind GeoLite2"
@pytest.fixture(autouse=True)
def cache(tmp_path, monkeypatch):
monkeypatch.setattr(geolocation, "GEOIP_DIR", tmp_path)
monkeypatch.setattr(geolocation, "MMDB_DIR", tmp_path / "mmdb")
monkeypatch.setattr(geolocation, "GEOIP_CONFIG", tmp_path / "config.yml")
return tmp_path
def _saved(cache, **fields):
(cache / "config.yml").write_text(safe_dump(fields))
def test_default_is_aio():
assert geolocation.load_geoip_config()["name"] == DEFAULT
repos, _ = geolocation._load_geoip_repos()
assert [r["name"] for r in repos if r.get("deprecated")] == [DEPRECATED]
def test_implicit_deprecated_source_moves_to_default(cache):
# What every cache written before the default changed holds
_saved(cache, name=DEPRECATED)
assert geolocation.load_geoip_config()["name"] == DEFAULT
def test_explicit_deprecated_source_is_kept(cache):
_saved(cache, name=DEPRECATED, explicit=True)
assert geolocation.load_geoip_config()["name"] == DEPRECATED
def test_save_records_explicit_choice(cache):
config = geolocation._get_geoip_config_by_name(DEPRECATED)
geolocation.save_geoip_config(config, explicit=True)
assert geolocation.load_geoip_config()["name"] == DEPRECATED
geolocation.save_geoip_config(config)
assert geolocation.load_geoip_config()["name"] == DEFAULT
def test_refresh_keeps_an_explicit_choice(cache, monkeypatch):
# `camoufox fetch` and the weekly refresh download without naming a source
_saved(cache, name=DEPRECATED, explicit=True)
monkeypatch.setattr(geolocation, "webdl", lambda url, buffer, **kw: buffer.write(b"x"))
monkeypatch.setattr(geolocation, "_build_age_days", lambda path: 1)
geolocation.download_mmdb()
assert geolocation.load_geoip_config()["name"] == DEPRECATED
def test_deprecated_source_warns():
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
geolocation.warn_if_deprecated(geolocation._get_geoip_config_by_name(DEPRECATED))
geolocation.warn_if_deprecated(geolocation._get_geoip_config_by_name(DEFAULT))
assert [w.category for w in caught] == [FutureWarning]
assert DEFAULT in str(caught[0].message)
def _database(cache, checked_days_ago):
config = geolocation.load_geoip_config()
path = geolocation.get_mmdb_path("ipv4", config)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"")
stamp = time.time() - checked_days_ago * 86400
os.utime(path, (stamp, stamp))
return config
@pytest.mark.parametrize(
"checked, built, stale",
[
(0.5, 400, False), # checked today: wait, even for an old build
(2, 3, False), # this week's build
(2, 9, True), # a release has been missed
(40, 3, False), # an old file holding a new build is not stale
(2, None, True), # unreadable database
],
)
def test_needs_update_reads_the_build_date(cache, monkeypatch, checked, built, stale):
config = _database(cache, checked)
monkeypatch.setattr(geolocation, "_build_age_days", lambda path: built)
assert geolocation.needs_update(config) is stale
def test_missing_database_needs_update():
assert geolocation.needs_update() is True
+43
View File
@@ -0,0 +1,43 @@
"""Every GeoIP source camoufox offers must be serving a current build.
The launcher always downloads a source's newest build, so a source that stops
publishing leaves every user on stale data with no error: GeoLite2's URLs
served the 2026-06-17 build for months after sapics/ip-location-db moved to
GitHub Releases (found in daijro/camoufox#815, which this test comes from).
This suite runs in tests.yml, which gates every release.
"""
import time
import pytest
from camoufox import geolocation
maxminddb = pytest.importorskip("maxminddb")
MAX_AGE_DAYS = 30
SOURCES = [repo["name"] for repo in geolocation._load_geoip_repos()[0] if not repo.get("deprecated")]
@pytest.mark.parametrize("name", SOURCES)
def test_source_serves_a_current_build(name, tmp_path, monkeypatch):
monkeypatch.setattr(geolocation, "GEOIP_DIR", tmp_path)
monkeypatch.setattr(geolocation, "MMDB_DIR", tmp_path / "mmdb")
monkeypatch.setattr(geolocation, "GEOIP_CONFIG", tmp_path / "config.yml")
geolocation.download_mmdb(name)
databases = sorted((tmp_path / "mmdb").glob("*.mmdb"))
assert databases, f"{name} downloaded no database"
for database in databases:
with maxminddb.open_database(str(database)) as reader:
age_days = (time.time() - reader.metadata().build_epoch) / 86400
record = reader.get("8.8.8.8")
assert age_days < MAX_AGE_DAYS, (
f"{name}: {database.name} was built {age_days:.0f} days ago, its source is no longer updated"
)
config = geolocation._get_geoip_config_by_name(name)
assert geolocation._find_in(record, config["paths"]["iso_code"]) == "US", (
f"{name}: the configured paths do not read this database"
)