Files
b09085c8f0 Bundle a committed uBlock Origin release, not AMO's newest (#1012)
* Bundle a committed uBlock Origin release, not AMO's newest

uBO 1.76.0 reached AMO and changed what pages observe: in every tab
after the first, RegExp moved after URL in the page's global order,
where stock 158 and 159 (every tab and window), no uBO and uBO 1.75.4
keep it after Number. main-world-eval-globals.py caught it, and
534430e0 then took the first tab out of the comparison, which hides
the difference instead of removing it.

fetch-ubo.py resolve now reads browser/scripts/data/ubo.json (1.75.4,
sha256-pinned); `newest` prints AMO's latest as a candidate, so a bump
is a reviewed commit. main-world-eval-globals.py compares the first
tab again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Describe the bundled uBlock Origin as the committed pin everywhere

README.md, ci/README.md, release.yml, package.py and ci/release.py still
said every package bundles AMO's newest uBO, and release.py that the pin is
in no commit. It is now browser/scripts/data/ubo.json.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Jake Writer <jake_writer@jobharvest.com>
2026-11-11 02:33:45 +00:00

326 lines
13 KiB
Python

#!/usr/bin/env python3
import argparse
import glob
import hashlib
import json
import os
import shutil
import stat
import sys
import tempfile
import zipfile
from shlex import join
from _mixin import find_src_dir, get_moz_target, list_files, run, temp_cd
# The GPU probe (gfxtest) is NOT here: without it nsIGfxInfo has no data and the
# driver blocklist refuses every WebGL context where stock Firefox gets one.
UNNEEDED_PATHS = {'uninstall', 'pingsender.exe', 'pingsender'}
# APPNOTE.TXT 4.4.2: "version made by" host 3 is Unix, whose external
# attributes carry st_mode in the high 16 bits.
ZIP_CREATED_ON_UNIX = 3
def inject_locales(target_dir, target, version, src_dir):
"""Bake the official language packs in as packaged locales.
Without them a spoofed non-English locale localizes Intl/number/date
formatting but leaves every browser-provided string that content can read
(input.validationMessage, XML parse errors) in English -- a mix no real
Firefox produces. A langpack add-on does not fix it: the parent pre-creates
those string bundles before add-ons start. Packaged locales are what a
Mozilla localized build has, selected by intl.locale.requested (pythonlib).
"""
langpacks = os.path.join('bundle', 'langpacks')
version_file = os.path.join(langpacks, 'VERSION')
base_version = version.split('-')[0]
have = open(version_file).read().strip() if os.path.exists(version_file) else None
if have != base_version:
# Not in git (bundle/langpacks is ignored): fetch them from
# archive.mozilla.org the way `make fetch` fetches the source.
run(join([sys.executable, os.path.join('scripts', 'fetch-langpacks.py'), base_version]))
have = open(version_file).read().strip() if os.path.exists(version_file) else None
if have != base_version:
raise FileNotFoundError(
f"bundle/langpacks is for Firefox {have}, need {base_version}: "
f"run scripts/fetch-langpacks.py {base_version}"
)
# Mozilla's macOS Japanese build is ja-JP-mac; every other platform ships ja.
skip = 'ja' if target == 'macos' else 'ja-JP-mac'
xpis = sorted(
path for path in glob.glob(os.path.join(langpacks, '*.xpi'))
if os.path.basename(path)[:-4] != skip
)
run(join([sys.executable, os.path.join('scripts', 'inject-locales.py'),
'--source-tree', src_dir, target_dir, *xpis]))
def font_groups_for(groups_file, oses):
"""Group directories the named OSes read."""
if not os.path.exists(groups_file):
raise FileNotFoundError(f'{groups_file} is missing: run `make fonts-extract`')
with open(groups_file, encoding='utf-8') as fh:
read_by = json.load(fh).get('readBy', {})
key = {'linux': 'lin', 'macos': 'mac', 'windows': 'win'}
out = set()
for o in oses:
out.update(read_by.get(key.get(o, o), []))
return sorted(out)
def ascii_file_names(root):
"""Rename every file under `root` whose name is not ASCII.
The release zip is extracted with `unzip`, and unzip 6.0 under a C/POSIX
locale (a Docker build, a bare SSH shell) reports each UTF-8 name as a
"mismatching local filename" and exits 1, stopping the consumer's install
-- though the zip stores the names correctly. The font bundle's Hiragino
faces are named in Japanese. Nothing reads a font by its file name
(fontconfig and the flat CoreText/DirectWrite directories take family names
from the font itself), so the ASCII part of the name is kept and the rest
replaced by a short digest of the original: stable across builds, unique.
"""
for dirpath, _, files in os.walk(root):
for name in files:
if name.isascii():
continue
stem, ext = os.path.splitext(name)
kept = ''.join(c for c in stem if c.isascii() and (c.isalnum() or c in '_-')).rstrip('_-')
digest = hashlib.sha256(name.encode('utf-8')).hexdigest()[:12]
ascii_name = f"{kept}_u{digest}{ext}"
os.rename(os.path.join(dirpath, name), os.path.join(dirpath, ascii_name))
def readable_by_all(root):
"""Give group and others read on every file and dir under root (and search
on dirs and on files the owner can execute), keeping the owner's bits.
The zip stores each entry's Unix mode and unzip restores it, so a 0700 dir
(a tempfile.mkdtemp staged in) makes everything under it unreadable to any
user but the one that unzipped it: a Docker image's app user then started
without uBO.
"""
for dirpath, dirs, files in os.walk(root):
os.chmod(dirpath, os.stat(dirpath).st_mode | 0o755)
for name in files:
path = os.path.join(dirpath, name)
if os.path.islink(path):
continue
mode = os.stat(path).st_mode
os.chmod(path, mode | 0o644 | (0o111 if mode & 0o100 else 0))
def unreadable_entries(zip_path):
"""Zip entries whose stored Unix mode denies other users read (or search,
for a dir). Entries not made on Unix carry no mode (unzip applies the
umask) and pass."""
out = []
with zipfile.ZipFile(zip_path) as z:
for info in z.infolist():
mode = info.external_attr >> 16
if info.create_system != ZIP_CREATED_ON_UNIX or not mode:
continue
need = 0o005 if stat.S_ISDIR(mode) or info.is_dir() else 0o004
if stat.S_ISLNK(mode):
continue
if mode & need != need:
out.append(f'{info.filename} {oct(mode & 0o7777)}')
return out
def add_includes_to_package(package_file, includes, fonts, new_file, target, version, src_dir):
with tempfile.TemporaryDirectory() as temp_dir:
# Extract package
run(join(['7z', 'x', package_file, f'-o{temp_dir}']), exit_on_fail=False)
# Delete package_file
os.remove(package_file)
if package_file.endswith('.tar.xz'):
# Rerun on the tar file
package_tar = package_file[:-3] # Remove ".xz"
return add_includes_to_package(
package_file=os.path.join(temp_dir, package_tar),
includes=includes,
fonts=fonts,
new_file=new_file,
target=target,
version=version,
src_dir=src_dir,
)
if target == 'macos':
# Move Camoufox/Camoufox.app -> Camoufox.app
nightly_dir = os.path.join(temp_dir, 'Camoufox')
shutil.move(
os.path.join(nightly_dir, 'Camoufox.app'), os.path.join(temp_dir, 'Camoufox.app')
)
# Remove old app dir and all content in it
shutil.rmtree(nightly_dir)
else:
# Move contents out of camoufox folder if it exists
old_camoufox_dir = os.path.join(temp_dir, 'camoufox')
camoufox_dir = os.path.join(temp_dir, 'camoufox-folder')
if os.path.exists(old_camoufox_dir):
# Rename camoufox_dir
os.rename(old_camoufox_dir, camoufox_dir)
for item in os.listdir(camoufox_dir):
shutil.move(os.path.join(camoufox_dir, item), temp_dir)
os.rmdir(camoufox_dir)
# Create target_dir
if target == 'macos':
target_dir = os.path.join(temp_dir, 'Camoufox.app', 'Contents', 'Resources')
else:
target_dir = temp_dir
# Add includes. A missing one is fatal: an unexpanded MSVC CRT glob used to
# be skipped, shipping a browser that needs the redistributable installed.
missing = [include for include in includes or [] if not os.path.exists(include)]
if missing:
raise FileNotFoundError(
"Missing --includes entries (unexpanded glob or wrong path):\n "
+ "\n ".join(missing)
)
for include in includes or []:
if os.path.isdir(include):
shutil.copytree(
include,
os.path.join(target_dir, os.path.basename(include)),
dirs_exist_ok=True,
)
else:
shutil.copy2(include, target_dir)
# Add the fonts under fonts/. The bundle stores each face once per OS set
# (bundle/fonts/groups.json), so look up the groups each OS reads.
fonts_dir = os.path.join(target_dir, 'fonts')
groups_file = os.path.join('bundle', 'fonts', 'groups.json')
wanted = font_groups_for(groups_file, fonts or [])
if target == 'linux':
# Linux resolves fonts through fontconfig, which is handed the exact
# group directories for the claimed OS at launch (utils._generate_fontconfig),
# so the subdirectories are the per-OS gate and must be preserved.
for g in wanted:
shutil.copytree(os.path.join('bundle', 'fonts', g),
os.path.join(fonts_dir, g), dirs_exist_ok=True)
shutil.copy2(groups_file, os.path.join(fonts_dir, 'groups.json'))
else:
# macOS (CoreText) and Windows (DirectWrite) activate ONE flat
# directory and cannot read subfolders, so there is no directory
# gate on those targets -- the font allowlist is what restricts a
# lookup (font-hijacker.patch). Flatten, skipping any face whose
# bytes are already present.
os.makedirs(fonts_dir, exist_ok=True)
seen = set()
for g in wanted:
for file in list_files(root_dir=os.path.join('bundle', 'fonts', g), suffix='*'):
with open(file, 'rb') as fh:
digest = hashlib.sha256(fh.read()).hexdigest()
if digest in seen:
continue
seen.add(digest)
shutil.copy2(file, os.path.join(fonts_dir, os.path.basename(file)))
if os.path.isdir(fonts_dir):
ascii_file_names(fonts_dir)
inject_locales(target_dir, target, version, src_dir)
# uBlock Origin, which both launchers load from addons/UBO/ beside the
# binary: the pin in data/ubo.json, or the release's in CAMOUFOX_UBO.
run(join([sys.executable, os.path.join('scripts', 'fetch-ubo.py'), 'stage', target_dir]))
# Remove unneeded paths
for path in UNNEEDED_PATHS:
if os.path.isdir(os.path.join(target_dir, path)):
shutil.rmtree(os.path.join(target_dir, path), ignore_errors=True)
elif os.path.exists(os.path.join(target_dir, path)):
os.remove(os.path.join(target_dir, path))
# Update package
readable_by_all(temp_dir)
run(join(['7z', 'u', new_file, f'{temp_dir}/*', '-r', '-mx=9']))
unreadable = unreadable_entries(new_file)
if unreadable:
raise SystemExit(f'{new_file} has entries other users cannot read: {unreadable[:10]}')
def get_args():
"""Get CLI parameters"""
parser = argparse.ArgumentParser(
description='Package Camoufox for different operating systems.'
)
parser.add_argument('os', choices=['linux', 'macos', 'windows'], help='Target operating system')
parser.add_argument(
'--includes', nargs='+', help='List of files or directories to include in the package'
)
parser.add_argument('--version', required=True, help='Camoufox version')
parser.add_argument('--release', required=True, help='Camoufox release number')
parser.add_argument(
'--arch', choices=['x86_64', 'i686', 'arm64'], help='Architecture for Windows build'
)
parser.add_argument('--fonts', nargs='+', help='Font directories to include under fonts/')
return parser.parse_args()
def main():
"""The main packaging function"""
args = get_args()
# Determine file extension based on OS
file_extensions = {'linux': 'tar.xz', 'macos': 'dmg', 'windows': 'zip'}
file_ext = file_extensions[args.os]
# Build the package
src_dir = find_src_dir('.', args.version, args.release)
moz_target = get_moz_target(target=args.os, arch=args.arch)
with temp_cd(src_dir):
# Create package files
run('./mach package')
# Find package files
search_path = os.path.abspath(
f'obj-{moz_target}/dist/camoufox-{args.version}-{args.release}.*.{file_ext}'
)
# Copy package files
for file in glob.glob(search_path):
if 'xpt_artifacts' in file:
print(f'Skipping xpt artifacts: {file}')
continue
print(f'Found package: {file}')
# Copy to root
shutil.copy2(file, '.')
break
else:
print(f"Error: No package file found matching pattern: {search_path}")
sys.exit(1)
# Find the package file
package_pattern = f'camoufox-{args.version}-{args.release}.en-US.*.{file_ext}'
package_files = glob.glob(package_pattern)
if not package_files:
print(f"Error: No package file found matching pattern: {package_pattern}")
exit(1)
package_file = package_files[0]
# Add includes to the package
new_name = f'camoufox-{args.version}-{args.release}-{args.os[:3]}.{args.arch}.zip'
add_includes_to_package(
package_file=package_file,
includes=args.includes,
fonts=args.fonts,
new_file=new_name,
target=args.os,
version=args.version,
src_dir=os.path.abspath(src_dir),
)
print(f"Packaging complete for {args.os}")
if __name__ == '__main__':
main()