Files
Jake WriterandClaude Fable 5.2 340cf16170 CI: build pull requests with the release's command, and preflight its other targets (#1055)
Four prerelease cuts failed on commits whose tests had passed (runs
41280578561, 43352392915, 43395652803, 43631788944), each in a step only
release.yml ran: midl.exe under Wine (twice), verify-fonts.py, and
make set-target. tests.yml built with `make build`; release.yml built with
multibuild.py after installing its own packages. A step only a release runs
is run for the first time by a release.

- make set-target (scripts/patch.py --mozconfig-only) writes the mozconfig
  and nothing else. It used to reset the tree and copy the additions over
  it, which in a tree without a git repository undid any patch hunk to an
  addition while leaving _READY in place, so the build went on with the
  reverted files. The mozconfig it writes is byte for byte the same.
- ci.run_build runs multibuild.py for one target. tests.yml's build job runs
  it for linux x86_64 and release.yml's for every target, so the pull
  request retargets, builds and packages exactly as a release does.
- multibuild.py --preflight (make preflight, scripts/preflight.py) checks a
  target without building it: configure, the pre-export, export and
  pre-compile tiers, then the object of every source file a patch or an
  addition touches. tests.yml preflights the five targets only a release
  builds, beside the build; the gate requires them. A target that passed is
  not run again for the same compiled inputs, except by the scheduled run.
- .github/actions/build-host frees the disk, installs the packages and adds
  the swap for all three jobs, so no build depends on a package only one of
  them installs.
- browser/multibuild.py is a browser input: changing it rebuilds.

The preflight does not link or package, so a link error on a cross-compiled
target is still first seen by a release.

Tests: ci/tests/test_set_target.py fails on main ('as added' != 'as patched');
test_preflight.py covers the source-to-object mapping and that an object left
by an earlier run is compiled again; test_ci.py holds the preflight matrix to
release.yml's and all three jobs to build-host; the tribal rule
one-build-entry-point holds all three to ci.run_build.

Run locally on Firefox 159 with CI's tree (make setup-minimal, make dir):
- All five preflights pass, in 2 to 6.5 minutes each on 16 cores.
- With the four additions as run 43631788944's tree had them, the linux
  arm64 preflight fails with that run's error,
  SkBitmapProcState_opts.cpp:41:13: use of undeclared identifier
  'sk_cpu_emulates_x86'.
- python3 -m ci.run_build --target linux --arch arm64, the release's
  command for a target that run never linked, builds and packages
  camoufox-159.0-beta.35-lin.arm64.zip in 41 minutes.
No workflow here has run on a GitHub runner yet; this pull request's run is
the first.

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-15 06:54:41 +00:00

227 lines
7.3 KiB
Python

#!/usr/bin/env python3
"""
The script that patches the Firefox source into the Camoufox source.
Based on LibreWolf's patch script:
https://gitlab.com/librewolf-community/browser/source/-/blob/main/scripts/librewolf-patches.py
Run:
python3 scripts/patch.py <version> <release>
"""
import os
import subprocess
import sys
from dataclasses import dataclass
from _mixin import (
find_src_dir,
get_moz_target,
get_options,
list_patches,
mozilla_rust_version,
run,
temp_cd,
)
import pgo
options, args = get_options()
"""
Main patcher functions
"""
def reset_to_unpatched():
"""Reset this source repository without discovering a parent repository."""
if not os.path.exists('.git'):
return
print("Resetting to unpatched state...")
run('git reset --hard unpatched && ./mach clobber && git clean -fdx')
@dataclass
class Patcher:
"""Patch and prepare the Camoufox source"""
moz_target: str
target: str
rust: str
pgo_profile: str
def camoufox_patches(self):
"""
Apply all patches, or with --mozconfig-only retarget the patched tree
"""
version, release = extract_args()
with temp_cd(find_src_dir('.', version, release)):
if not options.mozconfig_only:
# Reset only when the source tree has its own local repository.
reset_to_unpatched()
# Re-copy additions and settings after reset
print("Re-copying additions and settings...")
run(f'bash ../scripts/copy-additions.sh {version} {release}')
print(f'Using target: {self.moz_target}')
self._write_mozconfig()
if not options.mozconfig_only:
failed_patches = []
for patch_file in list_patches():
rejects = self._apply_and_check(patch_file)
if rejects:
failed_patches.append((patch_file, rejects))
# Report failures
if failed_patches:
print('\n' + '='*70)
print(f'ERROR: {len(failed_patches)} patch(es) failed to apply cleanly:')
print('='*70)
for patch_file, rejects in failed_patches:
print(f'\n{patch_file}:')
for reject in rejects:
print(f' - {reject}')
print('='*70)
sys.exit(1)
print('Complete!')
def _apply_and_check(self, patch_file):
"""
Apply a patch and check for reject files.
Returns the reject files and any non-zero exit status, empty if it applied cleanly.
"""
import time
print(f"\n*** -> patch -p1 -i {patch_file}")
sys.stdout.flush()
# Record time before applying so we only detect .rej files from this patch
start_time = time.time()
# Apply patch interactively - don't capture stdout/stderr at all
# This allows prompts to show immediately and user can respond
# --forward flag: skip patches that appear to be already applied
# --binary flag: preserve line endings (helps with CRLF vs LF differences)
# -l flag: ignore whitespace differences
result = subprocess.run(
['patch', '-p1', '--forward', '-l', '--binary', '-i', patch_file],
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
text=True
)
# After patch completes, search for any .rej files created during this patch
rejects = []
for root, dirs, files in os.walk('.'):
for file in files:
if file.endswith('.rej'):
reject_path = os.path.join(root, file)
if os.path.exists(reject_path):
# Only include if created after this patch started
if os.path.getmtime(reject_path) >= start_time:
rejects.append(reject_path)
# Clean up .rej files so they don't interfere with subsequent patches
for rej in rejects:
os.remove(rej)
if result.returncode != 0:
rejects.append(f'patch exited with status {result.returncode}')
return rejects
def _write_mozconfig(self):
"""
Write the mozconfig: the target, assets/base.mozconfig, then assets/<target>.mozconfig
"""
# Set first: base.mozconfig picks stock's LTO mode for the target and
# reads the profile. configure resolves rustc and cargo through rustup,
# which honours RUSTUP_TOOLCHAIN.
content = (f"CAMOUFOX_TARGET={self.moz_target}\n"
f"CAMOUFOX_PGO_PROFILE={self.pgo_profile}\n"
"ac_add_options --target=$CAMOUFOX_TARGET\n"
f"export RUSTUP_TOOLCHAIN={self.rust}\n\n")
with open(os.path.join("..", "assets", "base.mozconfig"), encoding='utf-8') as f:
content += f.read()
# Linux has no mozconfig of its own
target_mozconfig = os.path.join("..", "assets", f"{self.target}.mozconfig")
if os.path.exists(target_mozconfig):
with open(target_mozconfig, encoding='utf-8') as f:
content += f.read()
print(f"-> Writing mozconfig, target is {self.moz_target}")
with open("mozconfig", 'w', encoding='utf-8') as f:
f.write(content)
RUST_TARGETS = {
"linux": ("x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"),
"windows": ("x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", "i686-pc-windows-msvc"),
"macos": ("x86_64-apple-darwin", "aarch64-apple-darwin"),
}
def install_rust(rust, target):
"""Install the Rust release Mozilla builds this tree with, and its std for the target"""
targets = ' '.join(f'--target {t}' for t in RUST_TARGETS[target])
run(f'~/.cargo/bin/rustup toolchain install {rust} --profile minimal {targets}')
"""
Preparation
"""
def extract_args():
"""Get version and release from args"""
if len(args) != 2:
sys.stderr.write('error: please specify version and release of camoufox source')
sys.exit(1)
return args[0], args[1]
AVAILABLE_TARGETS = ["linux", "windows", "macos"]
AVAILABLE_ARCHS = ["x86_64", "arm64", "i686"]
def extract_build_target():
"""Get moz_target if passed to BUILD_TARGET environment variable"""
if os.environ.get('BUILD_TARGET'):
target, arch = os.environ['BUILD_TARGET'].split(',')
assert target in AVAILABLE_TARGETS, f"Unsupported target: {target}"
assert arch in AVAILABLE_ARCHS, f"Unsupported architecture: {arch}"
else:
target, arch = "macos", "arm64"
return target, arch
"""
Launcher
"""
if __name__ == "__main__":
# Extract args
VERSION, RELEASE = extract_args()
TARGET, ARCH = extract_build_target()
MOZ_TARGET = get_moz_target(TARGET, ARCH)
# Check if the folder exists
if not os.path.exists(f'camoufox-{VERSION}-{RELEASE}/configure.py'):
sys.stderr.write('error: folder doesn\'t look like a Firefox folder.')
sys.exit(1)
RUST = mozilla_rust_version(f'camoufox-{VERSION}-{RELEASE}')
install_rust(RUST, TARGET)
PGO_PROFILE = pgo.fetch(TARGET, ARCH, VERSION)
# Apply the patches
patcher = Patcher(MOZ_TARGET, TARGET, RUST, PGO_PROFILE)
patcher.camoufox_patches()
sys.exit(0) # ensure 0 exit code