Files
Jake WriterandClaude Fable 5.2 340cf16170 CI: build pull requests with the release's command, and preflight its other targets (#1055)
Four prerelease cuts failed on commits whose tests had passed (runs
41280578561, 43352392915, 43395652803, 43631788944), each in a step only
release.yml ran: midl.exe under Wine (twice), verify-fonts.py, and
make set-target. tests.yml built with `make build`; release.yml built with
multibuild.py after installing its own packages. A step only a release runs
is run for the first time by a release.

- make set-target (scripts/patch.py --mozconfig-only) writes the mozconfig
  and nothing else. It used to reset the tree and copy the additions over
  it, which in a tree without a git repository undid any patch hunk to an
  addition while leaving _READY in place, so the build went on with the
  reverted files. The mozconfig it writes is byte for byte the same.
- ci.run_build runs multibuild.py for one target. tests.yml's build job runs
  it for linux x86_64 and release.yml's for every target, so the pull
  request retargets, builds and packages exactly as a release does.
- multibuild.py --preflight (make preflight, scripts/preflight.py) checks a
  target without building it: configure, the pre-export, export and
  pre-compile tiers, then the object of every source file a patch or an
  addition touches. tests.yml preflights the five targets only a release
  builds, beside the build; the gate requires them. A target that passed is
  not run again for the same compiled inputs, except by the scheduled run.
- .github/actions/build-host frees the disk, installs the packages and adds
  the swap for all three jobs, so no build depends on a package only one of
  them installs.
- browser/multibuild.py is a browser input: changing it rebuilds.

The preflight does not link or package, so a link error on a cross-compiled
target is still first seen by a release.

Tests: ci/tests/test_set_target.py fails on main ('as added' != 'as patched');
test_preflight.py covers the source-to-object mapping and that an object left
by an earlier run is compiled again; test_ci.py holds the preflight matrix to
release.yml's and all three jobs to build-host; the tribal rule
one-build-entry-point holds all three to ci.run_build.

Run locally on Firefox 159 with CI's tree (make setup-minimal, make dir):
- All five preflights pass, in 2 to 6.5 minutes each on 16 cores.
- With the four additions as run 43631788944's tree had them, the linux
  arm64 preflight fails with that run's error,
  SkBitmapProcState_opts.cpp:41:13: use of undeclared identifier
  'sk_cpu_emulates_x86'.
- python3 -m ci.run_build --target linux --arch arm64, the release's
  command for a target that run never linked, builds and packages
  camoufox-159.0-beta.35-lin.arm64.zip in 41 minutes.
No workflow here has run on a GitHub runner yet; this pull request's run is
the first.

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-15 06:54:41 +00:00

89 lines
3.9 KiB
Python

#!/usr/bin/env python3
"""
Check that a target builds, without building it: configure, run the export
tiers, then compile the object of every source file a patch or an addition
touches. Linking and packaging are left to a full build.
Run:
python3 scripts/preflight.py <source dir>
"""
import json
import os
import re
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
from _mixin import run, temp_cd
SOURCE_SUFFIXES = {'.c', '.cc', '.cpp', '.m', '.mm', '.S', '.s', '.asm'}
def touched_sources(browser):
"""Every compiled file a patch changes or an addition provides, relative to the tree"""
touched = {str(f.relative_to(browser / 'additions'))
for f in (browser / 'additions').rglob('*') if f.is_file()}
for patch in (browser / 'patches').rglob('*.patch'):
touched.update(re.findall(r'^\+\+\+ b/(\S+)', patch.read_text(encoding='utf-8', errors='replace'), re.M))
return {path for path in touched if Path(path).suffix in SOURCE_SUFFIXES}
def objects_of(touched, tree, objdir):
"""{objdir directory: its objects that hold a touched source}, and the sources this target builds"""
suffix = re.search(r'^OBJ_SUFFIX = (\S+)', (objdir / 'config' / 'autoconf.mk').read_text(), re.M).group(1)
objects, built = defaultdict(set), set()
def record(source, directory, name):
source = os.path.relpath(os.path.normpath(source), tree)
if source in touched:
objects[directory].add(f'{Path(name).stem}.{suffix}')
built.add(source)
# A unified source includes the files it stands for
for unified in objdir.rglob('Unified_*'):
if unified.suffix in SOURCE_SUFFIXES:
for included in re.findall(r'^#include "([^"]+)"', unified.read_text(errors='replace'), re.M):
record(unified.parent / included, unified.parent, unified.name)
for backend in objdir.rglob('backend.mk'):
makefile = (backend.parent / 'Makefile').read_text()
srcdir = tree / re.search(r'^relativesrcdir := (\S+)', makefile, re.M).group(1)
for source in re.findall(r'^\w*(?:SRCS|ASFILES) \+= (\S+)', backend.read_text(errors='replace'), re.M):
source = source.replace('$(srcdir)', str(srcdir)).replace('$(topsrcdir)', str(tree))
record(backend.parent / source, backend.parent, source)
return objects, built
def compile_objects(objects, objdir):
"""Compile them under one make, so its job server shares the cores between directories"""
# make does not load an object's header dependencies when the object is
# the goal, so one left by an earlier run would pass without compiling.
for directory, names in objects.items():
for name in names:
(directory / name).unlink(missing_ok=True)
rules = [f'{n}:\n\t+$(MAKE) -C {directory} {" ".join(sorted(names))}\n'
for n, (directory, names) in enumerate(sorted(objects.items()))]
wrapper = objdir / 'preflight.mk'
targets = ' '.join(str(n) for n in range(len(rules)))
wrapper.write_text(f'.PHONY: all {targets}\nall: {targets}\n' + ''.join(rules))
run(f'make -f {wrapper} -k -j{os.cpu_count()} --output-sync=target')
if __name__ == '__main__':
browser = Path.cwd()
tree = (browser / sys.argv[1]).resolve()
with temp_cd(tree):
run('./mach configure')
# pre-compile generates the last headers the objects include
run('./mach build pre-export export pre-compile')
objdir = Path(json.loads(subprocess.check_output(['./mach', 'environment', '--format=json']))['topobjdir'])
touched = touched_sources(browser)
objects, built = objects_of(touched, tree, objdir)
print(f'{len(touched)} sources are patched or added; this target builds {len(built)} of them, '
f'in {sum(map(len, objects.values()))} objects')
if not objects:
sys.exit(f'error: found no object for a patched source in {objdir}')
compile_objects(objects, objdir)