From f949d5fcc40445c83327314ccfb867f175f9f06b Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 17 Sep 2026 02:34:03 -0400 Subject: [PATCH] ci(mobile): fail CI when the RPC recording pin leaves main's history or the corpus does not reproduce (#21156) * test(mobile): fail CI when the RPC recording pin leaves main's history `mobile/rpc-foundation/pilot-scenarios.json` carries the commit every golden claims it was recorded from, and `--record` refuses on any other tree. A behaviour-change branch pins its own last fenced commit, which stops being reachable the moment the branch squash-merges: nobody can record on main again until a hand-made repin lands, and until now only a human noticed. #21123 was that, and so was the repin after #20954. `scripts/rpc-recording-pin-guard.mts ancestry` fails when the pin is not an ancestor of the commit under test, and prints the repin recipe. It refuses to answer on a shallow clone rather than trusting grafted history, so the job checks out with `fetch-depth: 0`. Ordinary product drift past a reachable pin is not a failure. `reproduce` makes the other claim the corpus header makes, which the recording suites do not: they replay the goldens against the CURRENT tree, so a golden recorded somewhere other than the pin -- a merge that auto-merged golden JSON, a refresh copied back from a scratch directory -- passes them and is what the header exists to deny. It checks the pin out detached, lays this tree's recorder and manifest over it, and lets the same suites compare in place, so the comparison is `compareGolden` with lockfile and platform masked as ever. It runs unconditionally on a push to main, which has no `verify` job and is where a squash lands a spliced corpus. On a pull request it runs only when the corpus, the manifest or the recorder moved: nothing else can move the verdict away from the one the base commit published, and `verify` replays the corpus against the branch tree meanwhile. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): judge the recording pin against the tree it was read from Round-1 review of the pin guard. The pull_request ancestry check read the pin out of the merge preview and judged it against the branch head. Those differ whenever main repins after the branch point, so ordinary stale branches failed, and the instruction told the author to repin to their own head -- which creates the unreachable pin the guard exists to catch. Judge the checked-out tree instead. `git worktree prune` in the reproduce teardown was repository-wide. This git directory is shared by every worktree on the machine (611 registered here), so it could deregister an unrelated one whose directory was momentarily missing. `worktree remove --force` alone is enough; a failure to remove is now reported rather than papered over. Also: the concurrency group is per commit on main, because GitHub cancels a pending run in a group whatever `cancel-in-progress` says; the skip gate fails closed when a provenance path stops matching instead of skipping forever; the census-boundary comment states the rule the code uses; and five exports with no consumer are now module-private. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): let an untracked golden and the guard itself buy a reproduction Two bot findings on the skip gate. `git diff` sees tracked paths only, but the reproduction's overlay copy and its census both read the corpus directory as it sits on disk, so an untracked golden or manifest is input to the verdict and used to skip the run that would judge it. Enumerate untracked entries under the provenance paths the way the recorder already does, and run rather than skip: an unjudged local addition is the case the reproduction exists for. The guard script is now a provenance path of its own, so a change to it re-runs the reproduction it implements. Left alone deliberately: run-process.ts and the workflow's `paths:` scope over src/shared, which is a pre-existing gap for the whole mobile workflow rather than this job's. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): refuse to reproduce when the suite list has drifted from the files Round-2 review. The suite names reach vitest as positional filename filters, and vitest exits 0 when only some of them match. A renamed census suite therefore dropped out of the reproduction silently and the guard still printed that the corpus reproduces: three files and 761 tests instead of four and 762, exit 0. Resolve every name under the recorder overlay before spawning, and throw naming the drifted entry. The unit case walks the list and omits each name in turn, so no single rename can slip past it. This is the same fail-open shape as the renamed-pathspec finding. Also: pass an explicit directory type to `symlink`, since Windows needs one and a junction needs no privilege where a real symlink does; and build the throwaway test repositories with `symbolic-ref` rather than `--initial-branch`, which needs git 2.28 against a declared baseline of 2.25. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .github/workflows/mobile.yml | 68 +++- mobile/scripts/rpc-recording-pin-guard.mts | 332 ++++++++++++++++++ .../scripts/rpc-recording-pin-guard.test.ts | 262 ++++++++++++++ mobile/vitest.config.ts | 4 +- 4 files changed, 664 insertions(+), 2 deletions(-) create mode 100644 mobile/scripts/rpc-recording-pin-guard.mts create mode 100644 mobile/scripts/rpc-recording-pin-guard.test.ts diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 8a648a1e4ba..c16a1e9892b 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -31,13 +31,26 @@ on: - '.github/workflows/mobile.yml' - '.github/actions/install-node-dependencies/**' - '.github/workflows/mobile-ios-release.yml' + # Why main too: a behaviour-change branch legitimately pins its own last fenced commit, and that + # commit only stops being reachable when the branch squash-merges. The pull_request run cannot + # see that; this one is where the pin guard finds it. + push: + branches: + - main + paths: + - 'mobile/**' + - '.github/workflows/mobile.yml' concurrency: - group: mobile-${{ github.event.pull_request.number || github.ref }} + # Per commit on main, not per branch. GitHub cancels any PENDING run in a group when a new one + # queues, whatever `cancel-in-progress` says, so one shared main group drops the middle merge of + # three -- and a pin that breaks there is exactly what this workflow now checks for. + group: mobile-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: verify: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest env: @@ -101,3 +114,56 @@ jobs: - name: Check formatting run: pnpm format:check + + recording-pin: + name: RPC recording pin + runs-on: ubuntu-latest + + defaults: + run: + working-directory: mobile + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The ancestry verdict is read straight off history. On a shallow checkout + # `git merge-base --is-ancestor` answers from grafted parents, so the guard refuses to + # answer at all rather than reporting a pass it has no evidence for -- and the pinned tree + # below has to be checkable out. + fetch-depth: 0 + + - uses: ./.github/actions/install-node-dependencies + with: + cache-dependency-path: | + pnpm-lock.yaml + mobile/pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Seconds. No `--ref`, so the pin is judged against the same tree it was read out of. On a + # pull request that is the merge preview, which already carries main's repins; judging the + # branch head instead fails every branch cut before the day's repin, and its instruction would + # tell the author to pin their own head -- creating the break this guard exists to catch. A + # branch that pins its own commit passes here and fails on the push after the squash, which is + # where the pin actually leaves the history. + - name: Check the recording pin is reachable + shell: bash + run: pnpm exec tsx scripts/rpc-recording-pin-guard.mts ancestry + + # ~2 min locally for the record itself, so it is gated rather than run twice over. A pull + # request that moves none of the corpus, the manifest or the recorder cannot move this + # verdict away from the one the base commit already published, and `verify` replays the + # corpus against the branch tree in the meantime. A push to main has no `verify` job and is + # where a squash lands a spliced corpus, so there it always runs. + - name: Reproduce the corpus from the pinned tree + shell: bash + env: + PIN_GUARD_BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ -n "$PIN_GUARD_BASE" ]; then + pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce --if-changed-since "$PIN_GUARD_BASE" + else + pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce + fi diff --git a/mobile/scripts/rpc-recording-pin-guard.mts b/mobile/scripts/rpc-recording-pin-guard.mts new file mode 100644 index 00000000000..c7b11a48495 --- /dev/null +++ b/mobile/scripts/rpc-recording-pin-guard.mts @@ -0,0 +1,332 @@ +/** + * Guards the two claims `mobile/rpc-foundation/goldens` makes about its pin. + * + * `ancestry` — `baseline` names a commit in this history. A behaviour-change branch pins its own + * last fenced commit; that commit stops being reachable the moment the branch + * squash-merges, and nobody can run `--record` on main again until a hand-made repin + * lands. Ordinary product drift past a reachable pin is normal and is not a failure. + * `reproduce` — the goldens on disk are what the recorder produces from the PINNED tree. The + * recording suites replay the corpus against the CURRENT tree on every run, which is + * the same claim only while the fenced tree still matches the pin. + */ +import { existsSync } from 'node:fs' +import { createRequire } from 'node:module' +import { cp, mkdtemp, rm, symlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { runProcess } from '../../src/shared/child-process/run-process.ts' +import { RECORDING_DRIVERS } from '../src/test-support/rpc-recording/recording-drivers.ts' +import { readScenarios } from '../src/test-support/rpc-recording/scenario-input.ts' + +/** The recorder is exempt from the fence, so a reproduction lays the candidate copy over the pin. */ +const RECORDER_OVERLAY = 'mobile/src/test-support/rpc-recording' +/** + * Everything whose change can move the reproduction's verdict: the corpus it compares against, the + * manifest that names the pin and derives the scenarios, the recorder it lays over the pinned + * sources, and this guard, which drives the run. A revision that moves none of these cannot move + * the verdict, which is what lets a pull request skip the run. + */ +const CORPUS_PROVENANCE_PATHS = [ + 'mobile/rpc-foundation', + RECORDER_OVERLAY, + 'mobile/scripts/rpc-recording-pin-guard.mts' +] as const +/** + * The corpus readers that are not drivers. Boundary: a suite belongs here when its verdict is a + * function of the corpus bytes themselves. The `mutants/` suites read the same directory but assert + * that the corpus DETECTS a planted mutation, which is a different claim than reproducing it. + * `derived-goldens` is the census a whole golden spliced in by a merge trips, which no per-golden + * compare can see. + */ +const CORPUS_CENSUS_SUITES = [ + 'derived-goldens.test.ts', + 'golden-recorder-failure-absence.test.ts' +] as const +/** Every suite the reproduction runs against the pinned tree. */ +export const REPRODUCTION_SUITES = [...RECORDING_DRIVERS, ...CORPUS_CENSUS_SUITES] as const +const RECORDING_TIMEOUT_MS = 900_000 +// Windows needs an explicit type for a directory link and a junction needs no privilege, where a +// real symlink does; POSIX ignores the argument. Same rule as src/main/ipc/worktree-symlinks.ts. +const DIRECTORY_LINK = process.platform === 'win32' ? 'junction' : 'dir' +// A failing reproduction prints one diff per golden; 8 MB clips that mid-report. +const RECORDING_OUTPUT_BYTES = 64 * 1024 * 1024 + +/** + * These names reach vitest as positional filename filters, and vitest exits 0 when only some of + * them match. A renamed suite would drop out of the run silently and still report a reproduction, + * so resolve every one of them first. + */ +export function assertReproductionSuitesExist(root: string): void { + for (const suite of REPRODUCTION_SUITES) { + if (!existsSync(resolve(root, RECORDER_OVERLAY, suite))) { + throw new Error( + `${suite} is not in ${RECORDER_OVERLAY}. This list names the suites the reproduction runs ` + + 'and has drifted from the files, which vitest would pass over without a word.' + ) + } + } +} + +export type PinAncestryFailure = 'shallow' | 'unreachable' | 'not-an-ancestor' +export type PinAncestryVerdict = + | { ok: true; baseline: string; ref: string } + | { ok: false; baseline: string; ref: string; failure: PinAncestryFailure; message: string } + +async function git(cwd: string, args: readonly string[]) { + return await runProcess({ program: 'git', args: [...args], cwd }) +} +function readPinnedBaseline(root: string): string { + return readScenarios(resolve(root, 'mobile/rpc-foundation/pilot-scenarios.json')).baseline +} +export function repinInstruction(baseline: string, ref: string, cause: string): string { + return [ + `The RPC recording corpus is pinned to a commit that ${cause}.`, + '', + ` baseline ${baseline} (mobile/rpc-foundation/pilot-scenarios.json)`, + ` head ${ref}`, + '', + 'Every golden under mobile/rpc-foundation/goldens claims it was recorded from that tree, and', + '`--record` refuses on any other tree, so the corpus cannot be refreshed until the pin names a', + 'commit that is reachable from here. Repin and re-record, both in one commit:', + '', + ` git switch -c repin-rpc-recording ${ref}`, + ` # set "baseline" in mobile/rpc-foundation/pilot-scenarios.json to ${ref}`, + ' ORCA_BACKGROUND_LAUNCH=1 RPC_FOUNDATION_RECORD=1 \\', + ' pnpm --dir mobile exec tsx scripts/rpc-recording.mts --record', + '', + 'Re-record everything: the repin rewrites the `baseline` header of every golden, so a partial', + 'refresh leaves the corpus pinned to two different trees. See', + 'mobile/src/test-support/rpc-recording/README.md, "Recording a behaviour change".' + ].join('\n') +} +const SHALLOW_MESSAGE = [ + 'Cannot judge the recording pin: this is a shallow clone.', + '', + '`git merge-base --is-ancestor` answers from grafted history, so it would report a verdict this', + 'guard has no evidence for. Check out with `fetch-depth: 0`.' +].join('\n') + +export async function checkPinAncestry( + root: string, + baseline: string, + ref: string +): Promise { + const shallow = await git(root, ['rev-parse', '--is-shallow-repository']) + if (shallow.code !== 0) { + throw new Error(`Could not ask git whether the clone is shallow: ${shallow.stderr.trim()}`) + } + if (shallow.stdout.trim() !== 'false') { + return { ok: false, baseline, ref, failure: 'shallow', message: SHALLOW_MESSAGE } + } + const head = await git(root, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`]) + if (head.code !== 0) { + throw new Error(`Cannot resolve ${ref} to a commit in this repository`) + } + // Resolved, because the instruction below is a command to paste: `HEAD` in it moves with whatever + // the reader has checked out by the time they read the log. + const resolved = head.stdout.trim() + const pinned = await git(root, ['rev-parse', '--verify', '--quiet', `${baseline}^{commit}`]) + if (pinned.code !== 0) { + return { + ok: false, + baseline, + ref, + failure: 'unreachable', + message: repinInstruction(baseline, resolved, 'is not a commit in this repository at all') + } + } + const ancestor = await git(root, ['merge-base', '--is-ancestor', baseline, ref]) + if (ancestor.code === 0) { + return { ok: true, baseline, ref } + } + // Why only 1: git reserves higher codes for real errors, and treating one as "not an ancestor" + // would turn a broken repository into a repin instruction nobody can act on. + if (ancestor.code !== 1) { + throw new Error(`git merge-base --is-ancestor failed: ${ancestor.stderr.trim()}`) + } + return { + ok: false, + baseline, + ref, + failure: 'not-an-ancestor', + message: repinInstruction(baseline, resolved, 'is not an ancestor of this commit') + } +} + +/** Whether anything a reproduction reads from the candidate tree moved since `since`. */ +export async function corpusProvenanceChanged(root: string, since: string): Promise { + // Fail closed on a rename: `git diff --quiet` reports "nothing changed" for a pathspec that + // matches no file, which would skip the reproduction forever and report success. + for (const path of CORPUS_PROVENANCE_PATHS) { + const tracked = await git(root, ['ls-files', '--error-unmatch', '--', path]) + if (tracked.code !== 0) { + throw new Error( + `${path} is not a tracked path. This gate decides whether to reproduce the corpus by ` + + 'diffing it, so a rename has to move this list with it.' + ) + } + } + // The branch point, not the base tip: a base that moved on without this branch would otherwise + // read as this branch's change. This is for local invocations, which pass a branch tip. Under CI + // `HEAD` is the merge preview whose first parent is the base, so it resolves to `since` itself. + const branchPoint = await git(root, ['merge-base', since, 'HEAD']) + const from = branchPoint.code === 0 ? branchPoint.stdout.trim() : since + // `git diff` sees tracked paths only, but the overlay copy and the census both read these + // directories as they sit on disk, so an untracked golden or manifest is input to the verdict. + // Run rather than skip: an unjudged local addition is the case the reproduction exists for. + const untracked = await git(root, [ + 'ls-files', + '--others', + '--exclude-standard', + '--', + ...CORPUS_PROVENANCE_PATHS + ]) + if (untracked.code !== 0) { + throw new Error(`Could not enumerate untracked corpus files: ${untracked.stderr.trim()}`) + } + if (untracked.stdout.trim() !== '') { + return true + } + const diff = await git(root, ['diff', '--quiet', from, '--', ...CORPUS_PROVENANCE_PATHS]) + if (diff.code !== 0 && diff.code !== 1) { + throw new Error(`Could not diff the corpus against ${from}: ${diff.stderr.trim()}`) + } + return diff.code === 1 +} + +/** + * Replay the corpus against the pinned tree instead of the current one: check the pin out detached, + * lay this tree's recorder and manifest over it (both exempt from the fence, and both are what the + * goldens pin by digest rather than by commit), and let the recording suites compare in place. The + * comparison is `compareGolden`, so lockfile and platform stay masked the way they are on every + * other run. + */ +async function reproduceFromPin(root: string, baseline: string): Promise { + assertReproductionSuitesExist(root) + const scratch = await mkdtemp(join(tmpdir(), 'rpc-recording-pin-')) + const tree = join(scratch, 'tree') + try { + const added = await git(root, ['worktree', 'add', '--detach', tree, baseline]) + if (added.code !== 0) { + throw new Error(`Could not check out the pinned tree ${baseline}: ${added.stderr.trim()}`) + } + await symlink(resolve(root, 'node_modules'), join(tree, 'node_modules'), DIRECTORY_LINK) + await symlink( + resolve(root, 'mobile/node_modules'), + join(tree, 'mobile/node_modules'), + DIRECTORY_LINK + ) + await rm(join(tree, RECORDER_OVERLAY), { recursive: true, force: true }) + await cp(resolve(root, RECORDER_OVERLAY), join(tree, RECORDER_OVERLAY), { recursive: true }) + const require = createRequire(resolve(root, 'mobile/package.json')) + const result = await runProcess({ + program: process.execPath, + args: [ + resolve(require.resolve('vitest/package.json'), '../vitest.mjs'), + 'run', + ...REPRODUCTION_SUITES.map((suite) => `src/test-support/rpc-recording/${suite}`) + ], + cwd: join(tree, 'mobile'), + timeoutMs: RECORDING_TIMEOUT_MS, + maxOutputBytes: RECORDING_OUTPUT_BYTES, + env: { + ...process.env, + ORCA_BACKGROUND_LAUNCH: '1', + // Replay, never `--record`: the suites read these two from the candidate tree and compare. + RPC_FOUNDATION_GOLDENS: resolve(root, 'mobile/rpc-foundation/goldens'), + RPC_FOUNDATION_SCENARIOS: resolve(root, 'mobile/rpc-foundation/pilot-scenarios.json') + } + }) + process.stdout.write(result.stdout) + process.stderr.write(result.stderr) + if (result.outputTruncated) { + process.stderr.write('\nReproduction output was clipped; the report above is incomplete.\n') + } + if (result.timedOut) { + throw new Error( + `Reproducing the corpus did not finish within ${RECORDING_TIMEOUT_MS / 1000}s and was killed.` + ) + } + return result.code === 0 + } finally { + await removeScratchWorktree(root, tree) + await rm(scratch, { recursive: true, force: true }) + } +} + +/** + * Deregisters the scratch checkout and nothing else. Never `git worktree prune`: that is + * repository-wide, and this git directory is shared by every worktree on the machine, so a prune + * deregisters any of them whose directory is momentarily missing. + */ +export async function removeScratchWorktree(root: string, tree: string): Promise { + const removed = await git(root, ['worktree', 'remove', '--force', tree]) + if (removed.code !== 0) { + process.stderr.write( + `Could not deregister the scratch worktree ${tree}: ${removed.stderr.trim()}\n` + + 'It stays registered against this repository until you prune it yourself.\n' + ) + } +} + +const REPRODUCTION_FAILURE = [ + 'The goldens on disk are not what the recorder produces from the pinned tree.', + '', + 'Each divergence above is a golden whose header names a tree that does not produce it. A merge', + 'that auto-merged golden JSON, or a refresh recorded somewhere other than the pin, both land', + 'here. Re-record the whole corpus from the pin rather than editing a golden:', + '', + ' ORCA_BACKGROUND_LAUNCH=1 RPC_FOUNDATION_RECORD=1 \\', + ' pnpm --dir mobile exec tsx scripts/rpc-recording.mts --record' +].join('\n') + +async function main(argv: readonly string[]): Promise { + const check = argv[0] + const root = resolve(import.meta.dirname, '../..') + const baseline = readPinnedBaseline(root) + if (check === 'ancestry') { + const ref = argv.includes('--ref') ? argv[argv.indexOf('--ref') + 1] : 'HEAD' + if (!ref) { + throw new Error('--ref needs a commit') + } + const verdict = await checkPinAncestry(root, baseline, ref) + if (!verdict.ok) { + process.stderr.write(`${verdict.message}\n`) + process.exitCode = 1 + return + } + process.stdout.write(`Recording pin ${baseline} is an ancestor of ${ref}.\n`) + return + } + if (check === 'reproduce') { + const since = argv.includes('--if-changed-since') + ? argv[argv.indexOf('--if-changed-since') + 1] + : undefined + if (argv.includes('--if-changed-since')) { + if (!since) { + throw new Error('--if-changed-since needs a commit') + } + if (!(await corpusProvenanceChanged(root, since))) { + process.stdout.write( + `Nothing this run reads from the working tree moved since ${since}: not the corpus, not\nthe manifest, not the recorder. The verdict is the one that commit already carries.\n` + ) + return + } + } + if (!(await reproduceFromPin(root, baseline))) { + process.stderr.write(`\n${REPRODUCTION_FAILURE}\n`) + process.exitCode = 1 + return + } + process.stdout.write(`\nThe corpus reproduces from the pinned tree ${baseline}.\n`) + return + } + throw new Error( + 'Usage: rpc-recording-pin-guard.mts ancestry [--ref ]\n | reproduce [--if-changed-since ]' + ) +} + +// Importing this module for its verdicts must not run a check; the unit test beside it does that. +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) { + await main(process.argv.slice(2)) +} diff --git a/mobile/scripts/rpc-recording-pin-guard.test.ts b/mobile/scripts/rpc-recording-pin-guard.test.ts new file mode 100644 index 00000000000..e6154d51c2c --- /dev/null +++ b/mobile/scripts/rpc-recording-pin-guard.test.ts @@ -0,0 +1,262 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { runProcess } from '../../src/shared/child-process/run-process' +import { + assertReproductionSuitesExist, + checkPinAncestry, + corpusProvenanceChanged, + removeScratchWorktree, + repinInstruction, + REPRODUCTION_SUITES +} from './rpc-recording-pin-guard.mts' + +const scratch: string[] = [] +async function git(cwd: string, ...args: string[]): Promise { + const result = await runProcess({ program: 'git', args, cwd }) + if (result.code !== 0) { + throw new Error(`git ${args.join(' ')} failed in ${cwd}: ${result.stderr}`) + } + return result.stdout.trim() +} +/** A repository of our own, so no verdict in this file can depend on — or touch — the real refs. */ +async function throwawayRepository(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'pin-guard-')) + scratch.push(directory) + const repository = join(directory, 'repo') + await git(directory, 'init', '--quiet', 'repo') + // `--initial-branch=main` needs git >= 2.28; symbolic-ref before the first commit works on any. + await git(repository, 'symbolic-ref', 'HEAD', 'refs/heads/main') + await git(repository, 'config', 'user.email', 'pin-guard@example.invalid') + await git(repository, 'config', 'user.name', 'Pin Guard') + return repository +} +async function commit(repository: string, body: string): Promise { + await writeFile(join(repository, 'product.ts'), `${body}\n`) + await git(repository, 'add', 'product.ts') + await git(repository, 'commit', '--quiet', '--no-verify', '--message', body) + return await git(repository, 'rev-parse', 'HEAD') +} +async function commitAt(repository: string, path: string, body: string): Promise { + await mkdir(dirname(join(repository, path)), { recursive: true }) + await writeFile(join(repository, path), `${body}\n`) + await git(repository, 'add', path) + await git(repository, 'commit', '--quiet', '--no-verify', '--message', path) + return await git(repository, 'rev-parse', 'HEAD') +} +const RECORDER_FILE = 'mobile/src/test-support/rpc-recording/run-recording.ts' +const GUARD_FILE = 'mobile/scripts/rpc-recording-pin-guard.mts' +/** Every provenance path: the gate refuses to answer when any one of them names nothing. */ +async function seedProvenance(repository: string): Promise { + await commitAt(repository, 'mobile/rpc-foundation/pilot-scenarios.json', 'pin') + await commitAt(repository, GUARD_FILE, 'guard') + return await commitAt(repository, RECORDER_FILE, 'recorder') +} +const MISSING_SHA = '0123456789abcdef0123456789abcdef01234567' + +// File scope, not per-suite: a suite-local hook fires before the later suites have made theirs. +afterAll(async () => { + for (const directory of scratch) { + await rm(directory, { recursive: true, force: true }) + } +}) + +describe('recording pin ancestry', () => { + it('passes when the pin is the commit itself', async () => { + const repository = await throwawayRepository() + const first = await commit(repository, 'one') + expect(await checkPinAncestry(repository, first, first)).toMatchObject({ ok: true }) + }) + + it('passes on ordinary drift: the tree has moved on, the pin is still reachable', async () => { + const repository = await throwawayRepository() + const pin = await commit(repository, 'one') + const head = await commit(repository, 'two') + expect(await checkPinAncestry(repository, pin, head)).toMatchObject({ ok: true }) + }) + + it('passes on a branch that pinned its own commit, judged against that branch', async () => { + const repository = await throwawayRepository() + await commit(repository, 'one') + await git(repository, 'switch', '--quiet', '--create', 'behaviour-change') + const branchPin = await commit(repository, 'two') + const branchHead = await commit(repository, 'three') + expect(await checkPinAncestry(repository, branchPin, branchHead)).toMatchObject({ ok: true }) + }) + + it('passes a branch cut before main repinned, judged against the merge preview', async () => { + const repository = await throwawayRepository() + const branchPoint = await commitAt(repository, 'mobile/src/session/route.ts', 'base') + const mainPin = await commitAt( + repository, + 'mobile/rpc-foundation/pilot-scenarios.json', + 'repin' + ) + await git(repository, 'switch', '--quiet', '--create', 'refactor', branchPoint) + const branchHead = await commitAt(repository, 'mobile/src/session/route.ts', 'migrated') + await git(repository, 'merge', '--quiet', '--no-edit', 'main') + const preview = await git(repository, 'rev-parse', 'HEAD') + // The preview is the tree CI checks out and reads the pin from, so it is the tree to judge. + expect(await checkPinAncestry(repository, mainPin, preview)).toMatchObject({ ok: true }) + // The head sha would have failed this ordinary branch, and told the author to repin to it. + expect(await checkPinAncestry(repository, mainPin, branchHead)).toMatchObject({ + ok: false, + failure: 'not-an-ancestor' + }) + }) + + it('fails once that branch squash-merges and the pin leaves the history', async () => { + const repository = await throwawayRepository() + const base = await commit(repository, 'one') + await git(repository, 'switch', '--quiet', '--create', 'behaviour-change') + const branchPin = await commit(repository, 'two') + await git(repository, 'switch', '--quiet', 'main') + await git(repository, 'reset', '--quiet', '--hard', base) + const squashed = await commit(repository, 'two, squashed') + const verdict = await checkPinAncestry(repository, branchPin, squashed) + expect(verdict).toMatchObject({ ok: false, failure: 'not-an-ancestor' }) + expect(verdict.ok).toBe(false) + if (verdict.ok) { + return + } + expect(verdict.message).toContain(branchPin) + expect(verdict.message).toContain('scripts/rpc-recording.mts --record') + expect(verdict.message).toContain('mobile/rpc-foundation/pilot-scenarios.json') + }) + + it('fails with the same instruction when the pin is no commit at all', async () => { + const repository = await throwawayRepository() + const head = await commit(repository, 'one') + const verdict = await checkPinAncestry(repository, MISSING_SHA, head) + expect(verdict).toMatchObject({ ok: false, failure: 'unreachable' }) + expect(verdict.ok ? '' : verdict.message).toContain('scripts/rpc-recording.mts --record') + }) + + it('refuses to answer on a shallow clone instead of trusting grafted history', async () => { + const repository = await throwawayRepository() + const pin = await commit(repository, 'one') + await commit(repository, 'two') + const head = await commit(repository, 'three') + const clone = join(repository, '..', 'shallow') + await git(repository, 'clone', '--quiet', '--depth', '1', `file://${repository}`, clone) + // The pin is real and reachable in the full repository; only the missing history hides it. + expect(await checkPinAncestry(repository, pin, head)).toMatchObject({ ok: true }) + const verdict = await checkPinAncestry(clone, pin, 'HEAD') + expect(verdict).toMatchObject({ ok: false, failure: 'shallow' }) + expect(verdict.ok ? '' : verdict.message).toContain('fetch-depth: 0') + }) + + it('names the head and the pin in the instruction', () => { + expect( + repinInstruction('a'.repeat(40), 'b'.repeat(40), 'is not an ancestor of this commit') + ).toContain(`git switch -c repin-rpc-recording ${'b'.repeat(40)}`) + }) +}) + +describe('what a reproduction reads from the candidate tree', () => { + it('skips the run when only product sources moved', async () => { + const repository = await throwawayRepository() + await seedProvenance(repository) + const base = await commitAt(repository, 'mobile/src/session/route.ts', 'before') + await commitAt(repository, 'mobile/src/session/route.ts', 'after') + expect(await corpusProvenanceChanged(repository, base)).toBe(false) + }) + + it('runs when a golden moved', async () => { + const repository = await throwawayRepository() + await seedProvenance(repository) + const base = await commitAt(repository, 'mobile/rpc-foundation/goldens/a.json', '{}') + await commitAt(repository, 'mobile/rpc-foundation/goldens/a.json', '{"spliced": true}') + expect(await corpusProvenanceChanged(repository, base)).toBe(true) + }) + + it('runs when the pin itself moved', async () => { + const repository = await throwawayRepository() + await seedProvenance(repository) + const base = await commitAt(repository, 'mobile/rpc-foundation/pilot-scenarios.json', 'one') + await commitAt(repository, 'mobile/rpc-foundation/pilot-scenarios.json', 'two') + expect(await corpusProvenanceChanged(repository, base)).toBe(true) + }) + + it('ignores a corpus change the base branch made without this branch', async () => { + const repository = await throwawayRepository() + await seedProvenance(repository) + const branchPoint = await commitAt(repository, 'mobile/rpc-foundation/goldens/a.json', '{}') + await commitAt(repository, 'mobile/rpc-foundation/goldens/b.json', '{}') + const baseTip = await git(repository, 'rev-parse', 'HEAD') + await git(repository, 'switch', '--quiet', '--create', 'refactor', branchPoint) + await commitAt(repository, 'mobile/src/session/route.ts', 'migrated') + expect(await corpusProvenanceChanged(repository, baseTip)).toBe(false) + }) + + it('runs when the recorder moved, because every golden pins it by digest', async () => { + const repository = await throwawayRepository() + const base = await seedProvenance(repository) + await commitAt(repository, RECORDER_FILE, 'two') + expect(await corpusProvenanceChanged(repository, base)).toBe(true) + }) + + it('runs when the guard itself moved, because it decides the skip and drives the run', async () => { + const repository = await throwawayRepository() + const base = await seedProvenance(repository) + await commitAt(repository, GUARD_FILE, 'changed') + expect(await corpusProvenanceChanged(repository, base)).toBe(true) + }) + + it('runs on an untracked golden, which no diff of tracked paths can see', async () => { + const repository = await throwawayRepository() + const base = await seedProvenance(repository) + const golden = join(repository, 'mobile/rpc-foundation/goldens/local.json') + await mkdir(dirname(golden), { recursive: true }) + await writeFile(golden, '{}\n') + // The overlay copy and the census both read the directory as it sits on disk. + expect(await corpusProvenanceChanged(repository, base)).toBe(true) + }) + + it('refuses to answer when a provenance path names nothing, instead of skipping forever', async () => { + const repository = await throwawayRepository() + const base = await seedProvenance(repository) + await git(repository, 'mv', 'mobile/rpc-foundation', 'mobile/rpc-corpus') + await git(repository, 'commit', '--quiet', '--no-verify', '--message', 'rename the corpus') + await expect(corpusProvenanceChanged(repository, base)).rejects.toThrow('not a tracked path') + }) +}) + +describe('scratch worktree teardown', () => { + it('leaves an unrelated worktree registered when its directory is missing', async () => { + const repository = await throwawayRepository() + await commit(repository, 'one') + const trees = join(repository, '..', 'trees') + for (const name of ['scratch', 'kept', 'unmounted']) { + await git(repository, 'worktree', 'add', '--quiet', '--detach', join(trees, name)) + } + // Stands in for a worktree on an unmounted volume: `git worktree prune` would deregister it. + await rm(join(trees, 'unmounted'), { recursive: true, force: true }) + await removeScratchWorktree(repository, join(trees, 'scratch')) + const registered = await git(repository, 'worktree', 'list', '--porcelain') + expect(registered).toContain('trees/kept') + expect(registered).toContain('trees/unmounted') + expect(registered).not.toContain('trees/scratch') + }) +}) + +describe('the suites a reproduction runs', () => { + const overlay = 'mobile/src/test-support/rpc-recording' + + it('every name resolves to a file in this repository', () => { + expect(() => assertReproductionSuitesExist(resolve(import.meta.dirname, '../..'))).not.toThrow() + }) + + it('throws for the one that drifted, rather than letting vitest pass over it', async () => { + for (const renamed of REPRODUCTION_SUITES) { + const root = await mkdtemp(join(tmpdir(), 'pin-guard-suites-')) + scratch.push(root) + await mkdir(join(root, overlay), { recursive: true }) + for (const suite of REPRODUCTION_SUITES.filter((name) => name !== renamed)) { + await writeFile(join(root, overlay, suite), '') + } + expect(() => assertReproductionSuitesExist(root)).toThrow(renamed) + } + }) +}) diff --git a/mobile/vitest.config.ts b/mobile/vitest.config.ts index 981a1b55e74..b5dcfb98de1 100644 --- a/mobile/vitest.config.ts +++ b/mobile/vitest.config.ts @@ -13,6 +13,8 @@ export default defineConfig({ onConsoleLog: (log) => !log.includes('react-test-renderer is deprecated'), // .tsx too: component tests exist (react-test-renderer + mocked react-native) and were // silently never collected, so render-level regressions shipped untested. - include: ['src/**/*.test.ts', 'src/**/*.test.tsx'] + // scripts/ too: the CI guards under it (the RPC recording pin) had no runnable test home, + // and a test vitest never collects is not a gate. + include: ['src/**/*.test.ts', 'src/**/*.test.tsx', 'scripts/**/*.test.ts'] } })