diff --git a/.gitignore b/.gitignore index 3b83c9921b8..bea9d620320 100644 --- a/.gitignore +++ b/.gitignore @@ -115,6 +115,7 @@ docs/** !docs/reference/ssh-execution-boundary.md !docs/reference/ssh-host-key-verification.md !docs/reference/ssh-reconnect-source-recovery.md +!docs/reference/upstream-breakage-diagnosis.md !docs/reference/windows-setup-shell.md !docs/reference/worktree-scan-fingerprint.md !docs/reference/wsl-command-execution.md diff --git a/AGENTS.md b/AGENTS.md index 2ae36ccd4fd..dd4e94b68db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,17 @@ Never use vague names like `helpers`, `utils`, `common`, `misc`, or `shared-stuf Always use the primary working directory (the worktree) for all file reads and edits. Never follow absolute paths from subagent results that point to the main repo. +## Red After Merging Main + +Before diagnosing a branch — or a whole stack — that went red after merging `main`, check whether `main` was already broken. No workflow runs the test suite on a push to `main`, so its commits carry no checks and the question has to be answered from the PRs whose CI ran against `main` at that moment: + +```sh +pnpm run diagnose:upstream-breakage at --window-hours 1 +pnpm run diagnose:upstream-breakage compare ... +``` + +An identical failure set across independent branches is upstream by construction, and the repair is to merge a newer `main` — not to fix the branches. The probe answers `broken` / `clean` / `unknown` and never reports `clean` without positive evidence. See [`docs/reference/upstream-breakage-diagnosis.md`](./docs/reference/upstream-breakage-diagnosis.md). + ## Cross-Platform Support Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior behind runtime checks: diff --git a/config/scripts/upstream-breakage-check-runs.fixture.json b/config/scripts/upstream-breakage-check-runs.fixture.json new file mode 100644 index 00000000000..cfaab10aa09 --- /dev/null +++ b/config/scripts/upstream-breakage-check-runs.fixture.json @@ -0,0 +1,514 @@ +{ + "_source": "Captured verbatim from GitHub: gh api repos/{owner}/{repo}/commits//check-runs", + "prs": { + "16917": { + "pr": { + "baseRefName": "main", + "headRefName": "brennanb2025/mob-create-flow-perf", + "headRefOid": "29be0cc2e032ef03b93ebda9c03809bf9f488c1f", + "mergedAt": "2026-08-28T22:45:54Z", + "number": 16917, + "title": "fix(mobile): create-worktree sheet dies after picking a source and loses the picked PR" + }, + "runs": [ + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "test", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "managed hooks on Node 18", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "e2e", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "detect changed e2e specs", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "Git compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "static analysis", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "package", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "cross-version wire compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "shell contracts", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "package (windows)", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "orcad browser provider", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "prepare test native cache node 24", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "typecheck", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:36Z", + "conclusion": "success", + "name": "verify", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "skipped", + "name": "xterm patch sync", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:29Z", + "conclusion": "success", + "name": "root directory guard", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:30Z", + "conclusion": "success", + "name": "detect code-relevant changes", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:23:12Z", + "conclusion": "success", + "name": "test vs non-test LoC", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T11:25:42Z", + "conclusion": "success", + "name": "verify", + "status": "completed" + } + ] + }, + "17002": { + "pr": { + "baseRefName": "main", + "headRefName": "nwparker/headless-linux-server-doc-fixes", + "headRefOid": "3d4f9bb9158cf5aaa6f61f88afc63fa00f0add4a", + "mergedAt": "2026-08-28T09:37:52Z", + "number": 17002, + "title": "docs(headless-server): fix package list, extraction perms, and ldd command" + }, + "runs": [ + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "Git compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "prepare test native cache node 24", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "managed hooks on Node 18", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "xterm patch sync", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "test", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "shell contracts", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "package (windows)", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "e2e", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "package", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "orcad browser provider", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "static analysis", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "detect changed e2e specs", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "typecheck", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:30Z", + "conclusion": "skipped", + "name": "cross-version wire compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:35Z", + "conclusion": "success", + "name": "verify", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:29Z", + "conclusion": "success", + "name": "detect code-relevant changes", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:29Z", + "conclusion": "success", + "name": "root directory guard", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-28T09:13:13Z", + "conclusion": "success", + "name": "test vs non-test LoC", + "status": "completed" + } + ] + }, + "17358": { + "pr": { + "baseRefName": "main", + "headRefName": "nwparker/react-purity-lint-cleanup", + "headRefOid": "fb2fde0bcc0290c96b5d5a7005d118dd89de63f5", + "mergedAt": "2026-08-30T08:17:29Z", + "number": 17358, + "title": "Consolidate the renderer now-clock and drop the epoch setState round-trip" + }, + "runs": [ + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:17:08Z", + "conclusion": "success", + "name": "verify", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:08:11Z", + "conclusion": "skipped", + "name": "e2e", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "prepare test native cache node 24", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "orcad browser provider", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "managed hooks on Node 18", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "cross-version wire compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "xterm patch sync", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "Git compatibility", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "skipped", + "name": "shell contracts", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:15:35Z", + "conclusion": "success", + "name": "test / tests node 24 1/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:12:59Z", + "conclusion": "success", + "name": "test / tests node 24 2/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:13:04Z", + "conclusion": "success", + "name": "test / tests node 24 5/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:13:37Z", + "conclusion": "success", + "name": "test / tests node 24 6/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:11:28Z", + "conclusion": "success", + "name": "test / tests node 24 4/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:16:59Z", + "conclusion": "success", + "name": "test / tests node 24 7/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:11:51Z", + "conclusion": "success", + "name": "test / tests node 24 3/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:11:49Z", + "conclusion": "success", + "name": "test / tests node 24 8/8", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:13:14Z", + "conclusion": "success", + "name": "static analysis", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:11:58Z", + "conclusion": "success", + "name": "package (windows)", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:10:58Z", + "conclusion": "success", + "name": "package", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:08:11Z", + "conclusion": "success", + "name": "detect changed e2e specs", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:09:16Z", + "conclusion": "success", + "name": "typecheck", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:07:45Z", + "conclusion": "success", + "name": "root directory guard", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:30Z", + "conclusion": "success", + "name": "detect code-relevant changes", + "status": "completed" + }, + { + "appSlug": "greptile-apps", + "completedAt": "2026-08-30T08:07:17Z", + "conclusion": "success", + "name": "Greptile Review", + "status": "completed" + }, + { + "appSlug": "github-actions", + "completedAt": "2026-08-30T08:06:39Z", + "conclusion": "success", + "name": "test vs non-test LoC", + "status": "completed" + } + ] + } + }, + "siblings": [ + { + "baseRefName": "brennanb2025/claude-structured-mobile", + "headRefName": "brennanb2025/claude-structured-core-merge", + "number": 15356, + "title": "Merge codex-structured-core into Claude lane" + }, + { + "baseRefName": "brennanb2025/claude-structured-mobile", + "headRefName": "brennanb2025/claude-lane-replay-waiter-identity", + "number": 15657, + "title": "fix(native-chat): adopt only the real user replay as a Claude send's identity" + }, + { + "baseRefName": "brennanb2025/probe-liveness-coercion-r1", + "headRefName": "brennanb2025/window-close-guard-r1", + "number": 17044, + "title": "fix(terminal): ask before closing the window when a terminal probe could not answer" + }, + { + "baseRefName": "brennanb2025/probe-liveness-coercion-r1", + "headRefName": "brennanb2025/pty-absence-verdict-r1", + "number": 17058, + "title": "fix(pty): tell a watched local exit apart from a lost route" + }, + { + "baseRefName": "brennanb2025/probe-liveness-coercion-r1", + "headRefName": "brennanb2025/cleanup-post-confirm-liveness-r1", + "number": 17074, + "title": "fix(cleanup): stop deleting a workspace after confirm when it turns out to be busy" + } + ] +} diff --git a/config/scripts/upstream-breakage-evidence.mjs b/config/scripts/upstream-breakage-evidence.mjs new file mode 100644 index 00000000000..7205f7af95f --- /dev/null +++ b/config/scripts/upstream-breakage-evidence.mjs @@ -0,0 +1,519 @@ +// Decides whether a set of red branches is red because `main` was already broken +// (upstream) or because the branches themselves broke something. +// +// Why this exists: `main` carries no checks of its own. `pr.yml` — the workflow +// that holds the tests, typecheck and static analysis — is `on: pull_request` +// only, and `unit-tests.yml` is `workflow_call` only, so no commit on `main` has +// a test result attached to it. Asking "was main green at SHA X?" directly +// returns nothing, which is why nobody reaches for it. The answer has to be +// reconstructed from the PRs whose CI ran against `main` at that moment. +// +// See docs/reference/upstream-breakage-diagnosis.md. + +// Verdicts. `unknown` is the default: a probe that cannot see enough evidence +// must never report `clean`, because "no evidence of breakage" and "evidence of +// no breakage" are the two states this tool exists to keep apart. +export const VERDICT = { + broken: 'broken', + clean: 'clean', + unknown: 'unknown', + upstream: 'upstream', + sharedAncestor: 'shared-ancestor', + divergent: 'divergent', + noFailures: 'no-failures' +} + +// Roll-up jobs reprint another job's failure; counting one as an independent +// failure double-counts the same breakage and makes divergent sets look identical. +export const ROLLUP_CHECKS = new Set(['verify']) + +// Reds that are red for reasons unrelated to the commit under test. Counting +// these as breakage makes every window look broken. +export const KNOWN_FALSE_REDS = new Set([ + 'test / tests node 24 1/8', + 'test / tests node 24 6/8', + 'e2e / ssh docker watcher isolation' +]) + +// Third-party review bots post checks on the same commit. They are not signal +// about the tree, so only first-party Actions checks count by default. +export const DEFAULT_APP_SLUGS = new Set(['github-actions']) + +// Lanes that actually execute the tree, so a green one is evidence `main` built, +// type-checked or ran. Everything else on a PR — the path classifier, the +// root-directory guard, the LoC counter, the community-PR labeller — stays green +// on a `main` that is entirely broken, so a witness carrying only those witnessed +// nothing. +// +// An allowlist, and deliberately so: an unrecognised name here costs a witness +// and the verdict degrades to `unknown`, whereas an unrecognised name in a +// denylist would count as a full witness and hand back `clean` — the failure this +// list exists to close. `normalizeChecks` reports every name it did not +// recognise so a renamed lane is loud rather than silently dropped. +export const WITNESSING_CHECK_PATTERNS = [ + // pr.yml: the lanes that build, type-check, lint or run the suite. + /^typecheck$/, + /^static analysis$/, + /^test$/, + /^test \/ tests\b/, + /^prepare test native cache\b/, + /^package$/, + /^package \(/, + /^managed hooks on Node \d+$/, + /^shell contracts$/, + /^Git compatibility$/, + /^xterm patch sync$/, + /^cross-version wire compatibility$/, + /^orcad browser provider$/, + // e2e.yml: the shards and the app build. `changed e2e specs` only reads the + // diff, so it is not one of these. + /^e2e$/, + /^e2e \d+-of-\d+$/, + /^e2e \/ (?!changed e2e specs$)/, + /^build e2e app$/, + /^prepare Electron native cache$/, + /^ssh docker watcher isolation$/, + // Platform smoke and IME lanes. + /^real IME/, + /^native-smoke \(/, + /^mac-native-owner-smoke$/ +] + +// Would this check have gone red if `main` were broken? +export function isWitnessingCheck(name, patterns = WITNESSING_CHECK_PATTERNS) { + return patterns.some((pattern) => pattern.test(name)) +} + +// Branches everything else forks from. Two PRs based on trunk are independent; +// two based on the same feature branch share that branch's diff. +export const TRUNK_REFS = new Set(['main', 'master']) + +const FAILING_CONCLUSIONS = new Set(['failure', 'timed_out', 'action_required']) +const NON_RUN_CONCLUSIONS = new Set(['skipped', 'neutral']) + +// Minimum independent witnesses before any positive verdict. One witness cannot +// separate "main is broken" from "this branch is broken". +export const MIN_WITNESSES = 2 + +// Two checks that ran days apart observed two different `main`s, so agreeing +// failures say nothing about either. Beyond this span the answer is `unknown`. +export const MAX_EVIDENCE_SPAN_HOURS = 24 + +// Splits a commit's check runs into the real failures and everything deliberately +// dropped. `excluded` is returned rather than discarded so callers can print it — +// a silently narrowed population reads as full coverage. +export function normalizeChecks(checkRuns, options = {}) { + const includeKnownFalse = options.includeKnownFalse === true + const appSlugs = options.appSlugs ?? DEFAULT_APP_SLUGS + const witnessPatterns = options.witnessingCheckPatterns ?? WITNESSING_CHECK_PATTERNS + const failures = [] + const ran = [] + const excluded = { rollup: [], knownFalse: [], foreignApp: [], nonWitnessing: [] } + let incomplete = 0 + let completedAt = null + + for (const run of checkRuns) { + const name = run.name + const slug = run.app?.slug ?? run.appSlug ?? null + if (slug !== null && !appSlugs.has(slug)) { + if (FAILING_CONCLUSIONS.has(run.conclusion)) { + excluded.foreignApp.push(name) + } + continue + } + if (ROLLUP_CHECKS.has(name)) { + if (FAILING_CONCLUSIONS.has(run.conclusion)) { + excluded.rollup.push(name) + } + continue + } + if (run.status !== 'completed') { + incomplete += 1 + continue + } + if (NON_RUN_CONCLUSIONS.has(run.conclusion)) { + continue + } + if (!includeKnownFalse && KNOWN_FALSE_REDS.has(name)) { + if (FAILING_CONCLUSIONS.has(run.conclusion)) { + excluded.knownFalse.push(name) + } + continue + } + ran.push(name) + if ( + run.completedAt !== null && + run.completedAt !== undefined && + run.completedAt > (completedAt ?? '') + ) { + completedAt = run.completedAt + } + if (FAILING_CONCLUSIONS.has(run.conclusion)) { + failures.push(name) + } + } + + // Dedupe: a re-run posts the same check name twice, and a repeated name would + // make two identical failure sets compare as different multisets. + const uniqueRan = [...new Set(ran)].sort() + const witnessing = uniqueRan.filter((name) => isWitnessingCheck(name, witnessPatterns)) + excluded.nonWitnessing = uniqueRan.filter((name) => !isWitnessingCheck(name, witnessPatterns)) + return { + failures: [...new Set(failures)].sort(), + ran: uniqueRan, + // The subset of `ran` that would have gone red had `main` been broken. + witnessing, + excluded, + incomplete, + completedAt, + // Only a PR on which some lane actually executed the tree witnessed anything. + // A path-filtered PR still posts its path classifier, root-directory guard and + // LoC counter green; counting those as a witness is how `clean` gets returned + // for a window nothing was tested in. + usable: incomplete === 0 && witnessing.length > 0 + } +} + +// Groups PRs into stacks by branch parentage: a PR whose base branch is another +// PR's head branch is stacked on it. Two refs in the same stack share a diff, so +// they are one witness, not two. +export function buildStacks(prs, options = {}) { + const trunkRefs = options.trunkRefs ?? TRUNK_REFS + const headToNumber = new Map() + for (const pr of prs) { + headToNumber.set(pr.headRefName, pr.number) + } + const parent = new Map(prs.map((pr) => [pr.number, pr.number])) + const find = (n) => { + let root = n + while (parent.get(root) !== root) { + root = parent.get(root) + } + return root + } + const union = (a, b) => { + const left = find(a) + const right = find(b) + if (left !== right) { + parent.set(left, right) + } + } + // Siblings on one non-trunk base share that base's diff whether or not the base + // itself is in this list. Grouping only by listed parents counts two children of + // an unlisted parent as two independent stacks, so the corroboration the + // `broken` and `upstream` verdicts rest on is one witness counted twice. + const firstOnBase = new Map() + for (const pr of prs) { + const baseRef = pr.baseRefName + if (baseRef === null || baseRef === undefined || trunkRefs.has(baseRef)) { + continue + } + if (baseRef === pr.headRefName) { + continue + } + const listedParent = headToNumber.get(baseRef) + if (listedParent !== undefined && listedParent !== pr.number) { + union(pr.number, listedParent) + } + const sibling = firstOnBase.get(baseRef) + if (sibling === undefined) { + firstOnBase.set(baseRef, pr.number) + } else { + union(pr.number, sibling) + } + } + const groups = new Map() + for (const pr of prs) { + const root = find(pr.number) + if (!groups.has(root)) { + groups.set(root, []) + } + groups.get(root).push(pr.number) + } + return [...groups.values()].map((members) => members.sort((a, b) => a - b)) +} + +// Wall-clock spread between the earliest and latest witness. Returns null when +// fewer than two witnesses carry a timestamp, so callers cannot read a missing +// span as a tight one. +export function evidenceSpanHours(witnesses) { + const stamps = witnesses + .map((w) => w.completedAt) + .filter((s) => s !== null && s !== undefined) + .map((s) => new Date(s).getTime()) + .filter((t) => Number.isFinite(t)) + if (stamps.length < 2) { + return null + } + return (Math.max(...stamps) - Math.min(...stamps)) / 3600000 +} + +// How a single check behaved across the witnesses. +export const CHECK_KIND = { + // Red in every witness that ran it: main was broken for the whole window. + alwaysRed: 'always-red', + // Every red falls inside one unbroken stretch of time with no green in it: + // main broke at the start of that stretch. Covers a break that is still open + // (no greens after), one that was fixed (greens after), and one that both + // opened and closed inside the window. + windowed: 'windowed', + // Greens fall inside the red stretch: the branches differ, not main. + interleaved: 'interleaved', + // Too few witnesses or stacks to attribute either way. + tooNarrow: 'too-narrow' +} + +function timeOf(w) { + const t = + w.completedAt === null || w.completedAt === undefined + ? Number.NaN + : new Date(w.completedAt).getTime() + return Number.isFinite(t) ? t : null +} + +// Classifies one check name by whether its reds and greens separate cleanly in +// time. A clean split is a main-side transition; interleaving is branch-specific. +function classifyOneCheck(name, usable, minWitnesses) { + const red = usable.filter((w) => w.failures.includes(name)) + const green = usable.filter((w) => !w.failures.includes(name) && w.ran.includes(name)) + const redStacks = new Set(red.map((w) => w.stackId ?? w.ref)) + const redTimes = red.map(timeOf).filter((t) => t !== null) + const greenTimes = green.map(timeOf).filter((t) => t !== null) + const base = { + name, + redRefs: red.map((w) => w.ref), + greenRefs: green.map((w) => w.ref), + redStacks: redStacks.size, + firstRed: redTimes.length > 0 ? Math.min(...redTimes) : null, + lastRed: redTimes.length > 0 ? Math.max(...redTimes) : null, + firstGreen: greenTimes.length > 0 ? Math.min(...greenTimes) : null, + lastGreen: greenTimes.length > 0 ? Math.max(...greenTimes) : null + } + // A green inside the red stretch means the tree was fine there, so the reds + // around it belong to their own branches. This is checked before the witness + // thresholds: one red against many greens is branch-specific evidence, not + // thin evidence, and calling it `too-narrow` would make `clean` unreachable. + const greenInsideRedStretch = + base.firstRed !== null && greenTimes.some((t) => t >= base.firstRed && t <= base.lastRed) + if (greenInsideRedStretch) { + return { ...base, kind: CHECK_KIND.interleaved } + } + if (base.firstRed === null) { + return { ...base, kind: CHECK_KIND.tooNarrow, reason: 'no red carries a timestamp' } + } + // A red confined to one stack with greens on both sides of it: main would have + // had to break and be fixed between two adjacent witnesses, which no other + // branch could have inherited. That makes it the branch's own failure. + const bracketedByGreens = + greenTimes.some((t) => t < base.firstRed) && greenTimes.some((t) => t > base.lastRed) + if (redStacks.size < minWitnesses && bracketedByGreens) { + return { ...base, kind: CHECK_KIND.interleaved } + } + if (red.length < minWitnesses || redStacks.size < minWitnesses) { + return { + ...base, + kind: CHECK_KIND.tooNarrow, + reason: + redStacks.size < minWitnesses + ? `red in only ${redStacks.size} independent stack(s)` + : `red in only ${red.length} witness(es)` + } + } + if (green.length === 0) { + return { ...base, kind: CHECK_KIND.alwaysRed } + } + return { + ...base, + kind: CHECK_KIND.windowed, + lastGreenBefore: Math.max( + ...greenTimes.filter((t) => t < base.firstRed), + Number.NEGATIVE_INFINITY + ), + firstGreenAfter: Math.min( + ...greenTimes.filter((t) => t > base.lastRed), + Number.POSITIVE_INFINITY + ) + } +} + +// True when `main` is known broken for this check at time `at` (ms). The blind +// gaps on either side of the red stretch answer false, not true. +export function brokenAt(check, at) { + if (check.kind === CHECK_KIND.alwaysRed) { + return true + } + if (check.kind === CHECK_KIND.windowed) { + return at >= check.firstRed && at <= check.lastRed + } + return false +} + +// Per-check attribution across witnesses. +export function classifyChecks(witnesses, options = {}) { + const minWitnesses = options.minWitnesses ?? MIN_WITNESSES + const usable = witnesses.filter((w) => w.usable) + const names = new Set() + for (const w of usable) { + for (const name of w.failures) { + names.add(name) + } + } + + const checks = [...names].sort().map((name) => classifyOneCheck(name, usable, minWitnesses)) + const upstream = checks.filter((c) => c.kind === CHECK_KIND.alwaysRed) + const transitions = checks.filter((c) => c.kind === CHECK_KIND.windowed) + const branchSpecific = checks.filter((c) => c.kind === CHECK_KIND.interleaved) + const inconclusive = checks.filter((c) => c.kind === CHECK_KIND.tooNarrow) + + const failing = usable.filter((w) => w.failures.length > 0) + const signatures = new Set(failing.map((w) => w.failures.join(' '))) + const failingStacks = new Set(failing.map((w) => w.stackId ?? w.ref)) + + return { + checks, + upstream, + transitions, + branchSpecific, + inconclusive, + evidenceSpanHours: evidenceSpanHours(usable), + identicalAcrossFailing: failing.length >= minWitnesses && signatures.size === 1, + failingWitnesses: failing.length, + failingStacks: failingStacks.size, + usableWitnesses: usable.length, + totalWitnesses: witnesses.length, + independentStacks: new Set(usable.map((w) => w.stackId ?? w.ref)).size + } +} + +// Rejects evidence whose checks are too far apart in time to have observed the +// same `main`. Returns null when the span is acceptable. +function staleEvidenceVerdict(classification, options) { + const maxSpan = options.maxSpanHours ?? MAX_EVIDENCE_SPAN_HOURS + const span = classification.evidenceSpanHours + if (span === null || span === undefined) { + return { verdict: VERDICT.unknown, why: 'fewer than two witnesses carry a check timestamp' } + } + if (span > maxSpan) { + return { + verdict: VERDICT.unknown, + why: `witness checks span ${span.toFixed(1)}h (max ${maxSpan}h); they ran against different mains, so agreeing failures prove nothing` + } + } + return null +} + +// `compare` verdict: are these branches red for the same reason or different ones? +export function compareVerdict(classification, options = {}) { + const minWitnesses = options.minWitnesses ?? MIN_WITNESSES + if (classification.usableWitnesses < minWitnesses) { + return { + verdict: VERDICT.unknown, + why: `only ${classification.usableWitnesses} of ${classification.totalWitnesses} refs ran a lane that exercises the tree (need ${minWitnesses})` + } + } + if (classification.failingWitnesses === 0) { + return { verdict: VERDICT.noFailures, why: 'no real failures on any ref' } + } + const stale = staleEvidenceVerdict(classification, options) + if (stale !== null) { + return stale + } + if (classification.failingWitnesses < minWitnesses) { + return { + verdict: VERDICT.unknown, + why: `only ${classification.failingWitnesses} ref is failing; a single red ref cannot be told apart from upstream breakage` + } + } + if (!classification.identicalAcrossFailing) { + return { + verdict: VERDICT.divergent, + why: 'failing refs do not share one failure set, so at least part of the damage is branch-specific' + } + } + if (classification.failingStacks < minWitnesses) { + return { + verdict: VERDICT.sharedAncestor, + why: 'identical failures, but every failing ref is in one stack: the cause is at or below that stack root (main, or the root PR itself)' + } + } + return { + verdict: VERDICT.upstream, + why: 'identical failure set across independent stacks; damage from a merge varies with what each branch changed, so this is upstream' + } +} + +// A commit sitting between the last green and the first red of a break — or +// between the last red and the green that follows — is in the blind gap: no +// witness observed main there, so the answer is unknown, not clean. +function inBlindGap(check, at) { + if (check.kind !== CHECK_KIND.windowed) { + return false + } + const beforeGap = at > check.lastGreenBefore && at < check.firstRed + const afterGap = at > check.lastRed && at < check.firstGreenAfter + return beforeGap || afterGap +} + +// `at` verdict: was main broken at this commit? Never returns `clean` without +// positive evidence from enough independent witnesses. +export function mainHealthVerdict(classification, at, options = {}) { + const minWitnesses = options.minWitnesses ?? MIN_WITNESSES + if (classification.usableWitnesses < minWitnesses) { + return { + verdict: VERDICT.unknown, + why: `only ${classification.usableWitnesses} of ${classification.totalWitnesses} witness PRs ran a lane that exercises the tree (need ${minWitnesses})`, + brokenChecks: [] + } + } + if (classification.independentStacks < minWitnesses) { + return { + verdict: VERDICT.unknown, + why: `witnesses span only ${classification.independentStacks} independent stack(s); they share a diff and cannot corroborate each other`, + brokenChecks: [] + } + } + const stale = staleEvidenceVerdict(classification, options) + if (stale !== null) { + return { ...stale, brokenChecks: [] } + } + const broken = classification.checks.filter((c) => brokenAt(c, at)) + if (broken.length > 0) { + return { + verdict: VERDICT.broken, + why: `${broken.length} check(s) were failing on main at this commit: ${broken.map((c) => c.name).join(', ')}`, + brokenChecks: broken.map((c) => c.name) + } + } + const gaps = classification.checks.filter((c) => inBlindGap(c, at)) + if (gaps.length > 0) { + return { + verdict: VERDICT.unknown, + why: `${gaps.length} check(s) changed state inside this window with no witness at the commit itself: ${gaps.map((c) => c.name).join(', ')}`, + brokenChecks: [] + } + } + if (classification.inconclusive.length > 0) { + return { + verdict: VERDICT.unknown, + why: `${classification.inconclusive.length} failing check(s) were seen too narrowly to attribute; treat main as unproven`, + brokenChecks: [] + } + } + return { + verdict: VERDICT.clean, + why: `${classification.usableWitnesses} witnesses across ${classification.independentStacks} independent stacks; every check that failed here failed only on its own branch`, + brokenChecks: [] + } +} + +// Keeps the PRs whose CI actually completed inside the window. A PR merged in +// the window may have last run CI days earlier, against a different `main`. +export function selectWitnessesInWindow(witnesses, from, to) { + return witnesses.filter((w) => { + if (w.completedAt === null || w.completedAt === undefined) { + return false + } + const at = new Date(w.completedAt).getTime() + return at >= from.getTime() && at <= to.getTime() + }) +} diff --git a/config/scripts/upstream-breakage-evidence.test.mjs b/config/scripts/upstream-breakage-evidence.test.mjs new file mode 100644 index 00000000000..21fc692d985 --- /dev/null +++ b/config/scripts/upstream-breakage-evidence.test.mjs @@ -0,0 +1,586 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' + +import { + CHECK_KIND, + MIN_WITNESSES, + VERDICT, + brokenAt, + buildStacks, + classifyChecks, + compareVerdict, + evidenceSpanHours, + isWitnessingCheck, + mainHealthVerdict, + normalizeChecks, + selectWitnessesInWindow +} from './upstream-breakage-evidence.mjs' + +// Verbatim GitHub check-run payloads, not hand-written shapes: the bug these pin +// is that real path-filtered PRs post three green meta jobs and nothing else. +const FIXTURE = JSON.parse( + readFileSync(new URL('./upstream-breakage-check-runs.fixture.json', import.meta.url), 'utf8') +) +const DOCS_ONLY = FIXTURE.prs['17002'] +const MOBILE_ONLY = FIXTURE.prs['16917'] +const FULL_CI = FIXTURE.prs['17358'] + +const HOUR = 3600 * 1000 +const T0 = Date.parse('2026-08-30T03:00:00Z') + +function check(name, conclusion, extra = {}) { + return { + name, + status: 'completed', + conclusion, + appSlug: 'github-actions', + completedAt: new Date(T0).toISOString(), + ...extra + } +} + +// Witnesses built from the captured payloads, stacked the way the probe stacks +// them, so the verdict path under test is the one the CLI runs. +function realWitnesses(entries) { + const stacks = buildStacks(entries.map((entry) => entry.pr)) + const stackOf = new Map() + for (const [index, members] of stacks.entries()) { + for (const number of members) { + stackOf.set(number, `stack-${index + 1}`) + } + } + return entries.map((entry) => ({ + ref: `#${entry.pr.number}`, + stackId: stackOf.get(entry.pr.number), + ...normalizeChecks(entry.runs) + })) +} + +// A usable witness with the given failures, all other named checks green. +function witness(ref, stackId, offsetMinutes, failures, ran = []) { + const names = [...new Set([...failures, ...ran])] + return { + ref, + stackId, + usable: true, + incomplete: 0, + completedAt: new Date(T0 + offsetMinutes * 60 * 1000).toISOString(), + failures: [...failures].sort(), + ran: names.sort(), + excluded: { rollup: [], knownFalse: [], foreignApp: [] } + } +} + +describe('normalizeChecks', () => { + it('keeps a real failure and the checks that ran green', () => { + const result = normalizeChecks([ + check('typecheck', 'failure'), + check('static analysis', 'success') + ]) + expect(result.failures).toEqual(['typecheck']) + expect(result.ran).toEqual(['static analysis', 'typecheck']) + expect(result.usable).toBe(true) + }) + + it('never counts the verify roll-up as an independent failure', () => { + const result = normalizeChecks([check('verify', 'failure'), check('typecheck', 'failure')]) + expect(result.failures).toEqual(['typecheck']) + expect(result.excluded.rollup).toEqual(['verify']) + }) + + it('does not report a green roll-up as an excluded red', () => { + expect( + normalizeChecks([check('verify', 'success'), check('typecheck', 'success')]).excluded.rollup + ).toEqual([]) + }) + + it('drops the known-false reds but reports what it dropped', () => { + const result = normalizeChecks([ + check('test / tests node 24 1/8', 'failure'), + check('test / tests node 24 6/8', 'failure'), + check('e2e / ssh docker watcher isolation', 'failure'), + check('test / tests node 24 2/8', 'failure') + ]) + expect(result.failures).toEqual(['test / tests node 24 2/8']) + expect(result.excluded.knownFalse).toHaveLength(3) + }) + + it('counts the known-false reds when explicitly asked to', () => { + const result = normalizeChecks([check('test / tests node 24 1/8', 'failure')], { + includeKnownFalse: true + }) + expect(result.failures).toEqual(['test / tests node 24 1/8']) + }) + + it('ignores third-party app checks', () => { + const result = normalizeChecks([ + check('Greptile Review', 'failure', { appSlug: 'greptile-apps' }), + check('typecheck', 'success') + ]) + expect(result.failures).toEqual([]) + expect(result.excluded.foreignApp).toEqual(['Greptile Review']) + }) + + it('treats a skipped check as not run rather than as a pass', () => { + const result = normalizeChecks([check('package', 'skipped')]) + expect(result.ran).toEqual([]) + expect(result.usable).toBe(false) + }) + + it('is unusable while a check is still in progress', () => { + const result = normalizeChecks([ + check('typecheck', null, { status: 'in_progress' }), + check('static analysis', 'success') + ]) + expect(result.usable).toBe(false) + expect(result.incomplete).toBe(1) + }) + + it('counts a timed-out check as a failure', () => { + expect(normalizeChecks([check('package', 'timed_out')]).failures).toEqual(['package']) + }) + + it('dedupes a check name a re-run posted twice', () => { + const result = normalizeChecks([check('typecheck', 'failure'), check('typecheck', 'failure')]) + expect(result.failures).toEqual(['typecheck']) + }) + + // The bug: `#17002` is a docs-only PR and `#16917` a mobile-only one. Both + // path-filter every lane in pr.yml away, and both still post three green + // meta jobs — the path classifier, the root-directory guard and the LoC + // counter. Counting those as a witness is how the probe reported `clean` for a + // window in which no test, typecheck or static-analysis lane had run at all. + it('is not a witness when only the always-on meta jobs ran (docs-only PR #17002)', () => { + const result = normalizeChecks(DOCS_ONLY.runs) + expect(result.incomplete).toBe(0) + expect(result.ran).toEqual([ + 'detect code-relevant changes', + 'root directory guard', + 'test vs non-test LoC' + ]) + expect(result.witnessing).toEqual([]) + expect(result.usable).toBe(false) + }) + + it('is not a witness when only the always-on meta jobs ran (mobile-only PR #16917)', () => { + const result = normalizeChecks(MOBILE_ONLY.runs) + expect(result.witnessing).toEqual([]) + expect(result.usable).toBe(false) + }) + + it('names the checks it refused to treat as witnesses rather than dropping them', () => { + expect(normalizeChecks(DOCS_ONLY.runs).excluded.nonWitnessing).toEqual([ + 'detect code-relevant changes', + 'root directory guard', + 'test vs non-test LoC' + ]) + }) + + // Positive control for the two above: the same gate on a PR whose lanes really + // ran leaves it usable, so `usable: false` is the meta jobs, not the gate. + it('is a witness when the real lanes ran (full-CI PR #17358)', () => { + const result = normalizeChecks(FULL_CI.runs) + expect(result.usable).toBe(true) + expect(result.witnessing).toContain('typecheck') + expect(result.witnessing).toContain('static analysis') + expect(result.witnessing).toContain('test / tests node 24 2/8') + }) + + it('reports the latest completion time it saw', () => { + const later = new Date(T0 + HOUR).toISOString() + const result = normalizeChecks([ + check('typecheck', 'success'), + check('static analysis', 'success', { completedAt: later }) + ]) + expect(result.completedAt).toBe(later) + }) +}) + +describe('isWitnessingCheck', () => { + it('counts the lanes that build, type-check or run the suite', () => { + for (const name of [ + 'typecheck', + 'static analysis', + 'test', + 'test / tests node 24 2/8', + 'package', + 'package (windows)', + 'e2e', + 'e2e / e2e 3-of-14', + 'shell contracts', + 'Git compatibility', + 'cross-version wire compatibility', + 'managed hooks on Node 18' + ]) { + expect(isWitnessingCheck(name)).toBe(true) + } + }) + + it('does not count a job that stays green on a broken main', () => { + for (const name of [ + 'detect code-relevant changes', + 'root directory guard', + 'test vs non-test LoC', + 'track-community-pr', + 'detect changed e2e specs', + 'e2e / changed e2e specs' + ]) { + expect(isWitnessingCheck(name)).toBe(false) + } + }) + + // An unrecognised name costs a witness (verdict degrades to `unknown`) instead + // of buying one, which is the whole reason this is an allowlist. + it('does not count a name it has never seen', () => { + expect(isWitnessingCheck('some future bot check')).toBe(false) + }) +}) + +describe('buildStacks', () => { + it('puts unrelated PRs based on main in their own stacks', () => { + const stacks = buildStacks([ + { number: 1, headRefName: 'a', baseRefName: 'main' }, + { number: 2, headRefName: 'b', baseRefName: 'main' } + ]) + expect(stacks).toHaveLength(2) + }) + + it('groups a chain of stacked PRs into one stack', () => { + const stacks = buildStacks([ + { number: 1, headRefName: 'a', baseRefName: 'main' }, + { number: 2, headRefName: 'b', baseRefName: 'a' }, + { number: 3, headRefName: 'c', baseRefName: 'b' } + ]) + expect(stacks).toEqual([[1, 2, 3]]) + }) + + it('groups a chain listed leaf-first', () => { + const stacks = buildStacks([ + { number: 3, headRefName: 'c', baseRefName: 'b' }, + { number: 2, headRefName: 'b', baseRefName: 'a' }, + { number: 1, headRefName: 'a', baseRefName: 'main' } + ]) + expect(stacks).toEqual([[1, 2, 3]]) + }) + + // The bug: grouping only by *listed* parents counted these as two independent + // stacks, so the corroboration `broken` and `upstream` rest on was one witness + // counted twice. Both are real open PRs on a base branch that has no PR here. + it('counts two PRs on one unlisted parent branch as a single stack', () => { + const siblings = FIXTURE.siblings.filter( + (pr) => pr.baseRefName === 'brennanb2025/claude-structured-mobile' + ) + expect(siblings.map((pr) => pr.number).sort((a, b) => a - b)).toEqual([15356, 15657]) + expect(siblings.some((pr) => pr.headRefName === siblings[0].baseRefName)).toBe(false) + expect(buildStacks(siblings)).toEqual([[15356, 15657]]) + }) + + it('counts three PRs on one unlisted parent branch as a single stack', () => { + const siblings = FIXTURE.siblings.filter( + (pr) => pr.baseRefName === 'brennanb2025/probe-liveness-coercion-r1' + ) + expect(buildStacks(siblings)).toEqual([[17044, 17058, 17074]]) + }) + + it('still separates two PRs on different unlisted parent branches', () => { + const stacks = buildStacks([ + { number: 1, headRefName: 'a', baseRefName: 'parent-one' }, + { number: 2, headRefName: 'b', baseRefName: 'parent-two' } + ]) + expect(stacks).toEqual([[1], [2]]) + }) + + it('keeps two separate chains separate', () => { + const stacks = buildStacks([ + { number: 1, headRefName: 'a', baseRefName: 'main' }, + { number: 2, headRefName: 'b', baseRefName: 'a' }, + { number: 10, headRefName: 'x', baseRefName: 'main' }, + { number: 11, headRefName: 'y', baseRefName: 'x' } + ]) + expect(stacks).toEqual([ + [1, 2], + [10, 11] + ]) + }) +}) + +describe('evidenceSpanHours', () => { + it('returns null when fewer than two witnesses carry a timestamp', () => { + expect(evidenceSpanHours([witness('#1', 's1', 0, [])])).toBeNull() + expect(evidenceSpanHours([{ ref: '#1' }, { ref: '#2' }])).toBeNull() + }) + + it('measures the spread between the earliest and latest witness', () => { + const span = evidenceSpanHours([witness('#1', 's1', 0, []), witness('#2', 's2', 120, [])]) + expect(span).toBeCloseTo(2) + }) +}) + +describe('classifyChecks', () => { + it('calls a check red in every witness that ran it always-red', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, ['typecheck']) + ]) + expect(result.upstream.map((c) => c.name)).toEqual(['typecheck']) + }) + + it('finds a break that opens partway through the window', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, ['typecheck']), + witness('#3', 's3', 20, ['typecheck']) + ]) + expect(result.transitions).toHaveLength(1) + expect(result.transitions[0].kind).toBe(CHECK_KIND.windowed) + expect(result.transitions[0].lastGreenBefore).toBe(T0) + }) + + it('finds a break that opened and closed inside the window', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, ['typecheck']), + witness('#3', 's3', 20, ['typecheck']), + witness('#4', 's4', 30, [], ['typecheck']) + ]) + expect(result.transitions).toHaveLength(1) + expect(result.transitions[0].firstGreenAfter).toBe(T0 + 30 * 60 * 1000) + }) + + it('calls interleaved reds and greens branch-specific', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, [], ['typecheck']), + witness('#3', 's3', 20, ['typecheck']) + ]) + expect(result.branchSpecific.map((c) => c.name)).toEqual(['typecheck']) + expect(result.transitions).toEqual([]) + }) + + it('calls a lone red bracketed by greens branch-specific, not thin evidence', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, [], ['root directory guard']), + witness('#2', 's2', 10, ['root directory guard']), + witness('#3', 's3', 20, [], ['root directory guard']) + ]) + expect(result.branchSpecific.map((c) => c.name)).toEqual(['root directory guard']) + expect(result.inconclusive).toEqual([]) + }) + + it('refuses to attribute a red confined to one stack', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's1', 10, ['typecheck']) + ]) + expect(result.upstream).toEqual([]) + expect(result.inconclusive.map((c) => c.name)).toEqual(['typecheck']) + }) + + it('ignores witnesses whose checks never completed', () => { + const stalled = { ...witness('#3', 's3', 5, ['typecheck']), usable: false } + const result = classifyChecks([witness('#1', 's1', 0, ['typecheck']), stalled]) + expect(result.usableWitnesses).toBe(1) + }) + + it('reports identical failure sets across the failing witnesses', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, ['typecheck', 'static analysis']), + witness('#2', 's2', 10, ['static analysis', 'typecheck']) + ]) + expect(result.identicalAcrossFailing).toBe(true) + }) + + it('reports divergent failure sets as not identical', () => { + const result = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, ['package']) + ]) + expect(result.identicalAcrossFailing).toBe(false) + }) +}) + +describe('brokenAt', () => { + const windowed = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, ['typecheck']), + witness('#3', 's3', 20, ['typecheck']), + witness('#4', 's4', 30, [], ['typecheck']) + ]).transitions[0] + + it('is broken inside the red stretch', () => { + expect(brokenAt(windowed, T0 + 15 * 60 * 1000)).toBe(true) + }) + + it('is not broken before the stretch opens', () => { + expect(brokenAt(windowed, T0)).toBe(false) + }) + + it('is not broken after the stretch closes', () => { + expect(brokenAt(windowed, T0 + 30 * 60 * 1000)).toBe(false) + }) + + it('is broken everywhere for an always-red check', () => { + const always = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, ['typecheck']) + ]).upstream[0] + expect(brokenAt(always, T0 - HOUR)).toBe(true) + }) + + it('is never broken for an interleaved check', () => { + const interleaved = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, [], ['typecheck']), + witness('#3', 's3', 20, ['typecheck']) + ]).branchSpecific[0] + expect(brokenAt(interleaved, T0 + 5 * 60 * 1000)).toBe(false) + }) +}) + +describe('mainHealthVerdict', () => { + it('reports broken and names the checks', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, ['typecheck']), + witness('#3', 's3', 20, ['typecheck']) + ]) + const verdict = mainHealthVerdict(classification, T0 + 15 * 60 * 1000) + expect(verdict.verdict).toBe(VERDICT.broken) + expect(verdict.brokenChecks).toEqual(['typecheck']) + }) + + it('answers unknown, never clean, with a single witness', () => { + const verdict = mainHealthVerdict(classifyChecks([witness('#1', 's1', 0, [])]), T0) + expect(verdict.verdict).toBe(VERDICT.unknown) + }) + + it('answers unknown when every witness is in one stack', () => { + const classification = classifyChecks([witness('#1', 's1', 0, []), witness('#2', 's1', 10, [])]) + expect(mainHealthVerdict(classification, T0).verdict).toBe(VERDICT.unknown) + }) + + it('answers unknown when the witnesses observed different mains', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, []), + witness('#2', 's2', 60 * 48, []) + ]) + const verdict = mainHealthVerdict(classification, T0) + expect(verdict.verdict).toBe(VERDICT.unknown) + expect(verdict.why).toContain('different mains') + }) + + it('answers unknown in the blind gap between the last green and the first red', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 20, ['typecheck']), + witness('#3', 's3', 30, ['typecheck']) + ]) + const verdict = mainHealthVerdict(classification, T0 + 10 * 60 * 1000) + expect(verdict.verdict).toBe(VERDICT.unknown) + expect(verdict.why).toContain('no witness at the commit itself') + }) + + it('answers unknown when a failing check was seen too narrowly', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['package']), + witness('#2', 's2', 10, []), + witness('#3', 's3', 20, []) + ]) + expect(mainHealthVerdict(classification, T0 + 5 * 60 * 1000).verdict).toBe(VERDICT.unknown) + }) + + it('reports clean only with corroborating independent witnesses', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, [], ['typecheck']) + ]) + expect(mainHealthVerdict(classification, T0 + 5 * 60 * 1000).verdict).toBe(VERDICT.clean) + }) + + // End-to-end on the payload that reproduced this: two real merged PRs, 2.2h + // apart, in two independent stacks, on each of which nothing but the three + // always-on meta jobs ran. Before the witnessing-lane gate this returned + // `clean` — a green verdict for a window in which nothing was tested. + it('answers unknown, not clean, when no witness ran a lane that exercises the tree', () => { + const witnesses = realWitnesses([DOCS_ONLY, MOBILE_ONLY]) + const classification = classifyChecks(witnesses) + expect(classification.independentStacks).toBe(0) + const verdict = mainHealthVerdict(classification, Date.parse(DOCS_ONLY.runs[0].completedAt)) + expect(verdict.verdict).not.toBe(VERDICT.clean) + expect(verdict.verdict).toBe(VERDICT.unknown) + expect(verdict.why).toContain('exercises the tree') + }) + + it('needs at least the documented number of witnesses', () => { + expect(MIN_WITNESSES).toBe(2) + }) +}) + +describe('compareVerdict', () => { + it('calls an identical failure set across independent stacks upstream', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, ['typecheck']) + ]) + expect(compareVerdict(classification).verdict).toBe(VERDICT.upstream) + }) + + it('calls differing failure sets divergent', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 10, ['package']) + ]) + expect(compareVerdict(classification).verdict).toBe(VERDICT.divergent) + }) + + it('will not call one stack upstream on its own', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's1', 10, ['typecheck']) + ]) + expect(compareVerdict(classification).verdict).toBe(VERDICT.sharedAncestor) + }) + + it('refuses identical failures whose checks ran days apart', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['typecheck']), + witness('#2', 's2', 60 * 71, ['typecheck']) + ]) + const verdict = compareVerdict(classification) + expect(verdict.verdict).toBe(VERDICT.unknown) + expect(verdict.why).toContain('different mains') + }) + + it('reports no-failures when nothing is red', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, [], ['typecheck']), + witness('#2', 's2', 10, [], ['typecheck']) + ]) + expect(compareVerdict(classification).verdict).toBe(VERDICT.noFailures) + }) + + it('will not judge a single red ref', () => { + const classification = classifyChecks([ + witness('#1', 's1', 0, ['typecheck'], ['typecheck', 'package']), + witness('#2', 's2', 10, [], ['package']) + ]) + expect(compareVerdict(classification).verdict).toBe(VERDICT.unknown) + }) +}) + +describe('selectWitnessesInWindow', () => { + const from = new Date(T0) + const to = new Date(T0 + HOUR) + + it('keeps a witness whose CI completed inside the window', () => { + expect(selectWitnessesInWindow([witness('#1', 's1', 30, [])], from, to)).toHaveLength(1) + }) + + it('drops a witness whose CI completed before the window', () => { + expect(selectWitnessesInWindow([witness('#1', 's1', -30, [])], from, to)).toEqual([]) + }) + + it('drops a witness with no completion time rather than assuming it fits', () => { + expect(selectWitnessesInWindow([{ ref: '#1', completedAt: null }], from, to)).toEqual([]) + }) +}) diff --git a/config/scripts/upstream-breakage-probe.mjs b/config/scripts/upstream-breakage-probe.mjs new file mode 100644 index 00000000000..00c3d4b3f9c --- /dev/null +++ b/config/scripts/upstream-breakage-probe.mjs @@ -0,0 +1,379 @@ +import { execFile, execFileSync } from 'node:child_process' +import process from 'node:process' +import { pathToFileURL } from 'node:url' + +import { + MAX_EVIDENCE_SPAN_HOURS, + VERDICT, + buildStacks, + classifyChecks, + compareVerdict, + mainHealthVerdict, + normalizeChecks, + selectWitnessesInWindow +} from './upstream-breakage-evidence.mjs' + +// CLI over the upstream-breakage evidence model: talks to gh and git, prints the +// witness table, and reports a verdict. See +// docs/reference/upstream-breakage-diagnosis.md. + +const GH_BUFFER_BYTES = 32 * 1024 * 1024 +// Enough parallelism to keep a 40-witness window inside a few seconds without +// tripping GitHub's secondary rate limits. +const FETCH_CONCURRENCY = 8 + +function gh(args) { + return execFileSync('gh', args, { encoding: 'utf8', maxBuffer: GH_BUFFER_BYTES }) +} + +function ghAsync(args) { + return new Promise((resolve, reject) => { + execFile('gh', args, { encoding: 'utf8', maxBuffer: GH_BUFFER_BYTES }, (error, stdout) => { + if (error) { + reject(error) + return + } + resolve(stdout) + }) + }) +} + +function git(args) { + return execFileSync('git', args, { encoding: 'utf8' }).trim() +} + +const CHECK_RUN_JQ = + '.check_runs[] | {name, status, conclusion, completedAt: .completed_at, appSlug: .app.slug}' + +async function fetchCheckRuns(sha, { allAttempts }) { + const filter = allAttempts === true ? 'all' : 'latest' + const raw = await ghAsync([ + 'api', + `repos/{owner}/{repo}/commits/${sha}/check-runs?per_page=100&filter=${filter}`, + '--paginate', + '--jq', + CHECK_RUN_JQ + ]) + return raw + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => JSON.parse(line)) +} + +// Bounded-concurrency map; keeps the probe in the seconds range for wide windows. +async function mapWithConcurrency(items, worker) { + const results = Array.from({ length: items.length }) + let next = 0 + const runners = Array.from({ length: Math.min(FETCH_CONCURRENCY, items.length) }, async () => { + while (next < items.length) { + const index = next + next += 1 + results[index] = await worker(items[index]) + } + }) + await Promise.all(runners) + return results +} + +const PR_FIELDS = 'number,headRefName,headRefOid,baseRefName,mergedAt,title,url' + +// Always passes an explicit limit and reports the count, because `gh pr list` +// silently truncates at 30 and a truncated population reads as a complete one. +function listPullRequests(extraArgs, limit) { + const prs = JSON.parse( + gh(['pr', 'list', '--limit', String(limit), '--json', PR_FIELDS, ...extraArgs]) + ) + return { prs, truncated: prs.length >= limit, limit } +} + +function viewPullRequest(number) { + return JSON.parse(gh(['pr', 'view', String(number), '--json', PR_FIELDS])) +} + +function resolveCommit(rev) { + // Never derive a SHA from an abbreviation by hand; let git verify it. + return git(['rev-parse', '--verify', `${rev}^{commit}`]) +} + +async function witnessesFromPrs(prs, options) { + return mapWithConcurrency(prs, async (pr) => ({ + ref: `#${pr.number}`, + pr, + ...normalizeChecks(await fetchCheckRuns(pr.headRefOid, options), options) + })) +} + +function attachStacks(witnesses) { + const stacks = buildStacks(witnesses.map((w) => w.pr)) + const stackOf = new Map() + for (const [index, members] of stacks.entries()) { + for (const number of members) { + stackOf.set(number, `stack-${index + 1}`) + } + } + return witnesses.map((w) => ({ ...w, stackId: stackOf.get(w.pr.number) ?? w.ref })) +} + +function describeWitness(w) { + if (!w.usable) { + if (w.incomplete > 0) { + return `no verdict (${w.incomplete} check(s) not completed)` + } + return w.ran.length === 0 + ? 'no verdict (nothing ran; path filters skipped every job)' + : `no verdict (no lane exercised the tree; only ${w.ran.join(', ')} ran)` + } + return w.failures.length === 0 + ? `green (${w.ran.length} checks ran)` + : `RED: ${w.failures.join(', ')}` +} + +function printWitnesses(witnesses) { + // Chronological: a main-side break shows up as greens above and reds below. + const ordered = [...witnesses].sort((a, b) => + String(a.completedAt).localeCompare(String(b.completedAt)) + ) + for (const w of ordered) { + const at = + w.completedAt === null || w.completedAt === undefined + ? '?' + : clock(new Date(w.completedAt).getTime()) + console.log(` ${at} ${w.ref} [${w.stackId}] ${describeWitness(w)}`) + const dropped = [ + ...new Set([ + ...w.excluded.rollup.map((n) => `${n} (roll-up)`), + ...w.excluded.knownFalse.map((n) => `${n} (known-false red)`), + ...w.excluded.foreignApp.map((n) => `${n} (third-party app)`), + // Printed so a renamed lane that dropped out of the witnessing list is + // visible rather than quietly costing the run its evidence. + ...(w.excluded.nonWitnessing ?? []).map((n) => `${n} (does not exercise the tree)`) + ]) + ] + if (dropped.length > 0) { + console.log(` excluded: ${dropped.join('; ')}`) + } + } +} + +function clock(ms) { + return ms === null ? '?' : `${new Date(ms).toISOString().replace('T', ' ').slice(0, 16)}Z` +} + +function printClassification(c) { + if (c.upstream.length > 0) { + console.log('\nRed in every witness that ran it — main was broken for this whole window:') + for (const u of c.upstream) { + console.log(` ${u.name} — red in ${u.redRefs.join(', ')}`) + } + } + if (c.transitions.length > 0) { + console.log('\nRed for one unbroken stretch — main was broken across that stretch:') + for (const t of c.transitions) { + const opened = Number.isFinite(t.lastGreenBefore) + ? `after ${clock(t.lastGreenBefore)}` + : 'before the window' + const closed = Number.isFinite(t.firstGreenAfter) + ? `by ${clock(t.firstGreenAfter)}` + : 'still open' + console.log( + ` ${t.name} — red ${clock(t.firstRed)}..${clock(t.lastRed)} (broke ${opened}; fixed ${closed})` + ) + console.log(` red: ${t.redRefs.join(', ')}`) + } + } + if (c.branchSpecific.length > 0) { + console.log('\nReds and greens interleave in time — branch-specific, not main:') + for (const b of c.branchSpecific) { + console.log( + ` ${b.name} — red in ${b.redRefs.join(', ')}; green in ${b.greenRefs.join(', ')}` + ) + } + } + if (c.inconclusive.length > 0) { + console.log('\nSeen too narrowly to attribute:') + for (const i of c.inconclusive) { + console.log(` ${i.name} — red in ${i.redRefs.join(', ')}; ${i.reason}`) + } + } +} + +export function parseArgs(argv) { + const options = { + mode: argv[0] ?? null, + refs: [], + commit: null, + windowHours: 3, + limit: 100, + maxSpanHours: MAX_EVIDENCE_SPAN_HOURS, + allAttempts: false, + includeKnownFalse: false, + json: false + } + const rest = argv.slice(1) + while (rest.length > 0) { + const arg = rest.shift() + if (arg === '--window-hours') { + options.windowHours = Number(rest.shift()) + } else if (arg === '--limit') { + options.limit = Number(rest.shift()) + } else if (arg === '--max-span-hours') { + options.maxSpanHours = Number(rest.shift()) + } else if (arg === '--all-attempts') { + options.allAttempts = true + } else if (arg === '--include-known-false') { + options.includeKnownFalse = true + } else if (arg === '--json') { + options.json = true + } else if (arg.startsWith('-')) { + throw new Error(`unknown flag: ${arg}`) + } else if (options.mode === 'at' && options.commit === null) { + options.commit = arg + } else { + options.refs.push(arg.replace(/^#/, '')) + } + } + return options +} + +const USAGE = `Usage: + node config/scripts/upstream-breakage-probe.mjs at [--window-hours N] [--limit N] + node config/scripts/upstream-breakage-probe.mjs compare ... + + at Was main broken at/near this commit? Reconstructed from the PRs whose + CI ran against main in the surrounding window. + compare Do these red branches share one failure set (upstream) or not? + +Flags: + --window-hours N witness window for \`at\` (default 3) + --limit N explicit gh pr list limit (default 100) + --max-span-hours N reject evidence spread wider than this (default ${MAX_EVIDENCE_SPAN_HOURS}) + --all-attempts include earlier check attempts a re-run to green hid + --include-known-false count the known-false reds instead of excluding them + --json machine-readable output + +Verdicts never default to pass: without enough evidence the answer is "unknown".` + +function describeSpan(classification) { + return classification.evidenceSpanHours === null + ? 'evidence span: unknown (fewer than two timestamps)' + : `evidence span: ${classification.evidenceSpanHours.toFixed(1)}h` +} + +async function runCompare(options) { + if (options.refs.length === 0) { + throw new Error('compare needs at least one PR number') + } + const selected = options.refs.map(Number).map(viewPullRequest) + const witnesses = attachStacks(await witnessesFromPrs(selected, options)) + const classification = classifyChecks(witnesses) + const verdict = compareVerdict(classification, options) + + if (options.json) { + console.log(JSON.stringify({ mode: 'compare', verdict, classification, witnesses }, null, 2)) + return + } + + console.log( + `Compared ${witnesses.length} ref(s) across ${classification.independentStacks} independent stack(s); ${describeSpan(classification)}.` + ) + printWitnesses(witnesses) + printClassification(classification) + console.log(`\nVERDICT: ${verdict.verdict} — ${verdict.why}`) + if (verdict.verdict === VERDICT.upstream) { + console.log('Next step: merge a newer main into each branch. Do not repair the branches.') + } + if (verdict.verdict === VERDICT.sharedAncestor) { + console.log( + 'Next step: compare against a red PR from a different stack, or probe the stack root with `at`.' + ) + } +} + +async function runAt(options) { + if (options.commit === null) { + throw new Error('at needs a commit') + } + const sha = resolveCommit(options.commit) + const when = new Date(git(['show', '-s', '--format=%cI', sha])) + const windowMs = options.windowHours * 3600 * 1000 + const from = new Date(when.getTime() - windowMs) + const to = new Date(when.getTime() + windowMs) + + const { prs, truncated, limit } = listPullRequests(['--state', 'merged'], options.limit) + // CI completes before the merge, so preselect generously and let the + // completion-time filter below decide. + const candidates = prs.filter((pr) => { + if (pr.mergedAt === null || pr.mergedAt === undefined) { + return false + } + const merged = new Date(pr.mergedAt).getTime() + return merged >= from.getTime() && merged <= to.getTime() + windowMs + }) + + const fetched = await witnessesFromPrs(candidates, options) + const witnesses = attachStacks(selectWitnessesInWindow(fetched, from, to)) + const classification = classifyChecks(witnesses) + const verdict = mainHealthVerdict(classification, when.getTime(), options) + + if (options.json) { + console.log( + JSON.stringify( + { mode: 'at', sha, window: { from, to }, verdict, classification, witnesses }, + null, + 2 + ) + ) + return + } + + console.log(`main @ ${sha} (${when.toISOString()})`) + console.log( + `Scanned ${prs.length} merged PR(s) at gh limit ${limit}; ${candidates.length} merged near the window, ${witnesses.length} with CI inside ±${options.windowHours}h, ${classification.usableWitnesses} usable across ${classification.independentStacks} independent stack(s); ${describeSpan(classification)}.` + ) + if (truncated) { + console.log( + ` WARNING: gh pr list returned exactly ${limit} rows (the limit); the window may be truncated — raise --limit.` + ) + } + printWitnesses(witnesses) + printClassification(classification) + console.log(`\nVERDICT: main was ${verdict.verdict} at this commit — ${verdict.why}`) + if (verdict.verdict === VERDICT.broken) { + console.log( + 'A branch red on these checks is inheriting main. Merge a newer main; do not repair the branch.' + ) + } + if (verdict.verdict === VERDICT.unknown) { + console.log( + 'Unknown is not green. Widen --window-hours, or use `compare` on the red branches directly.' + ) + } +} + +async function main(argv) { + let options + try { + options = parseArgs(argv) + } catch (error) { + console.error(`${error.message}\n\n${USAGE}`) + process.exitCode = 2 + return + } + if (options.mode === 'at') { + await runAt(options) + return + } + if (options.mode === 'compare') { + await runCompare(options) + return + } + console.error(USAGE) + process.exitCode = 2 +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main(process.argv.slice(2)).catch((error) => { + console.error(error.message ?? error) + process.exitCode = 1 + }) +} diff --git a/docs/reference/upstream-breakage-diagnosis.md b/docs/reference/upstream-breakage-diagnosis.md new file mode 100644 index 00000000000..47097c0558f --- /dev/null +++ b/docs/reference/upstream-breakage-diagnosis.md @@ -0,0 +1,150 @@ +# Telling "my merge broke it" apart from "main was already broken" + +Read this before diagnosing a branch that went red after merging `main`, and +before writing a repair brief for a stack that went red together. + +## The mistake this exists to prevent + +An eleven-PR chain merged `main` inside a window during which `main` itself was +broken: a dependency bump had broken a group of test shards, and a lint violation +was already sitting on `main`. Six of the eleven nodes went red. The reds were +diagnosed as damage caused by the merge, written up, and acted on. All of it was +wrong — the fix was to merge a newer `main`. + +The tell was already in the data: **an identical failure multiset on every node, +including the root**. Damage from a merge varies with what each node changed. +Identical failures across an entire chain are upstream by construction. + +## Why you cannot just look up main's CI + +The obvious move is to ask GitHub whether `main` was green at a SHA. In this repo +that question has no answer, and it is worth knowing why before you go looking: + +- `pr.yml` — the workflow that holds the tests, typecheck, static analysis and + packaging — is `on: pull_request` only. +- `unit-tests.yml` is `workflow_call` only; nothing calls it from a push. +- No workflow runs the test suite on a push to `main`. The only lanes that ever + execute on `main`'s own commits are scheduled or manually dispatched (E2E, the + mac builds, terminal perf, node-next), plus a path-filtered skill round-trip. + +The result is that a commit on `main` carries **zero** check runs: + +``` +$ gh api "repos/{owner}/{repo}/commits//check-runs" --jq .total_count +0 +``` + +The one lane that does run on `main` is scheduled E2E, twice a day, and it has +been uniformly red for weeks — so it cannot discriminate either. + +So `main`'s health has to be reconstructed from the PRs whose CI ran against +`main` at that moment. That is what the probe does. + +## What to run + +```sh +# Was main broken at or near this commit? +pnpm run diagnose:upstream-breakage at --window-hours 1 + +# Are these red branches red for the same reason? +pnpm run diagnose:upstream-breakage compare 17330 17331 17332 +``` + +Both finish in a few seconds. Both take `--json`. + +`at` resolves the commit, finds the PRs whose checks completed in the surrounding +window, and reports each check's behaviour over time. `compare` does the same for +a list of PRs you name. + +## How it decides + +Every PR on which **a lane that actually executes the tree** ran in the window is +a **witness**. That qualification is the whole load-bearing part: a path-filtered +PR — docs-only, mobile-only — skips every lane in `pr.yml` and still posts three +green jobs (the path classifier, the root-directory guard and the LoC counter). +Those stay green on a `main` that is entirely broken, so a PR carrying only them +witnessed nothing and the probe refuses to count it. The list of qualifying lanes +is an allowlist in `upstream-breakage-evidence.mjs`: a name it does not recognise +costs a witness and pushes the verdict toward `unknown`, rather than buying one +and pushing it toward `clean`. Every unrecognised name is printed on each run. + +For each check name the probe then looks at when it was red and when it was green: + +| Shape | Meaning | +| --------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| Red in every witness that ran it | `main` was broken for the whole window | +| Red across one unbroken stretch of time, green outside it | `main` broke at the start of that stretch, and was fixed at the end if greens follow | +| Reds and greens interleave in time | branch-specific — the tree was fine between the reds | +| Red in fewer than two independent stacks | not attributable either way | + +Two PRs in the same stack share a diff, so they count as one witness, not two. +The probe reconstructs stacks from branch parentage: a PR whose base branch is +another PR's head branch is stacked on it, **and PRs that share one non-trunk base +branch are one stack even when that branch has no PR of its own** — otherwise two +children of an unlisted parent corroborate each other, which is one witness +counted twice. + +The output for the real incident looks like this — the exact window, named: + +``` +test / tests node 24 2/8 — red 04:09Z..04:23Z (broke after 04:05Z; fixed by 04:39Z) +``` + +## It says unknown, and unknown is not green + +The probe answers `broken`, `clean`, or `unknown`, and it will not say `clean` +without positive evidence. It answers `unknown` when: + +- fewer than two PRs ran a lane that exercises the tree — including the case where + every PR in the window was path-filtered down to its meta jobs; +- the witnesses are all in one stack, so they corroborate nothing; +- the witness checks are spread over more than a day, so they ran against + different `main`s and their agreement means nothing; +- the commit sits in a blind gap — between the last green and the first red — so + no witness observed `main` there; +- a failing check was seen too narrowly to attribute. + +This is the same discipline as the `live` / `unverifiable` / `exited` vocabulary +in [`ssh-execution-boundary.md`](./ssh-execution-boundary.md). A health check +that guesses green is exactly the failure it is meant to catch. + +## What it deliberately does not count + +Printed on every run, never dropped silently: + +- **`verify`** is a roll-up job that reprints another job's failure. Counting it + double-counts one breakage and makes divergent failure sets look identical. +- **Known-false reds** — `test / tests node 24 1/8`, `test / tests node 24 6/8`, + and `e2e / ssh docker watcher isolation` — are red for reasons unrelated to the + commit. Counting them makes every window look broken. `--include-known-false` + overrides this. +- **Third-party app checks** (review bots) say nothing about the tree. +- **Skipped checks** are not passes, and neither are the always-on jobs that never + execute the tree — `detect code-relevant changes`, `root directory guard`, + `test vs non-test LoC`, `track-community-pr`, and the e2e spec-list classifiers. + A PR left with only those is not a witness at all, so a window made up of + path-filtered PRs reads `unknown`, not `clean`. +- **`verify`** is also the job name of the mobile workflow's only lane, so a + mobile-only PR loses that check to the roll-up exclusion as well. It is not a + witness for `main`'s desktop health either way. + +## Traps to keep in mind + +- `gh pr list` and `gh run list` **truncate silently** — the default is 30 rows. + The probe always passes an explicit `--limit`, prints the count it scanned, and + warns when the result came back exactly at the limit. If you query by hand, do + the same, and print the count beside any conclusion. +- A run's `conclusion` **hides failed first attempts**: a run re-run to green + reads `success`. Pass `--all-attempts` when attempt-level truth matters. +- Read `main` and the workflow definitions from `origin/main`, not from a + worktree. A checkout fourteen commits behind gives a confidently wrong answer. +- The probe cannot see which base SHA each PR's CI actually merged against, only + when it ran. Two checks minutes apart can still straddle a merge. That is why + a narrow `--window-hours` sharpens the answer and why blind gaps read `unknown`. + +## When it says upstream + +Merge a newer `main` into each branch. Do not repair the branches, do not write +the failures up as merge damage, and do not rebase the chain — see +[`git-compatibility.md`](./git-compatibility.md) for the Git-side constraints on +restacking. diff --git a/package.json b/package.json index db62048b689..c7c3c6226c1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "check:code-quality:changed": "node config/scripts/check-changed-code-quality.mjs", "check:react-doctor:changed": "node config/scripts/check-react-doctor-changed.mjs", "check:zustand-selector-fanout": "node config/scripts/zustand-selector-fanout-benchmark.mjs --check", + "diagnose:upstream-breakage": "node config/scripts/upstream-breakage-probe.mjs", "doctor": "pnpm dlx react-doctor@0.9.1 . --no-telemetry", "lint:react-doctor": "oxlint --config config/oxlint-react-doctor.json", "lint:react-doctor:changed": "node config/scripts/lint-react-doctor-changed.mjs",