mirror of
https://github.com/stablyai/orca.git
synced 2026-10-02 16:02:15 +00:00
sim: merge PR #17346
This commit is contained in:
Merge Sim
@@ -115,6 +115,7 @@ docs/**
|
||||
!docs/reference/ssh-execution-boundary.md
|
||||
!docs/reference/ssh-host-key-verification.md
|
||||
!docs/reference/ssh-reconnect-source-recovery.md
|
||||
!docs/reference/upstream-breakage-diagnosis.md
|
||||
!docs/reference/windows-setup-shell.md
|
||||
!docs/reference/worktree-scan-fingerprint.md
|
||||
!docs/reference/wsl-command-execution.md
|
||||
|
||||
@@ -39,6 +39,17 @@ Never use vague names like `helpers`, `utils`, `common`, `misc`, or `shared-stuf
|
||||
|
||||
Always use the primary working directory (the worktree) for all file reads and edits. Never follow absolute paths from subagent results that point to the main repo.
|
||||
|
||||
## Red After Merging Main
|
||||
|
||||
Before diagnosing a branch — or a whole stack — that went red after merging `main`, check whether `main` was already broken. No workflow runs the test suite on a push to `main`, so its commits carry no checks and the question has to be answered from the PRs whose CI ran against `main` at that moment:
|
||||
|
||||
```sh
|
||||
pnpm run diagnose:upstream-breakage at <commit> --window-hours 1
|
||||
pnpm run diagnose:upstream-breakage compare <pr> <pr> ...
|
||||
```
|
||||
|
||||
An identical failure set across independent branches is upstream by construction, and the repair is to merge a newer `main` — not to fix the branches. The probe answers `broken` / `clean` / `unknown` and never reports `clean` without positive evidence. See [`docs/reference/upstream-breakage-diagnosis.md`](./docs/reference/upstream-breakage-diagnosis.md).
|
||||
|
||||
## Cross-Platform Support
|
||||
|
||||
Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior behind runtime checks:
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
{
|
||||
"_source": "Captured verbatim from GitHub: gh api repos/{owner}/{repo}/commits/<headRefOid>/check-runs",
|
||||
"prs": {
|
||||
"16917": {
|
||||
"pr": {
|
||||
"baseRefName": "main",
|
||||
"headRefName": "brennanb2025/mob-create-flow-perf",
|
||||
"headRefOid": "29be0cc2e032ef03b93ebda9c03809bf9f488c1f",
|
||||
"mergedAt": "2026-08-28T22:45:54Z",
|
||||
"number": 16917,
|
||||
"title": "fix(mobile): create-worktree sheet dies after picking a source and loses the picked PR"
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "test",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "managed hooks on Node 18",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "e2e",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "detect changed e2e specs",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "Git compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "static analysis",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "package",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "cross-version wire compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "shell contracts",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "package (windows)",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "orcad browser provider",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "prepare test native cache node 24",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "typecheck",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:36Z",
|
||||
"conclusion": "success",
|
||||
"name": "verify",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "xterm patch sync",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:29Z",
|
||||
"conclusion": "success",
|
||||
"name": "root directory guard",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:30Z",
|
||||
"conclusion": "success",
|
||||
"name": "detect code-relevant changes",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:23:12Z",
|
||||
"conclusion": "success",
|
||||
"name": "test vs non-test LoC",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T11:25:42Z",
|
||||
"conclusion": "success",
|
||||
"name": "verify",
|
||||
"status": "completed"
|
||||
}
|
||||
]
|
||||
},
|
||||
"17002": {
|
||||
"pr": {
|
||||
"baseRefName": "main",
|
||||
"headRefName": "nwparker/headless-linux-server-doc-fixes",
|
||||
"headRefOid": "3d4f9bb9158cf5aaa6f61f88afc63fa00f0add4a",
|
||||
"mergedAt": "2026-08-28T09:37:52Z",
|
||||
"number": 17002,
|
||||
"title": "docs(headless-server): fix package list, extraction perms, and ldd command"
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "Git compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "prepare test native cache node 24",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "managed hooks on Node 18",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "xterm patch sync",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "test",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "shell contracts",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "package (windows)",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "e2e",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "package",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "orcad browser provider",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "static analysis",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "detect changed e2e specs",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "typecheck",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "cross-version wire compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:35Z",
|
||||
"conclusion": "success",
|
||||
"name": "verify",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:29Z",
|
||||
"conclusion": "success",
|
||||
"name": "detect code-relevant changes",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:29Z",
|
||||
"conclusion": "success",
|
||||
"name": "root directory guard",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-28T09:13:13Z",
|
||||
"conclusion": "success",
|
||||
"name": "test vs non-test LoC",
|
||||
"status": "completed"
|
||||
}
|
||||
]
|
||||
},
|
||||
"17358": {
|
||||
"pr": {
|
||||
"baseRefName": "main",
|
||||
"headRefName": "nwparker/react-purity-lint-cleanup",
|
||||
"headRefOid": "fb2fde0bcc0290c96b5d5a7005d118dd89de63f5",
|
||||
"mergedAt": "2026-08-30T08:17:29Z",
|
||||
"number": 17358,
|
||||
"title": "Consolidate the renderer now-clock and drop the epoch setState round-trip"
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:17:08Z",
|
||||
"conclusion": "success",
|
||||
"name": "verify",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:08:11Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "e2e",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "prepare test native cache node 24",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "orcad browser provider",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "managed hooks on Node 18",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "cross-version wire compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "xterm patch sync",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "Git compatibility",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "skipped",
|
||||
"name": "shell contracts",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:15:35Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 1/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:12:59Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 2/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:13:04Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 5/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:13:37Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 6/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:11:28Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 4/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:16:59Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 7/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:11:51Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 3/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:11:49Z",
|
||||
"conclusion": "success",
|
||||
"name": "test / tests node 24 8/8",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:13:14Z",
|
||||
"conclusion": "success",
|
||||
"name": "static analysis",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:11:58Z",
|
||||
"conclusion": "success",
|
||||
"name": "package (windows)",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:10:58Z",
|
||||
"conclusion": "success",
|
||||
"name": "package",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:08:11Z",
|
||||
"conclusion": "success",
|
||||
"name": "detect changed e2e specs",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:09:16Z",
|
||||
"conclusion": "success",
|
||||
"name": "typecheck",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:07:45Z",
|
||||
"conclusion": "success",
|
||||
"name": "root directory guard",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:30Z",
|
||||
"conclusion": "success",
|
||||
"name": "detect code-relevant changes",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "greptile-apps",
|
||||
"completedAt": "2026-08-30T08:07:17Z",
|
||||
"conclusion": "success",
|
||||
"name": "Greptile Review",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"appSlug": "github-actions",
|
||||
"completedAt": "2026-08-30T08:06:39Z",
|
||||
"conclusion": "success",
|
||||
"name": "test vs non-test LoC",
|
||||
"status": "completed"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"siblings": [
|
||||
{
|
||||
"baseRefName": "brennanb2025/claude-structured-mobile",
|
||||
"headRefName": "brennanb2025/claude-structured-core-merge",
|
||||
"number": 15356,
|
||||
"title": "Merge codex-structured-core into Claude lane"
|
||||
},
|
||||
{
|
||||
"baseRefName": "brennanb2025/claude-structured-mobile",
|
||||
"headRefName": "brennanb2025/claude-lane-replay-waiter-identity",
|
||||
"number": 15657,
|
||||
"title": "fix(native-chat): adopt only the real user replay as a Claude send's identity"
|
||||
},
|
||||
{
|
||||
"baseRefName": "brennanb2025/probe-liveness-coercion-r1",
|
||||
"headRefName": "brennanb2025/window-close-guard-r1",
|
||||
"number": 17044,
|
||||
"title": "fix(terminal): ask before closing the window when a terminal probe could not answer"
|
||||
},
|
||||
{
|
||||
"baseRefName": "brennanb2025/probe-liveness-coercion-r1",
|
||||
"headRefName": "brennanb2025/pty-absence-verdict-r1",
|
||||
"number": 17058,
|
||||
"title": "fix(pty): tell a watched local exit apart from a lost route"
|
||||
},
|
||||
{
|
||||
"baseRefName": "brennanb2025/probe-liveness-coercion-r1",
|
||||
"headRefName": "brennanb2025/cleanup-post-confirm-liveness-r1",
|
||||
"number": 17074,
|
||||
"title": "fix(cleanup): stop deleting a workspace after confirm when it turns out to be busy"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
// Decides whether a set of red branches is red because `main` was already broken
|
||||
// (upstream) or because the branches themselves broke something.
|
||||
//
|
||||
// Why this exists: `main` carries no checks of its own. `pr.yml` — the workflow
|
||||
// that holds the tests, typecheck and static analysis — is `on: pull_request`
|
||||
// only, and `unit-tests.yml` is `workflow_call` only, so no commit on `main` has
|
||||
// a test result attached to it. Asking "was main green at SHA X?" directly
|
||||
// returns nothing, which is why nobody reaches for it. The answer has to be
|
||||
// reconstructed from the PRs whose CI ran against `main` at that moment.
|
||||
//
|
||||
// See docs/reference/upstream-breakage-diagnosis.md.
|
||||
|
||||
// Verdicts. `unknown` is the default: a probe that cannot see enough evidence
|
||||
// must never report `clean`, because "no evidence of breakage" and "evidence of
|
||||
// no breakage" are the two states this tool exists to keep apart.
|
||||
export const VERDICT = {
|
||||
broken: 'broken',
|
||||
clean: 'clean',
|
||||
unknown: 'unknown',
|
||||
upstream: 'upstream',
|
||||
sharedAncestor: 'shared-ancestor',
|
||||
divergent: 'divergent',
|
||||
noFailures: 'no-failures'
|
||||
}
|
||||
|
||||
// Roll-up jobs reprint another job's failure; counting one as an independent
|
||||
// failure double-counts the same breakage and makes divergent sets look identical.
|
||||
export const ROLLUP_CHECKS = new Set(['verify'])
|
||||
|
||||
// Reds that are red for reasons unrelated to the commit under test. Counting
|
||||
// these as breakage makes every window look broken.
|
||||
export const KNOWN_FALSE_REDS = new Set([
|
||||
'test / tests node 24 1/8',
|
||||
'test / tests node 24 6/8',
|
||||
'e2e / ssh docker watcher isolation'
|
||||
])
|
||||
|
||||
// Third-party review bots post checks on the same commit. They are not signal
|
||||
// about the tree, so only first-party Actions checks count by default.
|
||||
export const DEFAULT_APP_SLUGS = new Set(['github-actions'])
|
||||
|
||||
// Lanes that actually execute the tree, so a green one is evidence `main` built,
|
||||
// type-checked or ran. Everything else on a PR — the path classifier, the
|
||||
// root-directory guard, the LoC counter, the community-PR labeller — stays green
|
||||
// on a `main` that is entirely broken, so a witness carrying only those witnessed
|
||||
// nothing.
|
||||
//
|
||||
// An allowlist, and deliberately so: an unrecognised name here costs a witness
|
||||
// and the verdict degrades to `unknown`, whereas an unrecognised name in a
|
||||
// denylist would count as a full witness and hand back `clean` — the failure this
|
||||
// list exists to close. `normalizeChecks` reports every name it did not
|
||||
// recognise so a renamed lane is loud rather than silently dropped.
|
||||
export const WITNESSING_CHECK_PATTERNS = [
|
||||
// pr.yml: the lanes that build, type-check, lint or run the suite.
|
||||
/^typecheck$/,
|
||||
/^static analysis$/,
|
||||
/^test$/,
|
||||
/^test \/ tests\b/,
|
||||
/^prepare test native cache\b/,
|
||||
/^package$/,
|
||||
/^package \(/,
|
||||
/^managed hooks on Node \d+$/,
|
||||
/^shell contracts$/,
|
||||
/^Git compatibility$/,
|
||||
/^xterm patch sync$/,
|
||||
/^cross-version wire compatibility$/,
|
||||
/^orcad browser provider$/,
|
||||
// e2e.yml: the shards and the app build. `changed e2e specs` only reads the
|
||||
// diff, so it is not one of these.
|
||||
/^e2e$/,
|
||||
/^e2e \d+-of-\d+$/,
|
||||
/^e2e \/ (?!changed e2e specs$)/,
|
||||
/^build e2e app$/,
|
||||
/^prepare Electron native cache$/,
|
||||
/^ssh docker watcher isolation$/,
|
||||
// Platform smoke and IME lanes.
|
||||
/^real IME/,
|
||||
/^native-smoke \(/,
|
||||
/^mac-native-owner-smoke$/
|
||||
]
|
||||
|
||||
// Would this check have gone red if `main` were broken?
|
||||
export function isWitnessingCheck(name, patterns = WITNESSING_CHECK_PATTERNS) {
|
||||
return patterns.some((pattern) => pattern.test(name))
|
||||
}
|
||||
|
||||
// Branches everything else forks from. Two PRs based on trunk are independent;
|
||||
// two based on the same feature branch share that branch's diff.
|
||||
export const TRUNK_REFS = new Set(['main', 'master'])
|
||||
|
||||
const FAILING_CONCLUSIONS = new Set(['failure', 'timed_out', 'action_required'])
|
||||
const NON_RUN_CONCLUSIONS = new Set(['skipped', 'neutral'])
|
||||
|
||||
// Minimum independent witnesses before any positive verdict. One witness cannot
|
||||
// separate "main is broken" from "this branch is broken".
|
||||
export const MIN_WITNESSES = 2
|
||||
|
||||
// Two checks that ran days apart observed two different `main`s, so agreeing
|
||||
// failures say nothing about either. Beyond this span the answer is `unknown`.
|
||||
export const MAX_EVIDENCE_SPAN_HOURS = 24
|
||||
|
||||
// Splits a commit's check runs into the real failures and everything deliberately
|
||||
// dropped. `excluded` is returned rather than discarded so callers can print it —
|
||||
// a silently narrowed population reads as full coverage.
|
||||
export function normalizeChecks(checkRuns, options = {}) {
|
||||
const includeKnownFalse = options.includeKnownFalse === true
|
||||
const appSlugs = options.appSlugs ?? DEFAULT_APP_SLUGS
|
||||
const witnessPatterns = options.witnessingCheckPatterns ?? WITNESSING_CHECK_PATTERNS
|
||||
const failures = []
|
||||
const ran = []
|
||||
const excluded = { rollup: [], knownFalse: [], foreignApp: [], nonWitnessing: [] }
|
||||
let incomplete = 0
|
||||
let completedAt = null
|
||||
|
||||
for (const run of checkRuns) {
|
||||
const name = run.name
|
||||
const slug = run.app?.slug ?? run.appSlug ?? null
|
||||
if (slug !== null && !appSlugs.has(slug)) {
|
||||
if (FAILING_CONCLUSIONS.has(run.conclusion)) {
|
||||
excluded.foreignApp.push(name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (ROLLUP_CHECKS.has(name)) {
|
||||
if (FAILING_CONCLUSIONS.has(run.conclusion)) {
|
||||
excluded.rollup.push(name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (run.status !== 'completed') {
|
||||
incomplete += 1
|
||||
continue
|
||||
}
|
||||
if (NON_RUN_CONCLUSIONS.has(run.conclusion)) {
|
||||
continue
|
||||
}
|
||||
if (!includeKnownFalse && KNOWN_FALSE_REDS.has(name)) {
|
||||
if (FAILING_CONCLUSIONS.has(run.conclusion)) {
|
||||
excluded.knownFalse.push(name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
ran.push(name)
|
||||
if (
|
||||
run.completedAt !== null &&
|
||||
run.completedAt !== undefined &&
|
||||
run.completedAt > (completedAt ?? '')
|
||||
) {
|
||||
completedAt = run.completedAt
|
||||
}
|
||||
if (FAILING_CONCLUSIONS.has(run.conclusion)) {
|
||||
failures.push(name)
|
||||
}
|
||||
}
|
||||
|
||||
// Dedupe: a re-run posts the same check name twice, and a repeated name would
|
||||
// make two identical failure sets compare as different multisets.
|
||||
const uniqueRan = [...new Set(ran)].sort()
|
||||
const witnessing = uniqueRan.filter((name) => isWitnessingCheck(name, witnessPatterns))
|
||||
excluded.nonWitnessing = uniqueRan.filter((name) => !isWitnessingCheck(name, witnessPatterns))
|
||||
return {
|
||||
failures: [...new Set(failures)].sort(),
|
||||
ran: uniqueRan,
|
||||
// The subset of `ran` that would have gone red had `main` been broken.
|
||||
witnessing,
|
||||
excluded,
|
||||
incomplete,
|
||||
completedAt,
|
||||
// Only a PR on which some lane actually executed the tree witnessed anything.
|
||||
// A path-filtered PR still posts its path classifier, root-directory guard and
|
||||
// LoC counter green; counting those as a witness is how `clean` gets returned
|
||||
// for a window nothing was tested in.
|
||||
usable: incomplete === 0 && witnessing.length > 0
|
||||
}
|
||||
}
|
||||
|
||||
// Groups PRs into stacks by branch parentage: a PR whose base branch is another
|
||||
// PR's head branch is stacked on it. Two refs in the same stack share a diff, so
|
||||
// they are one witness, not two.
|
||||
export function buildStacks(prs, options = {}) {
|
||||
const trunkRefs = options.trunkRefs ?? TRUNK_REFS
|
||||
const headToNumber = new Map()
|
||||
for (const pr of prs) {
|
||||
headToNumber.set(pr.headRefName, pr.number)
|
||||
}
|
||||
const parent = new Map(prs.map((pr) => [pr.number, pr.number]))
|
||||
const find = (n) => {
|
||||
let root = n
|
||||
while (parent.get(root) !== root) {
|
||||
root = parent.get(root)
|
||||
}
|
||||
return root
|
||||
}
|
||||
const union = (a, b) => {
|
||||
const left = find(a)
|
||||
const right = find(b)
|
||||
if (left !== right) {
|
||||
parent.set(left, right)
|
||||
}
|
||||
}
|
||||
// Siblings on one non-trunk base share that base's diff whether or not the base
|
||||
// itself is in this list. Grouping only by listed parents counts two children of
|
||||
// an unlisted parent as two independent stacks, so the corroboration the
|
||||
// `broken` and `upstream` verdicts rest on is one witness counted twice.
|
||||
const firstOnBase = new Map()
|
||||
for (const pr of prs) {
|
||||
const baseRef = pr.baseRefName
|
||||
if (baseRef === null || baseRef === undefined || trunkRefs.has(baseRef)) {
|
||||
continue
|
||||
}
|
||||
if (baseRef === pr.headRefName) {
|
||||
continue
|
||||
}
|
||||
const listedParent = headToNumber.get(baseRef)
|
||||
if (listedParent !== undefined && listedParent !== pr.number) {
|
||||
union(pr.number, listedParent)
|
||||
}
|
||||
const sibling = firstOnBase.get(baseRef)
|
||||
if (sibling === undefined) {
|
||||
firstOnBase.set(baseRef, pr.number)
|
||||
} else {
|
||||
union(pr.number, sibling)
|
||||
}
|
||||
}
|
||||
const groups = new Map()
|
||||
for (const pr of prs) {
|
||||
const root = find(pr.number)
|
||||
if (!groups.has(root)) {
|
||||
groups.set(root, [])
|
||||
}
|
||||
groups.get(root).push(pr.number)
|
||||
}
|
||||
return [...groups.values()].map((members) => members.sort((a, b) => a - b))
|
||||
}
|
||||
|
||||
// Wall-clock spread between the earliest and latest witness. Returns null when
|
||||
// fewer than two witnesses carry a timestamp, so callers cannot read a missing
|
||||
// span as a tight one.
|
||||
export function evidenceSpanHours(witnesses) {
|
||||
const stamps = witnesses
|
||||
.map((w) => w.completedAt)
|
||||
.filter((s) => s !== null && s !== undefined)
|
||||
.map((s) => new Date(s).getTime())
|
||||
.filter((t) => Number.isFinite(t))
|
||||
if (stamps.length < 2) {
|
||||
return null
|
||||
}
|
||||
return (Math.max(...stamps) - Math.min(...stamps)) / 3600000
|
||||
}
|
||||
|
||||
// How a single check behaved across the witnesses.
|
||||
export const CHECK_KIND = {
|
||||
// Red in every witness that ran it: main was broken for the whole window.
|
||||
alwaysRed: 'always-red',
|
||||
// Every red falls inside one unbroken stretch of time with no green in it:
|
||||
// main broke at the start of that stretch. Covers a break that is still open
|
||||
// (no greens after), one that was fixed (greens after), and one that both
|
||||
// opened and closed inside the window.
|
||||
windowed: 'windowed',
|
||||
// Greens fall inside the red stretch: the branches differ, not main.
|
||||
interleaved: 'interleaved',
|
||||
// Too few witnesses or stacks to attribute either way.
|
||||
tooNarrow: 'too-narrow'
|
||||
}
|
||||
|
||||
function timeOf(w) {
|
||||
const t =
|
||||
w.completedAt === null || w.completedAt === undefined
|
||||
? Number.NaN
|
||||
: new Date(w.completedAt).getTime()
|
||||
return Number.isFinite(t) ? t : null
|
||||
}
|
||||
|
||||
// Classifies one check name by whether its reds and greens separate cleanly in
|
||||
// time. A clean split is a main-side transition; interleaving is branch-specific.
|
||||
function classifyOneCheck(name, usable, minWitnesses) {
|
||||
const red = usable.filter((w) => w.failures.includes(name))
|
||||
const green = usable.filter((w) => !w.failures.includes(name) && w.ran.includes(name))
|
||||
const redStacks = new Set(red.map((w) => w.stackId ?? w.ref))
|
||||
const redTimes = red.map(timeOf).filter((t) => t !== null)
|
||||
const greenTimes = green.map(timeOf).filter((t) => t !== null)
|
||||
const base = {
|
||||
name,
|
||||
redRefs: red.map((w) => w.ref),
|
||||
greenRefs: green.map((w) => w.ref),
|
||||
redStacks: redStacks.size,
|
||||
firstRed: redTimes.length > 0 ? Math.min(...redTimes) : null,
|
||||
lastRed: redTimes.length > 0 ? Math.max(...redTimes) : null,
|
||||
firstGreen: greenTimes.length > 0 ? Math.min(...greenTimes) : null,
|
||||
lastGreen: greenTimes.length > 0 ? Math.max(...greenTimes) : null
|
||||
}
|
||||
// A green inside the red stretch means the tree was fine there, so the reds
|
||||
// around it belong to their own branches. This is checked before the witness
|
||||
// thresholds: one red against many greens is branch-specific evidence, not
|
||||
// thin evidence, and calling it `too-narrow` would make `clean` unreachable.
|
||||
const greenInsideRedStretch =
|
||||
base.firstRed !== null && greenTimes.some((t) => t >= base.firstRed && t <= base.lastRed)
|
||||
if (greenInsideRedStretch) {
|
||||
return { ...base, kind: CHECK_KIND.interleaved }
|
||||
}
|
||||
if (base.firstRed === null) {
|
||||
return { ...base, kind: CHECK_KIND.tooNarrow, reason: 'no red carries a timestamp' }
|
||||
}
|
||||
// A red confined to one stack with greens on both sides of it: main would have
|
||||
// had to break and be fixed between two adjacent witnesses, which no other
|
||||
// branch could have inherited. That makes it the branch's own failure.
|
||||
const bracketedByGreens =
|
||||
greenTimes.some((t) => t < base.firstRed) && greenTimes.some((t) => t > base.lastRed)
|
||||
if (redStacks.size < minWitnesses && bracketedByGreens) {
|
||||
return { ...base, kind: CHECK_KIND.interleaved }
|
||||
}
|
||||
if (red.length < minWitnesses || redStacks.size < minWitnesses) {
|
||||
return {
|
||||
...base,
|
||||
kind: CHECK_KIND.tooNarrow,
|
||||
reason:
|
||||
redStacks.size < minWitnesses
|
||||
? `red in only ${redStacks.size} independent stack(s)`
|
||||
: `red in only ${red.length} witness(es)`
|
||||
}
|
||||
}
|
||||
if (green.length === 0) {
|
||||
return { ...base, kind: CHECK_KIND.alwaysRed }
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
kind: CHECK_KIND.windowed,
|
||||
lastGreenBefore: Math.max(
|
||||
...greenTimes.filter((t) => t < base.firstRed),
|
||||
Number.NEGATIVE_INFINITY
|
||||
),
|
||||
firstGreenAfter: Math.min(
|
||||
...greenTimes.filter((t) => t > base.lastRed),
|
||||
Number.POSITIVE_INFINITY
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// True when `main` is known broken for this check at time `at` (ms). The blind
|
||||
// gaps on either side of the red stretch answer false, not true.
|
||||
export function brokenAt(check, at) {
|
||||
if (check.kind === CHECK_KIND.alwaysRed) {
|
||||
return true
|
||||
}
|
||||
if (check.kind === CHECK_KIND.windowed) {
|
||||
return at >= check.firstRed && at <= check.lastRed
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Per-check attribution across witnesses.
|
||||
export function classifyChecks(witnesses, options = {}) {
|
||||
const minWitnesses = options.minWitnesses ?? MIN_WITNESSES
|
||||
const usable = witnesses.filter((w) => w.usable)
|
||||
const names = new Set()
|
||||
for (const w of usable) {
|
||||
for (const name of w.failures) {
|
||||
names.add(name)
|
||||
}
|
||||
}
|
||||
|
||||
const checks = [...names].sort().map((name) => classifyOneCheck(name, usable, minWitnesses))
|
||||
const upstream = checks.filter((c) => c.kind === CHECK_KIND.alwaysRed)
|
||||
const transitions = checks.filter((c) => c.kind === CHECK_KIND.windowed)
|
||||
const branchSpecific = checks.filter((c) => c.kind === CHECK_KIND.interleaved)
|
||||
const inconclusive = checks.filter((c) => c.kind === CHECK_KIND.tooNarrow)
|
||||
|
||||
const failing = usable.filter((w) => w.failures.length > 0)
|
||||
const signatures = new Set(failing.map((w) => w.failures.join(' ')))
|
||||
const failingStacks = new Set(failing.map((w) => w.stackId ?? w.ref))
|
||||
|
||||
return {
|
||||
checks,
|
||||
upstream,
|
||||
transitions,
|
||||
branchSpecific,
|
||||
inconclusive,
|
||||
evidenceSpanHours: evidenceSpanHours(usable),
|
||||
identicalAcrossFailing: failing.length >= minWitnesses && signatures.size === 1,
|
||||
failingWitnesses: failing.length,
|
||||
failingStacks: failingStacks.size,
|
||||
usableWitnesses: usable.length,
|
||||
totalWitnesses: witnesses.length,
|
||||
independentStacks: new Set(usable.map((w) => w.stackId ?? w.ref)).size
|
||||
}
|
||||
}
|
||||
|
||||
// Rejects evidence whose checks are too far apart in time to have observed the
|
||||
// same `main`. Returns null when the span is acceptable.
|
||||
function staleEvidenceVerdict(classification, options) {
|
||||
const maxSpan = options.maxSpanHours ?? MAX_EVIDENCE_SPAN_HOURS
|
||||
const span = classification.evidenceSpanHours
|
||||
if (span === null || span === undefined) {
|
||||
return { verdict: VERDICT.unknown, why: 'fewer than two witnesses carry a check timestamp' }
|
||||
}
|
||||
if (span > maxSpan) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `witness checks span ${span.toFixed(1)}h (max ${maxSpan}h); they ran against different mains, so agreeing failures prove nothing`
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
// `compare` verdict: are these branches red for the same reason or different ones?
|
||||
export function compareVerdict(classification, options = {}) {
|
||||
const minWitnesses = options.minWitnesses ?? MIN_WITNESSES
|
||||
if (classification.usableWitnesses < minWitnesses) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `only ${classification.usableWitnesses} of ${classification.totalWitnesses} refs ran a lane that exercises the tree (need ${minWitnesses})`
|
||||
}
|
||||
}
|
||||
if (classification.failingWitnesses === 0) {
|
||||
return { verdict: VERDICT.noFailures, why: 'no real failures on any ref' }
|
||||
}
|
||||
const stale = staleEvidenceVerdict(classification, options)
|
||||
if (stale !== null) {
|
||||
return stale
|
||||
}
|
||||
if (classification.failingWitnesses < minWitnesses) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `only ${classification.failingWitnesses} ref is failing; a single red ref cannot be told apart from upstream breakage`
|
||||
}
|
||||
}
|
||||
if (!classification.identicalAcrossFailing) {
|
||||
return {
|
||||
verdict: VERDICT.divergent,
|
||||
why: 'failing refs do not share one failure set, so at least part of the damage is branch-specific'
|
||||
}
|
||||
}
|
||||
if (classification.failingStacks < minWitnesses) {
|
||||
return {
|
||||
verdict: VERDICT.sharedAncestor,
|
||||
why: 'identical failures, but every failing ref is in one stack: the cause is at or below that stack root (main, or the root PR itself)'
|
||||
}
|
||||
}
|
||||
return {
|
||||
verdict: VERDICT.upstream,
|
||||
why: 'identical failure set across independent stacks; damage from a merge varies with what each branch changed, so this is upstream'
|
||||
}
|
||||
}
|
||||
|
||||
// A commit sitting between the last green and the first red of a break — or
|
||||
// between the last red and the green that follows — is in the blind gap: no
|
||||
// witness observed main there, so the answer is unknown, not clean.
|
||||
function inBlindGap(check, at) {
|
||||
if (check.kind !== CHECK_KIND.windowed) {
|
||||
return false
|
||||
}
|
||||
const beforeGap = at > check.lastGreenBefore && at < check.firstRed
|
||||
const afterGap = at > check.lastRed && at < check.firstGreenAfter
|
||||
return beforeGap || afterGap
|
||||
}
|
||||
|
||||
// `at` verdict: was main broken at this commit? Never returns `clean` without
|
||||
// positive evidence from enough independent witnesses.
|
||||
export function mainHealthVerdict(classification, at, options = {}) {
|
||||
const minWitnesses = options.minWitnesses ?? MIN_WITNESSES
|
||||
if (classification.usableWitnesses < minWitnesses) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `only ${classification.usableWitnesses} of ${classification.totalWitnesses} witness PRs ran a lane that exercises the tree (need ${minWitnesses})`,
|
||||
brokenChecks: []
|
||||
}
|
||||
}
|
||||
if (classification.independentStacks < minWitnesses) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `witnesses span only ${classification.independentStacks} independent stack(s); they share a diff and cannot corroborate each other`,
|
||||
brokenChecks: []
|
||||
}
|
||||
}
|
||||
const stale = staleEvidenceVerdict(classification, options)
|
||||
if (stale !== null) {
|
||||
return { ...stale, brokenChecks: [] }
|
||||
}
|
||||
const broken = classification.checks.filter((c) => brokenAt(c, at))
|
||||
if (broken.length > 0) {
|
||||
return {
|
||||
verdict: VERDICT.broken,
|
||||
why: `${broken.length} check(s) were failing on main at this commit: ${broken.map((c) => c.name).join(', ')}`,
|
||||
brokenChecks: broken.map((c) => c.name)
|
||||
}
|
||||
}
|
||||
const gaps = classification.checks.filter((c) => inBlindGap(c, at))
|
||||
if (gaps.length > 0) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `${gaps.length} check(s) changed state inside this window with no witness at the commit itself: ${gaps.map((c) => c.name).join(', ')}`,
|
||||
brokenChecks: []
|
||||
}
|
||||
}
|
||||
if (classification.inconclusive.length > 0) {
|
||||
return {
|
||||
verdict: VERDICT.unknown,
|
||||
why: `${classification.inconclusive.length} failing check(s) were seen too narrowly to attribute; treat main as unproven`,
|
||||
brokenChecks: []
|
||||
}
|
||||
}
|
||||
return {
|
||||
verdict: VERDICT.clean,
|
||||
why: `${classification.usableWitnesses} witnesses across ${classification.independentStacks} independent stacks; every check that failed here failed only on its own branch`,
|
||||
brokenChecks: []
|
||||
}
|
||||
}
|
||||
|
||||
// Keeps the PRs whose CI actually completed inside the window. A PR merged in
|
||||
// the window may have last run CI days earlier, against a different `main`.
|
||||
export function selectWitnessesInWindow(witnesses, from, to) {
|
||||
return witnesses.filter((w) => {
|
||||
if (w.completedAt === null || w.completedAt === undefined) {
|
||||
return false
|
||||
}
|
||||
const at = new Date(w.completedAt).getTime()
|
||||
return at >= from.getTime() && at <= to.getTime()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
CHECK_KIND,
|
||||
MIN_WITNESSES,
|
||||
VERDICT,
|
||||
brokenAt,
|
||||
buildStacks,
|
||||
classifyChecks,
|
||||
compareVerdict,
|
||||
evidenceSpanHours,
|
||||
isWitnessingCheck,
|
||||
mainHealthVerdict,
|
||||
normalizeChecks,
|
||||
selectWitnessesInWindow
|
||||
} from './upstream-breakage-evidence.mjs'
|
||||
|
||||
// Verbatim GitHub check-run payloads, not hand-written shapes: the bug these pin
|
||||
// is that real path-filtered PRs post three green meta jobs and nothing else.
|
||||
const FIXTURE = JSON.parse(
|
||||
readFileSync(new URL('./upstream-breakage-check-runs.fixture.json', import.meta.url), 'utf8')
|
||||
)
|
||||
const DOCS_ONLY = FIXTURE.prs['17002']
|
||||
const MOBILE_ONLY = FIXTURE.prs['16917']
|
||||
const FULL_CI = FIXTURE.prs['17358']
|
||||
|
||||
const HOUR = 3600 * 1000
|
||||
const T0 = Date.parse('2026-08-30T03:00:00Z')
|
||||
|
||||
function check(name, conclusion, extra = {}) {
|
||||
return {
|
||||
name,
|
||||
status: 'completed',
|
||||
conclusion,
|
||||
appSlug: 'github-actions',
|
||||
completedAt: new Date(T0).toISOString(),
|
||||
...extra
|
||||
}
|
||||
}
|
||||
|
||||
// Witnesses built from the captured payloads, stacked the way the probe stacks
|
||||
// them, so the verdict path under test is the one the CLI runs.
|
||||
function realWitnesses(entries) {
|
||||
const stacks = buildStacks(entries.map((entry) => entry.pr))
|
||||
const stackOf = new Map()
|
||||
for (const [index, members] of stacks.entries()) {
|
||||
for (const number of members) {
|
||||
stackOf.set(number, `stack-${index + 1}`)
|
||||
}
|
||||
}
|
||||
return entries.map((entry) => ({
|
||||
ref: `#${entry.pr.number}`,
|
||||
stackId: stackOf.get(entry.pr.number),
|
||||
...normalizeChecks(entry.runs)
|
||||
}))
|
||||
}
|
||||
|
||||
// A usable witness with the given failures, all other named checks green.
|
||||
function witness(ref, stackId, offsetMinutes, failures, ran = []) {
|
||||
const names = [...new Set([...failures, ...ran])]
|
||||
return {
|
||||
ref,
|
||||
stackId,
|
||||
usable: true,
|
||||
incomplete: 0,
|
||||
completedAt: new Date(T0 + offsetMinutes * 60 * 1000).toISOString(),
|
||||
failures: [...failures].sort(),
|
||||
ran: names.sort(),
|
||||
excluded: { rollup: [], knownFalse: [], foreignApp: [] }
|
||||
}
|
||||
}
|
||||
|
||||
describe('normalizeChecks', () => {
|
||||
it('keeps a real failure and the checks that ran green', () => {
|
||||
const result = normalizeChecks([
|
||||
check('typecheck', 'failure'),
|
||||
check('static analysis', 'success')
|
||||
])
|
||||
expect(result.failures).toEqual(['typecheck'])
|
||||
expect(result.ran).toEqual(['static analysis', 'typecheck'])
|
||||
expect(result.usable).toBe(true)
|
||||
})
|
||||
|
||||
it('never counts the verify roll-up as an independent failure', () => {
|
||||
const result = normalizeChecks([check('verify', 'failure'), check('typecheck', 'failure')])
|
||||
expect(result.failures).toEqual(['typecheck'])
|
||||
expect(result.excluded.rollup).toEqual(['verify'])
|
||||
})
|
||||
|
||||
it('does not report a green roll-up as an excluded red', () => {
|
||||
expect(
|
||||
normalizeChecks([check('verify', 'success'), check('typecheck', 'success')]).excluded.rollup
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('drops the known-false reds but reports what it dropped', () => {
|
||||
const result = normalizeChecks([
|
||||
check('test / tests node 24 1/8', 'failure'),
|
||||
check('test / tests node 24 6/8', 'failure'),
|
||||
check('e2e / ssh docker watcher isolation', 'failure'),
|
||||
check('test / tests node 24 2/8', 'failure')
|
||||
])
|
||||
expect(result.failures).toEqual(['test / tests node 24 2/8'])
|
||||
expect(result.excluded.knownFalse).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('counts the known-false reds when explicitly asked to', () => {
|
||||
const result = normalizeChecks([check('test / tests node 24 1/8', 'failure')], {
|
||||
includeKnownFalse: true
|
||||
})
|
||||
expect(result.failures).toEqual(['test / tests node 24 1/8'])
|
||||
})
|
||||
|
||||
it('ignores third-party app checks', () => {
|
||||
const result = normalizeChecks([
|
||||
check('Greptile Review', 'failure', { appSlug: 'greptile-apps' }),
|
||||
check('typecheck', 'success')
|
||||
])
|
||||
expect(result.failures).toEqual([])
|
||||
expect(result.excluded.foreignApp).toEqual(['Greptile Review'])
|
||||
})
|
||||
|
||||
it('treats a skipped check as not run rather than as a pass', () => {
|
||||
const result = normalizeChecks([check('package', 'skipped')])
|
||||
expect(result.ran).toEqual([])
|
||||
expect(result.usable).toBe(false)
|
||||
})
|
||||
|
||||
it('is unusable while a check is still in progress', () => {
|
||||
const result = normalizeChecks([
|
||||
check('typecheck', null, { status: 'in_progress' }),
|
||||
check('static analysis', 'success')
|
||||
])
|
||||
expect(result.usable).toBe(false)
|
||||
expect(result.incomplete).toBe(1)
|
||||
})
|
||||
|
||||
it('counts a timed-out check as a failure', () => {
|
||||
expect(normalizeChecks([check('package', 'timed_out')]).failures).toEqual(['package'])
|
||||
})
|
||||
|
||||
it('dedupes a check name a re-run posted twice', () => {
|
||||
const result = normalizeChecks([check('typecheck', 'failure'), check('typecheck', 'failure')])
|
||||
expect(result.failures).toEqual(['typecheck'])
|
||||
})
|
||||
|
||||
// The bug: `#17002` is a docs-only PR and `#16917` a mobile-only one. Both
|
||||
// path-filter every lane in pr.yml away, and both still post three green
|
||||
// meta jobs — the path classifier, the root-directory guard and the LoC
|
||||
// counter. Counting those as a witness is how the probe reported `clean` for a
|
||||
// window in which no test, typecheck or static-analysis lane had run at all.
|
||||
it('is not a witness when only the always-on meta jobs ran (docs-only PR #17002)', () => {
|
||||
const result = normalizeChecks(DOCS_ONLY.runs)
|
||||
expect(result.incomplete).toBe(0)
|
||||
expect(result.ran).toEqual([
|
||||
'detect code-relevant changes',
|
||||
'root directory guard',
|
||||
'test vs non-test LoC'
|
||||
])
|
||||
expect(result.witnessing).toEqual([])
|
||||
expect(result.usable).toBe(false)
|
||||
})
|
||||
|
||||
it('is not a witness when only the always-on meta jobs ran (mobile-only PR #16917)', () => {
|
||||
const result = normalizeChecks(MOBILE_ONLY.runs)
|
||||
expect(result.witnessing).toEqual([])
|
||||
expect(result.usable).toBe(false)
|
||||
})
|
||||
|
||||
it('names the checks it refused to treat as witnesses rather than dropping them', () => {
|
||||
expect(normalizeChecks(DOCS_ONLY.runs).excluded.nonWitnessing).toEqual([
|
||||
'detect code-relevant changes',
|
||||
'root directory guard',
|
||||
'test vs non-test LoC'
|
||||
])
|
||||
})
|
||||
|
||||
// Positive control for the two above: the same gate on a PR whose lanes really
|
||||
// ran leaves it usable, so `usable: false` is the meta jobs, not the gate.
|
||||
it('is a witness when the real lanes ran (full-CI PR #17358)', () => {
|
||||
const result = normalizeChecks(FULL_CI.runs)
|
||||
expect(result.usable).toBe(true)
|
||||
expect(result.witnessing).toContain('typecheck')
|
||||
expect(result.witnessing).toContain('static analysis')
|
||||
expect(result.witnessing).toContain('test / tests node 24 2/8')
|
||||
})
|
||||
|
||||
it('reports the latest completion time it saw', () => {
|
||||
const later = new Date(T0 + HOUR).toISOString()
|
||||
const result = normalizeChecks([
|
||||
check('typecheck', 'success'),
|
||||
check('static analysis', 'success', { completedAt: later })
|
||||
])
|
||||
expect(result.completedAt).toBe(later)
|
||||
})
|
||||
})
|
||||
|
||||
describe('isWitnessingCheck', () => {
|
||||
it('counts the lanes that build, type-check or run the suite', () => {
|
||||
for (const name of [
|
||||
'typecheck',
|
||||
'static analysis',
|
||||
'test',
|
||||
'test / tests node 24 2/8',
|
||||
'package',
|
||||
'package (windows)',
|
||||
'e2e',
|
||||
'e2e / e2e 3-of-14',
|
||||
'shell contracts',
|
||||
'Git compatibility',
|
||||
'cross-version wire compatibility',
|
||||
'managed hooks on Node 18'
|
||||
]) {
|
||||
expect(isWitnessingCheck(name)).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('does not count a job that stays green on a broken main', () => {
|
||||
for (const name of [
|
||||
'detect code-relevant changes',
|
||||
'root directory guard',
|
||||
'test vs non-test LoC',
|
||||
'track-community-pr',
|
||||
'detect changed e2e specs',
|
||||
'e2e / changed e2e specs'
|
||||
]) {
|
||||
expect(isWitnessingCheck(name)).toBe(false)
|
||||
}
|
||||
})
|
||||
|
||||
// An unrecognised name costs a witness (verdict degrades to `unknown`) instead
|
||||
// of buying one, which is the whole reason this is an allowlist.
|
||||
it('does not count a name it has never seen', () => {
|
||||
expect(isWitnessingCheck('some future bot check')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildStacks', () => {
|
||||
it('puts unrelated PRs based on main in their own stacks', () => {
|
||||
const stacks = buildStacks([
|
||||
{ number: 1, headRefName: 'a', baseRefName: 'main' },
|
||||
{ number: 2, headRefName: 'b', baseRefName: 'main' }
|
||||
])
|
||||
expect(stacks).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('groups a chain of stacked PRs into one stack', () => {
|
||||
const stacks = buildStacks([
|
||||
{ number: 1, headRefName: 'a', baseRefName: 'main' },
|
||||
{ number: 2, headRefName: 'b', baseRefName: 'a' },
|
||||
{ number: 3, headRefName: 'c', baseRefName: 'b' }
|
||||
])
|
||||
expect(stacks).toEqual([[1, 2, 3]])
|
||||
})
|
||||
|
||||
it('groups a chain listed leaf-first', () => {
|
||||
const stacks = buildStacks([
|
||||
{ number: 3, headRefName: 'c', baseRefName: 'b' },
|
||||
{ number: 2, headRefName: 'b', baseRefName: 'a' },
|
||||
{ number: 1, headRefName: 'a', baseRefName: 'main' }
|
||||
])
|
||||
expect(stacks).toEqual([[1, 2, 3]])
|
||||
})
|
||||
|
||||
// The bug: grouping only by *listed* parents counted these as two independent
|
||||
// stacks, so the corroboration `broken` and `upstream` rest on was one witness
|
||||
// counted twice. Both are real open PRs on a base branch that has no PR here.
|
||||
it('counts two PRs on one unlisted parent branch as a single stack', () => {
|
||||
const siblings = FIXTURE.siblings.filter(
|
||||
(pr) => pr.baseRefName === 'brennanb2025/claude-structured-mobile'
|
||||
)
|
||||
expect(siblings.map((pr) => pr.number).sort((a, b) => a - b)).toEqual([15356, 15657])
|
||||
expect(siblings.some((pr) => pr.headRefName === siblings[0].baseRefName)).toBe(false)
|
||||
expect(buildStacks(siblings)).toEqual([[15356, 15657]])
|
||||
})
|
||||
|
||||
it('counts three PRs on one unlisted parent branch as a single stack', () => {
|
||||
const siblings = FIXTURE.siblings.filter(
|
||||
(pr) => pr.baseRefName === 'brennanb2025/probe-liveness-coercion-r1'
|
||||
)
|
||||
expect(buildStacks(siblings)).toEqual([[17044, 17058, 17074]])
|
||||
})
|
||||
|
||||
it('still separates two PRs on different unlisted parent branches', () => {
|
||||
const stacks = buildStacks([
|
||||
{ number: 1, headRefName: 'a', baseRefName: 'parent-one' },
|
||||
{ number: 2, headRefName: 'b', baseRefName: 'parent-two' }
|
||||
])
|
||||
expect(stacks).toEqual([[1], [2]])
|
||||
})
|
||||
|
||||
it('keeps two separate chains separate', () => {
|
||||
const stacks = buildStacks([
|
||||
{ number: 1, headRefName: 'a', baseRefName: 'main' },
|
||||
{ number: 2, headRefName: 'b', baseRefName: 'a' },
|
||||
{ number: 10, headRefName: 'x', baseRefName: 'main' },
|
||||
{ number: 11, headRefName: 'y', baseRefName: 'x' }
|
||||
])
|
||||
expect(stacks).toEqual([
|
||||
[1, 2],
|
||||
[10, 11]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('evidenceSpanHours', () => {
|
||||
it('returns null when fewer than two witnesses carry a timestamp', () => {
|
||||
expect(evidenceSpanHours([witness('#1', 's1', 0, [])])).toBeNull()
|
||||
expect(evidenceSpanHours([{ ref: '#1' }, { ref: '#2' }])).toBeNull()
|
||||
})
|
||||
|
||||
it('measures the spread between the earliest and latest witness', () => {
|
||||
const span = evidenceSpanHours([witness('#1', 's1', 0, []), witness('#2', 's2', 120, [])])
|
||||
expect(span).toBeCloseTo(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('classifyChecks', () => {
|
||||
it('calls a check red in every witness that ran it always-red', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck'])
|
||||
])
|
||||
expect(result.upstream.map((c) => c.name)).toEqual(['typecheck'])
|
||||
})
|
||||
|
||||
it('finds a break that opens partway through the window', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck'])
|
||||
])
|
||||
expect(result.transitions).toHaveLength(1)
|
||||
expect(result.transitions[0].kind).toBe(CHECK_KIND.windowed)
|
||||
expect(result.transitions[0].lastGreenBefore).toBe(T0)
|
||||
})
|
||||
|
||||
it('finds a break that opened and closed inside the window', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck']),
|
||||
witness('#4', 's4', 30, [], ['typecheck'])
|
||||
])
|
||||
expect(result.transitions).toHaveLength(1)
|
||||
expect(result.transitions[0].firstGreenAfter).toBe(T0 + 30 * 60 * 1000)
|
||||
})
|
||||
|
||||
it('calls interleaved reds and greens branch-specific', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, [], ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck'])
|
||||
])
|
||||
expect(result.branchSpecific.map((c) => c.name)).toEqual(['typecheck'])
|
||||
expect(result.transitions).toEqual([])
|
||||
})
|
||||
|
||||
it('calls a lone red bracketed by greens branch-specific, not thin evidence', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['root directory guard']),
|
||||
witness('#2', 's2', 10, ['root directory guard']),
|
||||
witness('#3', 's3', 20, [], ['root directory guard'])
|
||||
])
|
||||
expect(result.branchSpecific.map((c) => c.name)).toEqual(['root directory guard'])
|
||||
expect(result.inconclusive).toEqual([])
|
||||
})
|
||||
|
||||
it('refuses to attribute a red confined to one stack', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's1', 10, ['typecheck'])
|
||||
])
|
||||
expect(result.upstream).toEqual([])
|
||||
expect(result.inconclusive.map((c) => c.name)).toEqual(['typecheck'])
|
||||
})
|
||||
|
||||
it('ignores witnesses whose checks never completed', () => {
|
||||
const stalled = { ...witness('#3', 's3', 5, ['typecheck']), usable: false }
|
||||
const result = classifyChecks([witness('#1', 's1', 0, ['typecheck']), stalled])
|
||||
expect(result.usableWitnesses).toBe(1)
|
||||
})
|
||||
|
||||
it('reports identical failure sets across the failing witnesses', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck', 'static analysis']),
|
||||
witness('#2', 's2', 10, ['static analysis', 'typecheck'])
|
||||
])
|
||||
expect(result.identicalAcrossFailing).toBe(true)
|
||||
})
|
||||
|
||||
it('reports divergent failure sets as not identical', () => {
|
||||
const result = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, ['package'])
|
||||
])
|
||||
expect(result.identicalAcrossFailing).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('brokenAt', () => {
|
||||
const windowed = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck']),
|
||||
witness('#4', 's4', 30, [], ['typecheck'])
|
||||
]).transitions[0]
|
||||
|
||||
it('is broken inside the red stretch', () => {
|
||||
expect(brokenAt(windowed, T0 + 15 * 60 * 1000)).toBe(true)
|
||||
})
|
||||
|
||||
it('is not broken before the stretch opens', () => {
|
||||
expect(brokenAt(windowed, T0)).toBe(false)
|
||||
})
|
||||
|
||||
it('is not broken after the stretch closes', () => {
|
||||
expect(brokenAt(windowed, T0 + 30 * 60 * 1000)).toBe(false)
|
||||
})
|
||||
|
||||
it('is broken everywhere for an always-red check', () => {
|
||||
const always = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck'])
|
||||
]).upstream[0]
|
||||
expect(brokenAt(always, T0 - HOUR)).toBe(true)
|
||||
})
|
||||
|
||||
it('is never broken for an interleaved check', () => {
|
||||
const interleaved = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, [], ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck'])
|
||||
]).branchSpecific[0]
|
||||
expect(brokenAt(interleaved, T0 + 5 * 60 * 1000)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('mainHealthVerdict', () => {
|
||||
it('reports broken and names the checks', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck']),
|
||||
witness('#3', 's3', 20, ['typecheck'])
|
||||
])
|
||||
const verdict = mainHealthVerdict(classification, T0 + 15 * 60 * 1000)
|
||||
expect(verdict.verdict).toBe(VERDICT.broken)
|
||||
expect(verdict.brokenChecks).toEqual(['typecheck'])
|
||||
})
|
||||
|
||||
it('answers unknown, never clean, with a single witness', () => {
|
||||
const verdict = mainHealthVerdict(classifyChecks([witness('#1', 's1', 0, [])]), T0)
|
||||
expect(verdict.verdict).toBe(VERDICT.unknown)
|
||||
})
|
||||
|
||||
it('answers unknown when every witness is in one stack', () => {
|
||||
const classification = classifyChecks([witness('#1', 's1', 0, []), witness('#2', 's1', 10, [])])
|
||||
expect(mainHealthVerdict(classification, T0).verdict).toBe(VERDICT.unknown)
|
||||
})
|
||||
|
||||
it('answers unknown when the witnesses observed different mains', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, []),
|
||||
witness('#2', 's2', 60 * 48, [])
|
||||
])
|
||||
const verdict = mainHealthVerdict(classification, T0)
|
||||
expect(verdict.verdict).toBe(VERDICT.unknown)
|
||||
expect(verdict.why).toContain('different mains')
|
||||
})
|
||||
|
||||
it('answers unknown in the blind gap between the last green and the first red', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 20, ['typecheck']),
|
||||
witness('#3', 's3', 30, ['typecheck'])
|
||||
])
|
||||
const verdict = mainHealthVerdict(classification, T0 + 10 * 60 * 1000)
|
||||
expect(verdict.verdict).toBe(VERDICT.unknown)
|
||||
expect(verdict.why).toContain('no witness at the commit itself')
|
||||
})
|
||||
|
||||
it('answers unknown when a failing check was seen too narrowly', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['package']),
|
||||
witness('#2', 's2', 10, []),
|
||||
witness('#3', 's3', 20, [])
|
||||
])
|
||||
expect(mainHealthVerdict(classification, T0 + 5 * 60 * 1000).verdict).toBe(VERDICT.unknown)
|
||||
})
|
||||
|
||||
it('reports clean only with corroborating independent witnesses', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, [], ['typecheck'])
|
||||
])
|
||||
expect(mainHealthVerdict(classification, T0 + 5 * 60 * 1000).verdict).toBe(VERDICT.clean)
|
||||
})
|
||||
|
||||
// End-to-end on the payload that reproduced this: two real merged PRs, 2.2h
|
||||
// apart, in two independent stacks, on each of which nothing but the three
|
||||
// always-on meta jobs ran. Before the witnessing-lane gate this returned
|
||||
// `clean` — a green verdict for a window in which nothing was tested.
|
||||
it('answers unknown, not clean, when no witness ran a lane that exercises the tree', () => {
|
||||
const witnesses = realWitnesses([DOCS_ONLY, MOBILE_ONLY])
|
||||
const classification = classifyChecks(witnesses)
|
||||
expect(classification.independentStacks).toBe(0)
|
||||
const verdict = mainHealthVerdict(classification, Date.parse(DOCS_ONLY.runs[0].completedAt))
|
||||
expect(verdict.verdict).not.toBe(VERDICT.clean)
|
||||
expect(verdict.verdict).toBe(VERDICT.unknown)
|
||||
expect(verdict.why).toContain('exercises the tree')
|
||||
})
|
||||
|
||||
it('needs at least the documented number of witnesses', () => {
|
||||
expect(MIN_WITNESSES).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('compareVerdict', () => {
|
||||
it('calls an identical failure set across independent stacks upstream', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, ['typecheck'])
|
||||
])
|
||||
expect(compareVerdict(classification).verdict).toBe(VERDICT.upstream)
|
||||
})
|
||||
|
||||
it('calls differing failure sets divergent', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 10, ['package'])
|
||||
])
|
||||
expect(compareVerdict(classification).verdict).toBe(VERDICT.divergent)
|
||||
})
|
||||
|
||||
it('will not call one stack upstream on its own', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's1', 10, ['typecheck'])
|
||||
])
|
||||
expect(compareVerdict(classification).verdict).toBe(VERDICT.sharedAncestor)
|
||||
})
|
||||
|
||||
it('refuses identical failures whose checks ran days apart', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck']),
|
||||
witness('#2', 's2', 60 * 71, ['typecheck'])
|
||||
])
|
||||
const verdict = compareVerdict(classification)
|
||||
expect(verdict.verdict).toBe(VERDICT.unknown)
|
||||
expect(verdict.why).toContain('different mains')
|
||||
})
|
||||
|
||||
it('reports no-failures when nothing is red', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, [], ['typecheck']),
|
||||
witness('#2', 's2', 10, [], ['typecheck'])
|
||||
])
|
||||
expect(compareVerdict(classification).verdict).toBe(VERDICT.noFailures)
|
||||
})
|
||||
|
||||
it('will not judge a single red ref', () => {
|
||||
const classification = classifyChecks([
|
||||
witness('#1', 's1', 0, ['typecheck'], ['typecheck', 'package']),
|
||||
witness('#2', 's2', 10, [], ['package'])
|
||||
])
|
||||
expect(compareVerdict(classification).verdict).toBe(VERDICT.unknown)
|
||||
})
|
||||
})
|
||||
|
||||
describe('selectWitnessesInWindow', () => {
|
||||
const from = new Date(T0)
|
||||
const to = new Date(T0 + HOUR)
|
||||
|
||||
it('keeps a witness whose CI completed inside the window', () => {
|
||||
expect(selectWitnessesInWindow([witness('#1', 's1', 30, [])], from, to)).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('drops a witness whose CI completed before the window', () => {
|
||||
expect(selectWitnessesInWindow([witness('#1', 's1', -30, [])], from, to)).toEqual([])
|
||||
})
|
||||
|
||||
it('drops a witness with no completion time rather than assuming it fits', () => {
|
||||
expect(selectWitnessesInWindow([{ ref: '#1', completedAt: null }], from, to)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,379 @@
|
||||
import { execFile, execFileSync } from 'node:child_process'
|
||||
import process from 'node:process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
import {
|
||||
MAX_EVIDENCE_SPAN_HOURS,
|
||||
VERDICT,
|
||||
buildStacks,
|
||||
classifyChecks,
|
||||
compareVerdict,
|
||||
mainHealthVerdict,
|
||||
normalizeChecks,
|
||||
selectWitnessesInWindow
|
||||
} from './upstream-breakage-evidence.mjs'
|
||||
|
||||
// CLI over the upstream-breakage evidence model: talks to gh and git, prints the
|
||||
// witness table, and reports a verdict. See
|
||||
// docs/reference/upstream-breakage-diagnosis.md.
|
||||
|
||||
const GH_BUFFER_BYTES = 32 * 1024 * 1024
|
||||
// Enough parallelism to keep a 40-witness window inside a few seconds without
|
||||
// tripping GitHub's secondary rate limits.
|
||||
const FETCH_CONCURRENCY = 8
|
||||
|
||||
function gh(args) {
|
||||
return execFileSync('gh', args, { encoding: 'utf8', maxBuffer: GH_BUFFER_BYTES })
|
||||
}
|
||||
|
||||
function ghAsync(args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile('gh', args, { encoding: 'utf8', maxBuffer: GH_BUFFER_BYTES }, (error, stdout) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
resolve(stdout)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
return execFileSync('git', args, { encoding: 'utf8' }).trim()
|
||||
}
|
||||
|
||||
const CHECK_RUN_JQ =
|
||||
'.check_runs[] | {name, status, conclusion, completedAt: .completed_at, appSlug: .app.slug}'
|
||||
|
||||
async function fetchCheckRuns(sha, { allAttempts }) {
|
||||
const filter = allAttempts === true ? 'all' : 'latest'
|
||||
const raw = await ghAsync([
|
||||
'api',
|
||||
`repos/{owner}/{repo}/commits/${sha}/check-runs?per_page=100&filter=${filter}`,
|
||||
'--paginate',
|
||||
'--jq',
|
||||
CHECK_RUN_JQ
|
||||
])
|
||||
return raw
|
||||
.split('\n')
|
||||
.filter((line) => line.trim() !== '')
|
||||
.map((line) => JSON.parse(line))
|
||||
}
|
||||
|
||||
// Bounded-concurrency map; keeps the probe in the seconds range for wide windows.
|
||||
async function mapWithConcurrency(items, worker) {
|
||||
const results = Array.from({ length: items.length })
|
||||
let next = 0
|
||||
const runners = Array.from({ length: Math.min(FETCH_CONCURRENCY, items.length) }, async () => {
|
||||
while (next < items.length) {
|
||||
const index = next
|
||||
next += 1
|
||||
results[index] = await worker(items[index])
|
||||
}
|
||||
})
|
||||
await Promise.all(runners)
|
||||
return results
|
||||
}
|
||||
|
||||
const PR_FIELDS = 'number,headRefName,headRefOid,baseRefName,mergedAt,title,url'
|
||||
|
||||
// Always passes an explicit limit and reports the count, because `gh pr list`
|
||||
// silently truncates at 30 and a truncated population reads as a complete one.
|
||||
function listPullRequests(extraArgs, limit) {
|
||||
const prs = JSON.parse(
|
||||
gh(['pr', 'list', '--limit', String(limit), '--json', PR_FIELDS, ...extraArgs])
|
||||
)
|
||||
return { prs, truncated: prs.length >= limit, limit }
|
||||
}
|
||||
|
||||
function viewPullRequest(number) {
|
||||
return JSON.parse(gh(['pr', 'view', String(number), '--json', PR_FIELDS]))
|
||||
}
|
||||
|
||||
function resolveCommit(rev) {
|
||||
// Never derive a SHA from an abbreviation by hand; let git verify it.
|
||||
return git(['rev-parse', '--verify', `${rev}^{commit}`])
|
||||
}
|
||||
|
||||
async function witnessesFromPrs(prs, options) {
|
||||
return mapWithConcurrency(prs, async (pr) => ({
|
||||
ref: `#${pr.number}`,
|
||||
pr,
|
||||
...normalizeChecks(await fetchCheckRuns(pr.headRefOid, options), options)
|
||||
}))
|
||||
}
|
||||
|
||||
function attachStacks(witnesses) {
|
||||
const stacks = buildStacks(witnesses.map((w) => w.pr))
|
||||
const stackOf = new Map()
|
||||
for (const [index, members] of stacks.entries()) {
|
||||
for (const number of members) {
|
||||
stackOf.set(number, `stack-${index + 1}`)
|
||||
}
|
||||
}
|
||||
return witnesses.map((w) => ({ ...w, stackId: stackOf.get(w.pr.number) ?? w.ref }))
|
||||
}
|
||||
|
||||
function describeWitness(w) {
|
||||
if (!w.usable) {
|
||||
if (w.incomplete > 0) {
|
||||
return `no verdict (${w.incomplete} check(s) not completed)`
|
||||
}
|
||||
return w.ran.length === 0
|
||||
? 'no verdict (nothing ran; path filters skipped every job)'
|
||||
: `no verdict (no lane exercised the tree; only ${w.ran.join(', ')} ran)`
|
||||
}
|
||||
return w.failures.length === 0
|
||||
? `green (${w.ran.length} checks ran)`
|
||||
: `RED: ${w.failures.join(', ')}`
|
||||
}
|
||||
|
||||
function printWitnesses(witnesses) {
|
||||
// Chronological: a main-side break shows up as greens above and reds below.
|
||||
const ordered = [...witnesses].sort((a, b) =>
|
||||
String(a.completedAt).localeCompare(String(b.completedAt))
|
||||
)
|
||||
for (const w of ordered) {
|
||||
const at =
|
||||
w.completedAt === null || w.completedAt === undefined
|
||||
? '?'
|
||||
: clock(new Date(w.completedAt).getTime())
|
||||
console.log(` ${at} ${w.ref} [${w.stackId}] ${describeWitness(w)}`)
|
||||
const dropped = [
|
||||
...new Set([
|
||||
...w.excluded.rollup.map((n) => `${n} (roll-up)`),
|
||||
...w.excluded.knownFalse.map((n) => `${n} (known-false red)`),
|
||||
...w.excluded.foreignApp.map((n) => `${n} (third-party app)`),
|
||||
// Printed so a renamed lane that dropped out of the witnessing list is
|
||||
// visible rather than quietly costing the run its evidence.
|
||||
...(w.excluded.nonWitnessing ?? []).map((n) => `${n} (does not exercise the tree)`)
|
||||
])
|
||||
]
|
||||
if (dropped.length > 0) {
|
||||
console.log(` excluded: ${dropped.join('; ')}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clock(ms) {
|
||||
return ms === null ? '?' : `${new Date(ms).toISOString().replace('T', ' ').slice(0, 16)}Z`
|
||||
}
|
||||
|
||||
function printClassification(c) {
|
||||
if (c.upstream.length > 0) {
|
||||
console.log('\nRed in every witness that ran it — main was broken for this whole window:')
|
||||
for (const u of c.upstream) {
|
||||
console.log(` ${u.name} — red in ${u.redRefs.join(', ')}`)
|
||||
}
|
||||
}
|
||||
if (c.transitions.length > 0) {
|
||||
console.log('\nRed for one unbroken stretch — main was broken across that stretch:')
|
||||
for (const t of c.transitions) {
|
||||
const opened = Number.isFinite(t.lastGreenBefore)
|
||||
? `after ${clock(t.lastGreenBefore)}`
|
||||
: 'before the window'
|
||||
const closed = Number.isFinite(t.firstGreenAfter)
|
||||
? `by ${clock(t.firstGreenAfter)}`
|
||||
: 'still open'
|
||||
console.log(
|
||||
` ${t.name} — red ${clock(t.firstRed)}..${clock(t.lastRed)} (broke ${opened}; fixed ${closed})`
|
||||
)
|
||||
console.log(` red: ${t.redRefs.join(', ')}`)
|
||||
}
|
||||
}
|
||||
if (c.branchSpecific.length > 0) {
|
||||
console.log('\nReds and greens interleave in time — branch-specific, not main:')
|
||||
for (const b of c.branchSpecific) {
|
||||
console.log(
|
||||
` ${b.name} — red in ${b.redRefs.join(', ')}; green in ${b.greenRefs.join(', ')}`
|
||||
)
|
||||
}
|
||||
}
|
||||
if (c.inconclusive.length > 0) {
|
||||
console.log('\nSeen too narrowly to attribute:')
|
||||
for (const i of c.inconclusive) {
|
||||
console.log(` ${i.name} — red in ${i.redRefs.join(', ')}; ${i.reason}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const options = {
|
||||
mode: argv[0] ?? null,
|
||||
refs: [],
|
||||
commit: null,
|
||||
windowHours: 3,
|
||||
limit: 100,
|
||||
maxSpanHours: MAX_EVIDENCE_SPAN_HOURS,
|
||||
allAttempts: false,
|
||||
includeKnownFalse: false,
|
||||
json: false
|
||||
}
|
||||
const rest = argv.slice(1)
|
||||
while (rest.length > 0) {
|
||||
const arg = rest.shift()
|
||||
if (arg === '--window-hours') {
|
||||
options.windowHours = Number(rest.shift())
|
||||
} else if (arg === '--limit') {
|
||||
options.limit = Number(rest.shift())
|
||||
} else if (arg === '--max-span-hours') {
|
||||
options.maxSpanHours = Number(rest.shift())
|
||||
} else if (arg === '--all-attempts') {
|
||||
options.allAttempts = true
|
||||
} else if (arg === '--include-known-false') {
|
||||
options.includeKnownFalse = true
|
||||
} else if (arg === '--json') {
|
||||
options.json = true
|
||||
} else if (arg.startsWith('-')) {
|
||||
throw new Error(`unknown flag: ${arg}`)
|
||||
} else if (options.mode === 'at' && options.commit === null) {
|
||||
options.commit = arg
|
||||
} else {
|
||||
options.refs.push(arg.replace(/^#/, ''))
|
||||
}
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
const USAGE = `Usage:
|
||||
node config/scripts/upstream-breakage-probe.mjs at <commit> [--window-hours N] [--limit N]
|
||||
node config/scripts/upstream-breakage-probe.mjs compare <pr-number> ...
|
||||
|
||||
at Was main broken at/near this commit? Reconstructed from the PRs whose
|
||||
CI ran against main in the surrounding window.
|
||||
compare Do these red branches share one failure set (upstream) or not?
|
||||
|
||||
Flags:
|
||||
--window-hours N witness window for \`at\` (default 3)
|
||||
--limit N explicit gh pr list limit (default 100)
|
||||
--max-span-hours N reject evidence spread wider than this (default ${MAX_EVIDENCE_SPAN_HOURS})
|
||||
--all-attempts include earlier check attempts a re-run to green hid
|
||||
--include-known-false count the known-false reds instead of excluding them
|
||||
--json machine-readable output
|
||||
|
||||
Verdicts never default to pass: without enough evidence the answer is "unknown".`
|
||||
|
||||
function describeSpan(classification) {
|
||||
return classification.evidenceSpanHours === null
|
||||
? 'evidence span: unknown (fewer than two timestamps)'
|
||||
: `evidence span: ${classification.evidenceSpanHours.toFixed(1)}h`
|
||||
}
|
||||
|
||||
async function runCompare(options) {
|
||||
if (options.refs.length === 0) {
|
||||
throw new Error('compare needs at least one PR number')
|
||||
}
|
||||
const selected = options.refs.map(Number).map(viewPullRequest)
|
||||
const witnesses = attachStacks(await witnessesFromPrs(selected, options))
|
||||
const classification = classifyChecks(witnesses)
|
||||
const verdict = compareVerdict(classification, options)
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({ mode: 'compare', verdict, classification, witnesses }, null, 2))
|
||||
return
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Compared ${witnesses.length} ref(s) across ${classification.independentStacks} independent stack(s); ${describeSpan(classification)}.`
|
||||
)
|
||||
printWitnesses(witnesses)
|
||||
printClassification(classification)
|
||||
console.log(`\nVERDICT: ${verdict.verdict} — ${verdict.why}`)
|
||||
if (verdict.verdict === VERDICT.upstream) {
|
||||
console.log('Next step: merge a newer main into each branch. Do not repair the branches.')
|
||||
}
|
||||
if (verdict.verdict === VERDICT.sharedAncestor) {
|
||||
console.log(
|
||||
'Next step: compare against a red PR from a different stack, or probe the stack root with `at`.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function runAt(options) {
|
||||
if (options.commit === null) {
|
||||
throw new Error('at needs a commit')
|
||||
}
|
||||
const sha = resolveCommit(options.commit)
|
||||
const when = new Date(git(['show', '-s', '--format=%cI', sha]))
|
||||
const windowMs = options.windowHours * 3600 * 1000
|
||||
const from = new Date(when.getTime() - windowMs)
|
||||
const to = new Date(when.getTime() + windowMs)
|
||||
|
||||
const { prs, truncated, limit } = listPullRequests(['--state', 'merged'], options.limit)
|
||||
// CI completes before the merge, so preselect generously and let the
|
||||
// completion-time filter below decide.
|
||||
const candidates = prs.filter((pr) => {
|
||||
if (pr.mergedAt === null || pr.mergedAt === undefined) {
|
||||
return false
|
||||
}
|
||||
const merged = new Date(pr.mergedAt).getTime()
|
||||
return merged >= from.getTime() && merged <= to.getTime() + windowMs
|
||||
})
|
||||
|
||||
const fetched = await witnessesFromPrs(candidates, options)
|
||||
const witnesses = attachStacks(selectWitnessesInWindow(fetched, from, to))
|
||||
const classification = classifyChecks(witnesses)
|
||||
const verdict = mainHealthVerdict(classification, when.getTime(), options)
|
||||
|
||||
if (options.json) {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{ mode: 'at', sha, window: { from, to }, verdict, classification, witnesses },
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
console.log(`main @ ${sha} (${when.toISOString()})`)
|
||||
console.log(
|
||||
`Scanned ${prs.length} merged PR(s) at gh limit ${limit}; ${candidates.length} merged near the window, ${witnesses.length} with CI inside ±${options.windowHours}h, ${classification.usableWitnesses} usable across ${classification.independentStacks} independent stack(s); ${describeSpan(classification)}.`
|
||||
)
|
||||
if (truncated) {
|
||||
console.log(
|
||||
` WARNING: gh pr list returned exactly ${limit} rows (the limit); the window may be truncated — raise --limit.`
|
||||
)
|
||||
}
|
||||
printWitnesses(witnesses)
|
||||
printClassification(classification)
|
||||
console.log(`\nVERDICT: main was ${verdict.verdict} at this commit — ${verdict.why}`)
|
||||
if (verdict.verdict === VERDICT.broken) {
|
||||
console.log(
|
||||
'A branch red on these checks is inheriting main. Merge a newer main; do not repair the branch.'
|
||||
)
|
||||
}
|
||||
if (verdict.verdict === VERDICT.unknown) {
|
||||
console.log(
|
||||
'Unknown is not green. Widen --window-hours, or use `compare` on the red branches directly.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
let options
|
||||
try {
|
||||
options = parseArgs(argv)
|
||||
} catch (error) {
|
||||
console.error(`${error.message}\n\n${USAGE}`)
|
||||
process.exitCode = 2
|
||||
return
|
||||
}
|
||||
if (options.mode === 'at') {
|
||||
await runAt(options)
|
||||
return
|
||||
}
|
||||
if (options.mode === 'compare') {
|
||||
await runCompare(options)
|
||||
return
|
||||
}
|
||||
console.error(USAGE)
|
||||
process.exitCode = 2
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
||||
main(process.argv.slice(2)).catch((error) => {
|
||||
console.error(error.message ?? error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
# Telling "my merge broke it" apart from "main was already broken"
|
||||
|
||||
Read this before diagnosing a branch that went red after merging `main`, and
|
||||
before writing a repair brief for a stack that went red together.
|
||||
|
||||
## The mistake this exists to prevent
|
||||
|
||||
An eleven-PR chain merged `main` inside a window during which `main` itself was
|
||||
broken: a dependency bump had broken a group of test shards, and a lint violation
|
||||
was already sitting on `main`. Six of the eleven nodes went red. The reds were
|
||||
diagnosed as damage caused by the merge, written up, and acted on. All of it was
|
||||
wrong — the fix was to merge a newer `main`.
|
||||
|
||||
The tell was already in the data: **an identical failure multiset on every node,
|
||||
including the root**. Damage from a merge varies with what each node changed.
|
||||
Identical failures across an entire chain are upstream by construction.
|
||||
|
||||
## Why you cannot just look up main's CI
|
||||
|
||||
The obvious move is to ask GitHub whether `main` was green at a SHA. In this repo
|
||||
that question has no answer, and it is worth knowing why before you go looking:
|
||||
|
||||
- `pr.yml` — the workflow that holds the tests, typecheck, static analysis and
|
||||
packaging — is `on: pull_request` only.
|
||||
- `unit-tests.yml` is `workflow_call` only; nothing calls it from a push.
|
||||
- No workflow runs the test suite on a push to `main`. The only lanes that ever
|
||||
execute on `main`'s own commits are scheduled or manually dispatched (E2E, the
|
||||
mac builds, terminal perf, node-next), plus a path-filtered skill round-trip.
|
||||
|
||||
The result is that a commit on `main` carries **zero** check runs:
|
||||
|
||||
```
|
||||
$ gh api "repos/{owner}/{repo}/commits/<main-sha>/check-runs" --jq .total_count
|
||||
0
|
||||
```
|
||||
|
||||
The one lane that does run on `main` is scheduled E2E, twice a day, and it has
|
||||
been uniformly red for weeks — so it cannot discriminate either.
|
||||
|
||||
So `main`'s health has to be reconstructed from the PRs whose CI ran against
|
||||
`main` at that moment. That is what the probe does.
|
||||
|
||||
## What to run
|
||||
|
||||
```sh
|
||||
# Was main broken at or near this commit?
|
||||
pnpm run diagnose:upstream-breakage at <commit> --window-hours 1
|
||||
|
||||
# Are these red branches red for the same reason?
|
||||
pnpm run diagnose:upstream-breakage compare 17330 17331 17332
|
||||
```
|
||||
|
||||
Both finish in a few seconds. Both take `--json`.
|
||||
|
||||
`at` resolves the commit, finds the PRs whose checks completed in the surrounding
|
||||
window, and reports each check's behaviour over time. `compare` does the same for
|
||||
a list of PRs you name.
|
||||
|
||||
## How it decides
|
||||
|
||||
Every PR on which **a lane that actually executes the tree** ran in the window is
|
||||
a **witness**. That qualification is the whole load-bearing part: a path-filtered
|
||||
PR — docs-only, mobile-only — skips every lane in `pr.yml` and still posts three
|
||||
green jobs (the path classifier, the root-directory guard and the LoC counter).
|
||||
Those stay green on a `main` that is entirely broken, so a PR carrying only them
|
||||
witnessed nothing and the probe refuses to count it. The list of qualifying lanes
|
||||
is an allowlist in `upstream-breakage-evidence.mjs`: a name it does not recognise
|
||||
costs a witness and pushes the verdict toward `unknown`, rather than buying one
|
||||
and pushing it toward `clean`. Every unrecognised name is printed on each run.
|
||||
|
||||
For each check name the probe then looks at when it was red and when it was green:
|
||||
|
||||
| Shape | Meaning |
|
||||
| --------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| Red in every witness that ran it | `main` was broken for the whole window |
|
||||
| Red across one unbroken stretch of time, green outside it | `main` broke at the start of that stretch, and was fixed at the end if greens follow |
|
||||
| Reds and greens interleave in time | branch-specific — the tree was fine between the reds |
|
||||
| Red in fewer than two independent stacks | not attributable either way |
|
||||
|
||||
Two PRs in the same stack share a diff, so they count as one witness, not two.
|
||||
The probe reconstructs stacks from branch parentage: a PR whose base branch is
|
||||
another PR's head branch is stacked on it, **and PRs that share one non-trunk base
|
||||
branch are one stack even when that branch has no PR of its own** — otherwise two
|
||||
children of an unlisted parent corroborate each other, which is one witness
|
||||
counted twice.
|
||||
|
||||
The output for the real incident looks like this — the exact window, named:
|
||||
|
||||
```
|
||||
test / tests node 24 2/8 — red 04:09Z..04:23Z (broke after 04:05Z; fixed by 04:39Z)
|
||||
```
|
||||
|
||||
## It says unknown, and unknown is not green
|
||||
|
||||
The probe answers `broken`, `clean`, or `unknown`, and it will not say `clean`
|
||||
without positive evidence. It answers `unknown` when:
|
||||
|
||||
- fewer than two PRs ran a lane that exercises the tree — including the case where
|
||||
every PR in the window was path-filtered down to its meta jobs;
|
||||
- the witnesses are all in one stack, so they corroborate nothing;
|
||||
- the witness checks are spread over more than a day, so they ran against
|
||||
different `main`s and their agreement means nothing;
|
||||
- the commit sits in a blind gap — between the last green and the first red — so
|
||||
no witness observed `main` there;
|
||||
- a failing check was seen too narrowly to attribute.
|
||||
|
||||
This is the same discipline as the `live` / `unverifiable` / `exited` vocabulary
|
||||
in [`ssh-execution-boundary.md`](./ssh-execution-boundary.md). A health check
|
||||
that guesses green is exactly the failure it is meant to catch.
|
||||
|
||||
## What it deliberately does not count
|
||||
|
||||
Printed on every run, never dropped silently:
|
||||
|
||||
- **`verify`** is a roll-up job that reprints another job's failure. Counting it
|
||||
double-counts one breakage and makes divergent failure sets look identical.
|
||||
- **Known-false reds** — `test / tests node 24 1/8`, `test / tests node 24 6/8`,
|
||||
and `e2e / ssh docker watcher isolation` — are red for reasons unrelated to the
|
||||
commit. Counting them makes every window look broken. `--include-known-false`
|
||||
overrides this.
|
||||
- **Third-party app checks** (review bots) say nothing about the tree.
|
||||
- **Skipped checks** are not passes, and neither are the always-on jobs that never
|
||||
execute the tree — `detect code-relevant changes`, `root directory guard`,
|
||||
`test vs non-test LoC`, `track-community-pr`, and the e2e spec-list classifiers.
|
||||
A PR left with only those is not a witness at all, so a window made up of
|
||||
path-filtered PRs reads `unknown`, not `clean`.
|
||||
- **`verify`** is also the job name of the mobile workflow's only lane, so a
|
||||
mobile-only PR loses that check to the roll-up exclusion as well. It is not a
|
||||
witness for `main`'s desktop health either way.
|
||||
|
||||
## Traps to keep in mind
|
||||
|
||||
- `gh pr list` and `gh run list` **truncate silently** — the default is 30 rows.
|
||||
The probe always passes an explicit `--limit`, prints the count it scanned, and
|
||||
warns when the result came back exactly at the limit. If you query by hand, do
|
||||
the same, and print the count beside any conclusion.
|
||||
- A run's `conclusion` **hides failed first attempts**: a run re-run to green
|
||||
reads `success`. Pass `--all-attempts` when attempt-level truth matters.
|
||||
- Read `main` and the workflow definitions from `origin/main`, not from a
|
||||
worktree. A checkout fourteen commits behind gives a confidently wrong answer.
|
||||
- The probe cannot see which base SHA each PR's CI actually merged against, only
|
||||
when it ran. Two checks minutes apart can still straddle a merge. That is why
|
||||
a narrow `--window-hours` sharpens the answer and why blind gaps read `unknown`.
|
||||
|
||||
## When it says upstream
|
||||
|
||||
Merge a newer `main` into each branch. Do not repair the branches, do not write
|
||||
the failures up as merge damage, and do not rebase the chain — see
|
||||
[`git-compatibility.md`](./git-compatibility.md) for the Git-side constraints on
|
||||
restacking.
|
||||
@@ -20,6 +20,7 @@
|
||||
"check:code-quality:changed": "node config/scripts/check-changed-code-quality.mjs",
|
||||
"check:react-doctor:changed": "node config/scripts/check-react-doctor-changed.mjs",
|
||||
"check:zustand-selector-fanout": "node config/scripts/zustand-selector-fanout-benchmark.mjs --check",
|
||||
"diagnose:upstream-breakage": "node config/scripts/upstream-breakage-probe.mjs",
|
||||
"doctor": "pnpm dlx react-doctor@0.9.1 . --no-telemetry",
|
||||
"lint:react-doctor": "oxlint --config config/oxlint-react-doctor.json",
|
||||
"lint:react-doctor:changed": "node config/scripts/lint-react-doctor-changed.mjs",
|
||||
|
||||
Reference in New Issue
Block a user