From 10505696a2c3c9882bbccdadce9a9a6cdcb2d53e Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 18 Oct 2026 02:35:04 -0700 Subject: [PATCH] ci: ratchet tests/ type errors so a broken spec import can't land unnoticed (#30286) tests/ is typechecked by nothing in CI, so the Windows CLI matrix spec imported a helper that never existed (#30231). Per-file error counts under tests/ (tsc -p config/tsconfig.e2e.json) may now only go down. --- .github/workflows/pr.yml | 4 + config/e2e-typecheck-baseline.txt | 129 ++++++++++++++++++ .../scripts/check-e2e-typecheck-ratchet.mjs | 87 ++++++++++++ .../check-e2e-typecheck-ratchet.test.mjs | 59 ++++++++ config/scripts/pr-preflight-gates.test.mjs | 1 + config/tsconfig.e2e.json | 9 +- package.json | 1 + 7 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 config/e2e-typecheck-baseline.txt create mode 100644 config/scripts/check-e2e-typecheck-ratchet.mjs create mode 100644 config/scripts/check-e2e-typecheck-ratchet.test.mjs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8f0b532c75a..cbf81160a09 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -404,6 +404,10 @@ jobs: if: needs.code_paths.outputs.static_analysis == 'true' run: node config/scripts/check-worktree-id-composer-ratchet.mjs + - name: Enforce e2e typecheck ratchet + if: needs.code_paths.outputs.static_analysis == 'true' + run: node config/scripts/check-e2e-typecheck-ratchet.mjs + - name: Check Node runtime pin if: needs.code_paths.outputs.static_analysis == 'true' run: node config/scripts/check-node-runtime-pin.mjs diff --git a/config/e2e-typecheck-baseline.txt b/config/e2e-typecheck-baseline.txt new file mode 100644 index 00000000000..c9672106826 --- /dev/null +++ b/config/e2e-typecheck-baseline.txt @@ -0,0 +1,129 @@ +# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json). +# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch. +# Prune after a fix: pnpm check:e2e-typecheck-ratchet --prune +2 tests/e2e/__mocks__/native-chat-ssh-delivery.ts +1 tests/e2e/agent-dashboard-status-burst.spec.ts +1 tests/e2e/agent-session-live-force-exit-resume.spec.ts +1 tests/e2e/ai-vault-typing-bench-renderer-probe.ts +1 tests/e2e/app-menu-paste-ownership.spec.ts +1 tests/e2e/artificial-opencode-main-pressure-scenario.ts +1 tests/e2e/automation-hidden-terminal-first-mount.spec.ts +1 tests/e2e/browser-floating-shortcut-scope.spec.ts +4 tests/e2e/browser-tab.spec.ts +3 tests/e2e/chinese-ime-chat-input-repro.spec.ts +1 tests/e2e/codex-account-switch-dismiss.spec.ts +3 tests/e2e/codex-child-approval-activity-row.unit.test.ts +1 tests/e2e/codex-ctrl-c-status.spec.ts +2 tests/e2e/combined-diff-scroll-restore.spec.ts +2 tests/e2e/completed-worker-retirement-resume.spec.ts +1 tests/e2e/completed-worker-retirement-resume.unit.test.ts +4 tests/e2e/cross-version-wire/agent-session-handle-replacement-downgrade.unit.test.ts +1 tests/e2e/cross-version-wire/agent-session-interrupted-tool-downgrade.unit.test.ts +2 tests/e2e/cross-version-wire/agent-session-newer-chat-refusal.unit.test.ts +1 tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts +3 tests/e2e/cross-version-wire/cross-version-stop-note-test-rig.ts +3 tests/e2e/cross-version-wire/cross-version-web-device-key.unit.test.ts +2 tests/e2e/cross-version-wire/session-search-agent-set-downgrade.unit.test.ts +1 tests/e2e/cross-version-wire/structured-agent-session-host-fixture.ts +1 tests/e2e/cross-version-wire/submission-positions-downgrade.unit.test.ts +1 tests/e2e/cross-version-wire/versioned-agent-session-wire.ts +1 tests/e2e/daemon-generation-legacy-close-safety.spec.ts +1 tests/e2e/daemon-generation-reconnect-safety.spec.ts +1 tests/e2e/editable-context-paste-ownership.spec.ts +1 tests/e2e/file-explorer-linked-watch-refresh.spec.ts +5 tests/e2e/fixtures/daemon-generation-legacy-close-client.ts +2 tests/e2e/fixtures/daemon-generation-reconnect-client.ts +1 tests/e2e/fixtures/daemon-lifecycle-entry.ts +2 tests/e2e/github-cli-stall-repro.spec.ts +1 tests/e2e/github-created-issue-start-prefill.spec.ts +1 tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts +1 tests/e2e/headless-serve-desktop-activation.spec.ts +1 tests/e2e/helpers/daemon-generation-safety-fixtures.ts +5 tests/e2e/helpers/headless-paired-runtime-host.ts +1 tests/e2e/helpers/native-chat-view.ts +2 tests/e2e/helpers/nested-runtime-ssh-client-route.ts +1 tests/e2e/helpers/orcad-managed-workspace.ts +2 tests/e2e/helpers/paired-terminal-cold-activation-oracle.ts +2 tests/e2e/helpers/shortcuts.ts +1 tests/e2e/host-agent-launch-settings-hook-bookkeeping.unit.test.ts +4 tests/e2e/issue-12656-terminal-link-tooltip.spec.ts +1 tests/e2e/korean-ime-terminal-shift-enter-commit.spec.ts +1 tests/e2e/large-diff-freeze-repro.spec.ts +1 tests/e2e/linear-issue-view-persistence.spec.ts +2 tests/e2e/live-background-terminal-mount-authority.spec.ts +1 tests/e2e/local-worktree-visibility-runtime-active.spec.ts +1 tests/e2e/markdown-tab-scroll-restore.spec.ts +2 tests/e2e/multi-client-navigation-isolation.spec.ts +2 tests/e2e/native-chat-history-prepend-anchor.spec.ts +2 tests/e2e/native-chat-startup-option-pickers.unit.test.ts +1 tests/e2e/nested-agent-idle-session-owner.spec.ts +2 tests/e2e/omp-model-picker.spec.ts +2 tests/e2e/orchestration-legacy-worker-missing-terminal-recovery.spec.ts +1 tests/e2e/orchestration-worker-settlement-release-cli.spec.ts +1 tests/e2e/orchestration-worker-terminal-visibility.spec.ts +1 tests/e2e/orchestration-worker-transcript-providers.spec.ts +6 tests/e2e/paired-quick-open-large-tree.spec.ts +3 tests/e2e/paired-remote-pane-layout-retry.spec.ts +3 tests/e2e/paired-remote-terminal-browser-link.spec.ts +1 tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts +1 tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts +4 tests/e2e/paired-remote-terminal-parked-scrollback-survives.spec.ts +3 tests/e2e/paired-remote-terminal-stall-recovery.spec.ts +1 tests/e2e/paired-startup-exec-readiness.spec.ts +1 tests/e2e/plugin-marketplace-content.spec.ts +2 tests/e2e/pr11346-selected-runtime-identity-oracle.ts +5 tests/e2e/project-group-manual-sort.spec.ts +1 tests/e2e/relay-region-correction.unit.test.ts +2 tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts +2 tests/e2e/remote-terminal-input-delivery-model.unit.test.ts +4 tests/e2e/remote-terminal-input-model-link.ts +1 tests/e2e/remote-terminal-tab-retirement.unit.test.ts +2 tests/e2e/repo-registration-async.spec.ts +1 tests/e2e/restored-terminal-input-readiness.unit.test.ts +5 tests/e2e/setup-guide-sidebar.spec.ts +1 tests/e2e/source-control-create-pr-intent-switch.spec.ts +2 tests/e2e/source-control-create-pr.spec.ts +4 tests/e2e/ssh-orcad-auto-convert.spec.ts +2 tests/e2e/ssh-orcad-browser-capabilities.spec.ts +2 tests/e2e/ssh-orcad-browser-routing.spec.ts +1 tests/e2e/ssh-orcad-browser-service-status.spec.ts +2 tests/e2e/ssh-orcad-explorer-selected-host.spec.ts +1 tests/e2e/ssh-orcad-legacy-relay-bridge.spec.ts +1 tests/e2e/ssh-orcad-terminal-network-outage.spec.ts +5 tests/e2e/staging-skill-sharing.spec.ts +3 tests/e2e/structured-agent-session-read-owner.unit.test.ts +2 tests/e2e/structured-attention-prompt-relay.unit.test.ts +1 tests/e2e/structured-attention-read-retirement.test-fixture.tsx +1 tests/e2e/structured-attention-read-retirement.unit.test.ts +1 tests/e2e/structured-chat-owner-status-activation.unit.test.ts +1 tests/e2e/tab-rename-paste-ownership.spec.ts +1 tests/e2e/tab-rename.spec.ts +3 tests/e2e/tasks-page.spec.ts +5 tests/e2e/terminal-account-restart.unit.test.ts +1 tests/e2e/terminal-codex-cursor-jitter-repro.spec.ts +4 tests/e2e/terminal-codex-hidden-startup-background.spec.ts +1 tests/e2e/terminal-codex-local-typing-latency.spec.ts +2 tests/e2e/terminal-codex-skill-preview-artifact-repro.spec.ts +1 tests/e2e/terminal-foreground-confirmation.unit.test.ts +2 tests/e2e/terminal-foreground-redraw-freeze.spec.ts +2 tests/e2e/terminal-ime-preedit-cell-grid.spec.ts +1 tests/e2e/terminal-long-table-scroll-restore.spec.ts +1 tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts +4 tests/e2e/terminal-osc8-cold-park-restore.spec.ts +2 tests/e2e/terminal-pane-divider-capture-loss.spec.ts +3 tests/e2e/terminal-raw-emoji-table-scroll-restore.spec.ts +2 tests/e2e/terminal-reveal-paused-render-repro.spec.ts +2 tests/e2e/terminal-search-results.spec.ts +2 tests/e2e/terminal-shortcuts.spec.ts +1 tests/e2e/terminal-sleep-wake-restore.spec.ts +1 tests/e2e/terminal-tab-switch-visual-restore.spec.ts +1 tests/e2e/terminal-tui-wheel-drain.spec.ts +2 tests/e2e/terminal-tui-wheel-reports.spec.ts +1 tests/e2e/terminal-warm-reattach-split-live-output.spec.ts +1 tests/e2e/workspace-space-git-status.spec.ts +1 tests/e2e/worktree-parent-reorder.spec.ts +1 tests/e2e/worktree-scroll-to-current.spec.ts +1 tests/tools/omp-child-history-rendered/fixture.tsx +1 tests/tools/omp-fresh-launch-windows-smoke.ts +1 tests/tools/omp-remote-sidebar-rendered/fixture.tsx +1 tests/tools/omp-transcript-rendered/fixture.tsx diff --git a/config/scripts/check-e2e-typecheck-ratchet.mjs b/config/scripts/check-e2e-typecheck-ratchet.mjs new file mode 100644 index 00000000000..941b6d99d03 --- /dev/null +++ b/config/scripts/check-e2e-typecheck-ratchet.mjs @@ -0,0 +1,87 @@ +import { spawnSync } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' +import { pathToFileURL } from 'node:url' +import { + diffCounts, + parseCountBaseline, + pruneCountBaseline, + totalCount +} from './ratchet-baseline.mjs' + +// Ratchet on type errors under tests/. tests/ has never been typechecked, so a spec can import a +// helper that doesn't exist (convertThenRetire, #30231) and nothing notices until a host-gated +// lane runs it. Counts per file may only go down. + +const BASELINE_PATH = 'config/e2e-typecheck-baseline.txt' +const TSCONFIG = 'config/tsconfig.e2e.json' +const HEADER = [ + '# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json).', + '# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch.', + '# Prune after a fix: pnpm check:e2e-typecheck-ratchet --prune' +] + +// Only tests/ rows: errors tsc reports in imported cloud/ or mobile/ sources depend on whether +// those workspaces are installed, which differs between CI jobs. +const TEST_ERROR_ROW = /^(tests\/[^(\s]+)\(\d+,\d+\): error TS\d+:/ + +export function countTestTypeErrors(tscOutput) { + const counts = new Map() + for (const line of tscOutput.split(/\r?\n/)) { + const match = TEST_ERROR_ROW.exec(line) + if (match) { + counts.set(match[1], (counts.get(match[1]) ?? 0) + 1) + } + } + return counts +} + +function runTsc(root) { + const result = spawnSync( + process.execPath, + ['node_modules/typescript/bin/tsc', '--noEmit', '--pretty', 'false', '-p', TSCONFIG], + { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 } + ) + const output = `${result.stdout ?? ''}${result.stderr ?? ''}` + // tsc exits 2 when it reports errors; anything else without diagnostics means it never ran. + if (result.error || (result.status !== 0 && !/error TS\d+:/.test(output))) { + throw new Error(`tsc -p ${TSCONFIG} failed to run:\n${result.error ?? output}`) + } + return output +} + +export function main(root = process.cwd(), arg = undefined, tscOutput = runTsc(root)) { + const current = countTestTypeErrors(tscOutput) + if (arg === '--prune') { + const total = pruneCountBaseline(root, BASELINE_PATH, HEADER, current) + console.log(`e2e-typecheck baseline pruned to ${total}.`) + return 0 + } + const baselineFile = path.join(root, BASELINE_PATH) + const baseline = parseCountBaseline( + fs.existsSync(baselineFile) ? fs.readFileSync(baselineFile, 'utf8') : '' + ) + const { grown, shrunk } = diffCounts(current, baseline) + for (const { file, now, allowed } of grown) { + console.error(`${file}: ${now} type error(s), baseline allows ${allowed}.`) + } + if (grown.length > 0) { + console.error(`See the errors with: pnpm typecheck:e2e`) + } + for (const { file, now, allowed } of shrunk) { + console.error(`${file}: now ${now}, baseline ${allowed}.`) + } + if (shrunk.length > 0) { + console.error('The baseline may only shrink. Fix it: pnpm check:e2e-typecheck-ratchet --prune') + } + if (grown.length + shrunk.length > 0) { + return 1 + } + console.log(`e2e-typecheck ratchet OK — ${totalCount(current)} grandfathered error(s).`) + return 0 +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exit(main(process.cwd(), process.argv[2])) +} diff --git a/config/scripts/check-e2e-typecheck-ratchet.test.mjs b/config/scripts/check-e2e-typecheck-ratchet.test.mjs new file mode 100644 index 00000000000..f61be826e5c --- /dev/null +++ b/config/scripts/check-e2e-typecheck-ratchet.test.mjs @@ -0,0 +1,59 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { countTestTypeErrors, main } from './check-e2e-typecheck-ratchet.mjs' + +const OUTPUT = [ + "tests/e2e/a.spec.ts(22,10): error TS2305: Module './helpers/flow' has no exported member 'convertThenRetire'.", + "tests/e2e/a.spec.ts(30,1): error TS2339: Property 'x' does not exist on type 'Y'.", + "tests/e2e/helpers/b.ts(5,3): error TS2322: Type 'string' is not assignable to type 'number'.", + "cloud/apps/relay/src/app.ts(18,8): error TS2307: Cannot find module '@orca-cloud/relay-contract'.", + ' Continuation line of a multi-line diagnostic.' +].join('\n') + +describe('countTestTypeErrors', () => { + it('counts tests/ errors per file and ignores rows from other workspaces', () => { + expect(countTestTypeErrors(OUTPUT)).toEqual( + new Map([ + ['tests/e2e/a.spec.ts', 2], + ['tests/e2e/helpers/b.ts', 1] + ]) + ) + }) + + it('reads CRLF output', () => { + expect(countTestTypeErrors(OUTPUT.replaceAll('\n', '\r\n')).get('tests/e2e/a.spec.ts')).toBe(2) + }) +}) + +describe('main', () => { + let root = '' + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }) + vi.restoreAllMocks() + }) + + function withBaseline(text) { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-typecheck-ratchet-')) + fs.mkdirSync(path.join(root, 'config')) + fs.writeFileSync(path.join(root, 'config/e2e-typecheck-baseline.txt'), text) + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(console, 'log').mockImplementation(() => {}) + } + + it('fails when a file gains an error, as a missing helper import does', () => { + withBaseline('1 tests/e2e/a.spec.ts\n1 tests/e2e/helpers/b.ts\n') + expect(main(root, undefined, OUTPUT)).toBe(1) + }) + + it('passes at the baseline and fails until a fixed file is pruned', () => { + withBaseline('2 tests/e2e/a.spec.ts\n1 tests/e2e/helpers/b.ts\n') + expect(main(root, undefined, OUTPUT)).toBe(0) + const fixed = OUTPUT.replace(/^tests\/e2e\/helpers\/b\.ts.*$/m, '') + expect(main(root, undefined, fixed)).toBe(1) + expect(main(root, '--prune', fixed)).toBe(0) + expect(main(root, undefined, fixed)).toBe(0) + }) +}) diff --git a/config/scripts/pr-preflight-gates.test.mjs b/config/scripts/pr-preflight-gates.test.mjs index 9c0c7a10308..08259181537 100644 --- a/config/scripts/pr-preflight-gates.test.mjs +++ b/config/scripts/pr-preflight-gates.test.mjs @@ -246,6 +246,7 @@ it('pins every foreground and background step to its selected phase', () => { ['Enforce deprecated runtime-owner importer ratchet', staticPhase], ['Enforce path-literal ratchet', staticPhase], ['Enforce worktree-id composer ratchet', staticPhase], + ['Enforce e2e typecheck ratchet', staticPhase], ['Check Node runtime pin', staticPhase], ['Decide whether the orcad terminal smoke runs', staticPhase], ['Verify the generated RPC params catalog', staticPhase], diff --git a/config/tsconfig.e2e.json b/config/tsconfig.e2e.json index 426c7340c9e..cfad2852b92 100644 --- a/config/tsconfig.e2e.json +++ b/config/tsconfig.e2e.json @@ -1,10 +1,7 @@ { - // NOT ENFORCED YET, and deliberately so: tests/ has never been typechecked, so this currently - // reports ~198 errors across ~94 files. Wiring it into `typecheck` or CI today would just paint - // the whole build red. It earns its place anyway — pointing it at the SSH reconnect spec found - // two real bugs a passing suite had hidden (a targetId read off the wrong type, and a required - // argument omitted), because that spec had never actually executed. Run `pnpm typecheck:e2e`, - // fix a file, repeat; when the count reaches zero, add it to the `typecheck` chain. + // Ratcheted, not yet clean: tests/ was never typechecked, so the per-file error counts in + // config/e2e-typecheck-baseline.txt may only go down (check-e2e-typecheck-ratchet.mjs). Run + // `pnpm typecheck:e2e`, fix a file, prune; when the count reaches zero, add it to `typecheck`. "extends": "@electron-toolkit/tsconfig/tsconfig.node.json", // Why: specs import renderer and main modules directly, so the ambient declarations those // modules rely on (vite client types, build-time defines) have to be in the program too. diff --git a/package.json b/package.json index 43b45f434cf..82e1fba9934 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "check:deprecated-runtime-owner-ratchet": "node config/scripts/check-deprecated-runtime-owner-ratchet.mjs", "check:path-literal-ratchet": "node config/scripts/check-path-literal-ratchet.mjs", "check:worktree-id-composer-ratchet": "node config/scripts/check-worktree-id-composer-ratchet.mjs", + "check:e2e-typecheck-ratchet": "node config/scripts/check-e2e-typecheck-ratchet.mjs", "check:runtime-electron-ratchet": "node config/scripts/check-runtime-electron-ratchet.mjs", "check:focus-read-ratchet": "node config/scripts/check-focus-read-ratchet.mjs", "check:layout-writer-ratchets": "node config/scripts/check-layout-writer-ratchets.mjs",