mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
ci: ratchet tests/ type errors so a broken spec import can't land unnoticed (#30286)
tests/ is typechecked by nothing in CI, so the Windows CLI matrix spec imported a helper that never existed (#30231). Per-file error counts under tests/ (tsc -p config/tsconfig.e2e.json) may now only go down.
This commit is contained in:
Neil
@@ -404,6 +404,10 @@ jobs:
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: node config/scripts/check-worktree-id-composer-ratchet.mjs
|
||||
|
||||
- name: Enforce e2e typecheck ratchet
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: node config/scripts/check-e2e-typecheck-ratchet.mjs
|
||||
|
||||
- name: Check Node runtime pin
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: node config/scripts/check-node-runtime-pin.mjs
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json).
|
||||
# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch.
|
||||
# Prune after a fix: pnpm check:e2e-typecheck-ratchet --prune
|
||||
2 tests/e2e/__mocks__/native-chat-ssh-delivery.ts
|
||||
1 tests/e2e/agent-dashboard-status-burst.spec.ts
|
||||
1 tests/e2e/agent-session-live-force-exit-resume.spec.ts
|
||||
1 tests/e2e/ai-vault-typing-bench-renderer-probe.ts
|
||||
1 tests/e2e/app-menu-paste-ownership.spec.ts
|
||||
1 tests/e2e/artificial-opencode-main-pressure-scenario.ts
|
||||
1 tests/e2e/automation-hidden-terminal-first-mount.spec.ts
|
||||
1 tests/e2e/browser-floating-shortcut-scope.spec.ts
|
||||
4 tests/e2e/browser-tab.spec.ts
|
||||
3 tests/e2e/chinese-ime-chat-input-repro.spec.ts
|
||||
1 tests/e2e/codex-account-switch-dismiss.spec.ts
|
||||
3 tests/e2e/codex-child-approval-activity-row.unit.test.ts
|
||||
1 tests/e2e/codex-ctrl-c-status.spec.ts
|
||||
2 tests/e2e/combined-diff-scroll-restore.spec.ts
|
||||
2 tests/e2e/completed-worker-retirement-resume.spec.ts
|
||||
1 tests/e2e/completed-worker-retirement-resume.unit.test.ts
|
||||
4 tests/e2e/cross-version-wire/agent-session-handle-replacement-downgrade.unit.test.ts
|
||||
1 tests/e2e/cross-version-wire/agent-session-interrupted-tool-downgrade.unit.test.ts
|
||||
2 tests/e2e/cross-version-wire/agent-session-newer-chat-refusal.unit.test.ts
|
||||
1 tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts
|
||||
3 tests/e2e/cross-version-wire/cross-version-stop-note-test-rig.ts
|
||||
3 tests/e2e/cross-version-wire/cross-version-web-device-key.unit.test.ts
|
||||
2 tests/e2e/cross-version-wire/session-search-agent-set-downgrade.unit.test.ts
|
||||
1 tests/e2e/cross-version-wire/structured-agent-session-host-fixture.ts
|
||||
1 tests/e2e/cross-version-wire/submission-positions-downgrade.unit.test.ts
|
||||
1 tests/e2e/cross-version-wire/versioned-agent-session-wire.ts
|
||||
1 tests/e2e/daemon-generation-legacy-close-safety.spec.ts
|
||||
1 tests/e2e/daemon-generation-reconnect-safety.spec.ts
|
||||
1 tests/e2e/editable-context-paste-ownership.spec.ts
|
||||
1 tests/e2e/file-explorer-linked-watch-refresh.spec.ts
|
||||
5 tests/e2e/fixtures/daemon-generation-legacy-close-client.ts
|
||||
2 tests/e2e/fixtures/daemon-generation-reconnect-client.ts
|
||||
1 tests/e2e/fixtures/daemon-lifecycle-entry.ts
|
||||
2 tests/e2e/github-cli-stall-repro.spec.ts
|
||||
1 tests/e2e/github-created-issue-start-prefill.spec.ts
|
||||
1 tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts
|
||||
1 tests/e2e/headless-serve-desktop-activation.spec.ts
|
||||
1 tests/e2e/helpers/daemon-generation-safety-fixtures.ts
|
||||
5 tests/e2e/helpers/headless-paired-runtime-host.ts
|
||||
1 tests/e2e/helpers/native-chat-view.ts
|
||||
2 tests/e2e/helpers/nested-runtime-ssh-client-route.ts
|
||||
1 tests/e2e/helpers/orcad-managed-workspace.ts
|
||||
2 tests/e2e/helpers/paired-terminal-cold-activation-oracle.ts
|
||||
2 tests/e2e/helpers/shortcuts.ts
|
||||
1 tests/e2e/host-agent-launch-settings-hook-bookkeeping.unit.test.ts
|
||||
4 tests/e2e/issue-12656-terminal-link-tooltip.spec.ts
|
||||
1 tests/e2e/korean-ime-terminal-shift-enter-commit.spec.ts
|
||||
1 tests/e2e/large-diff-freeze-repro.spec.ts
|
||||
1 tests/e2e/linear-issue-view-persistence.spec.ts
|
||||
2 tests/e2e/live-background-terminal-mount-authority.spec.ts
|
||||
1 tests/e2e/local-worktree-visibility-runtime-active.spec.ts
|
||||
1 tests/e2e/markdown-tab-scroll-restore.spec.ts
|
||||
2 tests/e2e/multi-client-navigation-isolation.spec.ts
|
||||
2 tests/e2e/native-chat-history-prepend-anchor.spec.ts
|
||||
2 tests/e2e/native-chat-startup-option-pickers.unit.test.ts
|
||||
1 tests/e2e/nested-agent-idle-session-owner.spec.ts
|
||||
2 tests/e2e/omp-model-picker.spec.ts
|
||||
2 tests/e2e/orchestration-legacy-worker-missing-terminal-recovery.spec.ts
|
||||
1 tests/e2e/orchestration-worker-settlement-release-cli.spec.ts
|
||||
1 tests/e2e/orchestration-worker-terminal-visibility.spec.ts
|
||||
1 tests/e2e/orchestration-worker-transcript-providers.spec.ts
|
||||
6 tests/e2e/paired-quick-open-large-tree.spec.ts
|
||||
3 tests/e2e/paired-remote-pane-layout-retry.spec.ts
|
||||
3 tests/e2e/paired-remote-terminal-browser-link.spec.ts
|
||||
1 tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts
|
||||
1 tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts
|
||||
4 tests/e2e/paired-remote-terminal-parked-scrollback-survives.spec.ts
|
||||
3 tests/e2e/paired-remote-terminal-stall-recovery.spec.ts
|
||||
1 tests/e2e/paired-startup-exec-readiness.spec.ts
|
||||
1 tests/e2e/plugin-marketplace-content.spec.ts
|
||||
2 tests/e2e/pr11346-selected-runtime-identity-oracle.ts
|
||||
5 tests/e2e/project-group-manual-sort.spec.ts
|
||||
1 tests/e2e/relay-region-correction.unit.test.ts
|
||||
2 tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts
|
||||
2 tests/e2e/remote-terminal-input-delivery-model.unit.test.ts
|
||||
4 tests/e2e/remote-terminal-input-model-link.ts
|
||||
1 tests/e2e/remote-terminal-tab-retirement.unit.test.ts
|
||||
2 tests/e2e/repo-registration-async.spec.ts
|
||||
1 tests/e2e/restored-terminal-input-readiness.unit.test.ts
|
||||
5 tests/e2e/setup-guide-sidebar.spec.ts
|
||||
1 tests/e2e/source-control-create-pr-intent-switch.spec.ts
|
||||
2 tests/e2e/source-control-create-pr.spec.ts
|
||||
4 tests/e2e/ssh-orcad-auto-convert.spec.ts
|
||||
2 tests/e2e/ssh-orcad-browser-capabilities.spec.ts
|
||||
2 tests/e2e/ssh-orcad-browser-routing.spec.ts
|
||||
1 tests/e2e/ssh-orcad-browser-service-status.spec.ts
|
||||
2 tests/e2e/ssh-orcad-explorer-selected-host.spec.ts
|
||||
1 tests/e2e/ssh-orcad-legacy-relay-bridge.spec.ts
|
||||
1 tests/e2e/ssh-orcad-terminal-network-outage.spec.ts
|
||||
5 tests/e2e/staging-skill-sharing.spec.ts
|
||||
3 tests/e2e/structured-agent-session-read-owner.unit.test.ts
|
||||
2 tests/e2e/structured-attention-prompt-relay.unit.test.ts
|
||||
1 tests/e2e/structured-attention-read-retirement.test-fixture.tsx
|
||||
1 tests/e2e/structured-attention-read-retirement.unit.test.ts
|
||||
1 tests/e2e/structured-chat-owner-status-activation.unit.test.ts
|
||||
1 tests/e2e/tab-rename-paste-ownership.spec.ts
|
||||
1 tests/e2e/tab-rename.spec.ts
|
||||
3 tests/e2e/tasks-page.spec.ts
|
||||
5 tests/e2e/terminal-account-restart.unit.test.ts
|
||||
1 tests/e2e/terminal-codex-cursor-jitter-repro.spec.ts
|
||||
4 tests/e2e/terminal-codex-hidden-startup-background.spec.ts
|
||||
1 tests/e2e/terminal-codex-local-typing-latency.spec.ts
|
||||
2 tests/e2e/terminal-codex-skill-preview-artifact-repro.spec.ts
|
||||
1 tests/e2e/terminal-foreground-confirmation.unit.test.ts
|
||||
2 tests/e2e/terminal-foreground-redraw-freeze.spec.ts
|
||||
2 tests/e2e/terminal-ime-preedit-cell-grid.spec.ts
|
||||
1 tests/e2e/terminal-long-table-scroll-restore.spec.ts
|
||||
1 tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts
|
||||
4 tests/e2e/terminal-osc8-cold-park-restore.spec.ts
|
||||
2 tests/e2e/terminal-pane-divider-capture-loss.spec.ts
|
||||
3 tests/e2e/terminal-raw-emoji-table-scroll-restore.spec.ts
|
||||
2 tests/e2e/terminal-reveal-paused-render-repro.spec.ts
|
||||
2 tests/e2e/terminal-search-results.spec.ts
|
||||
2 tests/e2e/terminal-shortcuts.spec.ts
|
||||
1 tests/e2e/terminal-sleep-wake-restore.spec.ts
|
||||
1 tests/e2e/terminal-tab-switch-visual-restore.spec.ts
|
||||
1 tests/e2e/terminal-tui-wheel-drain.spec.ts
|
||||
2 tests/e2e/terminal-tui-wheel-reports.spec.ts
|
||||
1 tests/e2e/terminal-warm-reattach-split-live-output.spec.ts
|
||||
1 tests/e2e/workspace-space-git-status.spec.ts
|
||||
1 tests/e2e/worktree-parent-reorder.spec.ts
|
||||
1 tests/e2e/worktree-scroll-to-current.spec.ts
|
||||
1 tests/tools/omp-child-history-rendered/fixture.tsx
|
||||
1 tests/tools/omp-fresh-launch-windows-smoke.ts
|
||||
1 tests/tools/omp-remote-sidebar-rendered/fixture.tsx
|
||||
1 tests/tools/omp-transcript-rendered/fixture.tsx
|
||||
@@ -0,0 +1,87 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
diffCounts,
|
||||
parseCountBaseline,
|
||||
pruneCountBaseline,
|
||||
totalCount
|
||||
} from './ratchet-baseline.mjs'
|
||||
|
||||
// Ratchet on type errors under tests/. tests/ has never been typechecked, so a spec can import a
|
||||
// helper that doesn't exist (convertThenRetire, #30231) and nothing notices until a host-gated
|
||||
// lane runs it. Counts per file may only go down.
|
||||
|
||||
const BASELINE_PATH = 'config/e2e-typecheck-baseline.txt'
|
||||
const TSCONFIG = 'config/tsconfig.e2e.json'
|
||||
const HEADER = [
|
||||
'# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json).',
|
||||
'# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch.',
|
||||
'# Prune after a fix: pnpm check:e2e-typecheck-ratchet --prune'
|
||||
]
|
||||
|
||||
// Only tests/ rows: errors tsc reports in imported cloud/ or mobile/ sources depend on whether
|
||||
// those workspaces are installed, which differs between CI jobs.
|
||||
const TEST_ERROR_ROW = /^(tests\/[^(\s]+)\(\d+,\d+\): error TS\d+:/
|
||||
|
||||
export function countTestTypeErrors(tscOutput) {
|
||||
const counts = new Map()
|
||||
for (const line of tscOutput.split(/\r?\n/)) {
|
||||
const match = TEST_ERROR_ROW.exec(line)
|
||||
if (match) {
|
||||
counts.set(match[1], (counts.get(match[1]) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
function runTsc(root) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
['node_modules/typescript/bin/tsc', '--noEmit', '--pretty', 'false', '-p', TSCONFIG],
|
||||
{ cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }
|
||||
)
|
||||
const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
|
||||
// tsc exits 2 when it reports errors; anything else without diagnostics means it never ran.
|
||||
if (result.error || (result.status !== 0 && !/error TS\d+:/.test(output))) {
|
||||
throw new Error(`tsc -p ${TSCONFIG} failed to run:\n${result.error ?? output}`)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
export function main(root = process.cwd(), arg = undefined, tscOutput = runTsc(root)) {
|
||||
const current = countTestTypeErrors(tscOutput)
|
||||
if (arg === '--prune') {
|
||||
const total = pruneCountBaseline(root, BASELINE_PATH, HEADER, current)
|
||||
console.log(`e2e-typecheck baseline pruned to ${total}.`)
|
||||
return 0
|
||||
}
|
||||
const baselineFile = path.join(root, BASELINE_PATH)
|
||||
const baseline = parseCountBaseline(
|
||||
fs.existsSync(baselineFile) ? fs.readFileSync(baselineFile, 'utf8') : ''
|
||||
)
|
||||
const { grown, shrunk } = diffCounts(current, baseline)
|
||||
for (const { file, now, allowed } of grown) {
|
||||
console.error(`${file}: ${now} type error(s), baseline allows ${allowed}.`)
|
||||
}
|
||||
if (grown.length > 0) {
|
||||
console.error(`See the errors with: pnpm typecheck:e2e`)
|
||||
}
|
||||
for (const { file, now, allowed } of shrunk) {
|
||||
console.error(`${file}: now ${now}, baseline ${allowed}.`)
|
||||
}
|
||||
if (shrunk.length > 0) {
|
||||
console.error('The baseline may only shrink. Fix it: pnpm check:e2e-typecheck-ratchet --prune')
|
||||
}
|
||||
if (grown.length + shrunk.length > 0) {
|
||||
return 1
|
||||
}
|
||||
console.log(`e2e-typecheck ratchet OK — ${totalCount(current)} grandfathered error(s).`)
|
||||
return 0
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
process.exit(main(process.cwd(), process.argv[2]))
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { countTestTypeErrors, main } from './check-e2e-typecheck-ratchet.mjs'
|
||||
|
||||
const OUTPUT = [
|
||||
"tests/e2e/a.spec.ts(22,10): error TS2305: Module './helpers/flow' has no exported member 'convertThenRetire'.",
|
||||
"tests/e2e/a.spec.ts(30,1): error TS2339: Property 'x' does not exist on type 'Y'.",
|
||||
"tests/e2e/helpers/b.ts(5,3): error TS2322: Type 'string' is not assignable to type 'number'.",
|
||||
"cloud/apps/relay/src/app.ts(18,8): error TS2307: Cannot find module '@orca-cloud/relay-contract'.",
|
||||
' Continuation line of a multi-line diagnostic.'
|
||||
].join('\n')
|
||||
|
||||
describe('countTestTypeErrors', () => {
|
||||
it('counts tests/ errors per file and ignores rows from other workspaces', () => {
|
||||
expect(countTestTypeErrors(OUTPUT)).toEqual(
|
||||
new Map([
|
||||
['tests/e2e/a.spec.ts', 2],
|
||||
['tests/e2e/helpers/b.ts', 1]
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('reads CRLF output', () => {
|
||||
expect(countTestTypeErrors(OUTPUT.replaceAll('\n', '\r\n')).get('tests/e2e/a.spec.ts')).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('main', () => {
|
||||
let root = ''
|
||||
afterEach(() => {
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function withBaseline(text) {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-typecheck-ratchet-'))
|
||||
fs.mkdirSync(path.join(root, 'config'))
|
||||
fs.writeFileSync(path.join(root, 'config/e2e-typecheck-baseline.txt'), text)
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
}
|
||||
|
||||
it('fails when a file gains an error, as a missing helper import does', () => {
|
||||
withBaseline('1 tests/e2e/a.spec.ts\n1 tests/e2e/helpers/b.ts\n')
|
||||
expect(main(root, undefined, OUTPUT)).toBe(1)
|
||||
})
|
||||
|
||||
it('passes at the baseline and fails until a fixed file is pruned', () => {
|
||||
withBaseline('2 tests/e2e/a.spec.ts\n1 tests/e2e/helpers/b.ts\n')
|
||||
expect(main(root, undefined, OUTPUT)).toBe(0)
|
||||
const fixed = OUTPUT.replace(/^tests\/e2e\/helpers\/b\.ts.*$/m, '')
|
||||
expect(main(root, undefined, fixed)).toBe(1)
|
||||
expect(main(root, '--prune', fixed)).toBe(0)
|
||||
expect(main(root, undefined, fixed)).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -246,6 +246,7 @@ it('pins every foreground and background step to its selected phase', () => {
|
||||
['Enforce deprecated runtime-owner importer ratchet', staticPhase],
|
||||
['Enforce path-literal ratchet', staticPhase],
|
||||
['Enforce worktree-id composer ratchet', staticPhase],
|
||||
['Enforce e2e typecheck ratchet', staticPhase],
|
||||
['Check Node runtime pin', staticPhase],
|
||||
['Decide whether the orcad terminal smoke runs', staticPhase],
|
||||
['Verify the generated RPC params catalog', staticPhase],
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{
|
||||
// NOT ENFORCED YET, and deliberately so: tests/ has never been typechecked, so this currently
|
||||
// reports ~198 errors across ~94 files. Wiring it into `typecheck` or CI today would just paint
|
||||
// the whole build red. It earns its place anyway — pointing it at the SSH reconnect spec found
|
||||
// two real bugs a passing suite had hidden (a targetId read off the wrong type, and a required
|
||||
// argument omitted), because that spec had never actually executed. Run `pnpm typecheck:e2e`,
|
||||
// fix a file, repeat; when the count reaches zero, add it to the `typecheck` chain.
|
||||
// Ratcheted, not yet clean: tests/ was never typechecked, so the per-file error counts in
|
||||
// config/e2e-typecheck-baseline.txt may only go down (check-e2e-typecheck-ratchet.mjs). Run
|
||||
// `pnpm typecheck:e2e`, fix a file, prune; when the count reaches zero, add it to `typecheck`.
|
||||
"extends": "@electron-toolkit/tsconfig/tsconfig.node.json",
|
||||
// Why: specs import renderer and main modules directly, so the ambient declarations those
|
||||
// modules rely on (vite client types, build-time defines) have to be in the program too.
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
"check:deprecated-runtime-owner-ratchet": "node config/scripts/check-deprecated-runtime-owner-ratchet.mjs",
|
||||
"check:path-literal-ratchet": "node config/scripts/check-path-literal-ratchet.mjs",
|
||||
"check:worktree-id-composer-ratchet": "node config/scripts/check-worktree-id-composer-ratchet.mjs",
|
||||
"check:e2e-typecheck-ratchet": "node config/scripts/check-e2e-typecheck-ratchet.mjs",
|
||||
"check:runtime-electron-ratchet": "node config/scripts/check-runtime-electron-ratchet.mjs",
|
||||
"check:focus-read-ratchet": "node config/scripts/check-focus-read-ratchet.mjs",
|
||||
"check:layout-writer-ratchets": "node config/scripts/check-layout-writer-ratchets.mjs",
|
||||
|
||||
Reference in New Issue
Block a user