diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index d476360b882..6dad79cd502 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -34,6 +34,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + fetch-depth: 2 + + - name: Load base mobile diagnostic baseline + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + git cat-file -e "$BASE_SHA^{commit}" + if git cat-file -e "$BASE_SHA:mobile/typecheck-test-diagnostics.json" 2>/dev/null; then + base_baseline="$RUNNER_TEMP/typecheck-mobile-base.json" + git show "$BASE_SHA:mobile/typecheck-test-diagnostics.json" > "$base_baseline" + echo "TYPECHECK_BASELINE_BASE_PATH=$base_baseline" >> "$GITHUB_ENV" + fi - name: Setup Node.js uses: actions/setup-node@v6 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 338bcda7dbd..f4b9adc5919 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -209,8 +209,21 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: + fetch-depth: 2 persist-credentials: false + - name: Load base E2E diagnostic baseline + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + git cat-file -e "$BASE_SHA^{commit}" + if git cat-file -e "$BASE_SHA:config/typecheck-e2e-diagnostics.json" 2>/dev/null; then + base_baseline="$RUNNER_TEMP/typecheck-e2e-base.json" + git show "$BASE_SHA:config/typecheck-e2e-diagnostics.json" > "$base_baseline" + echo "TYPECHECK_BASELINE_BASE_PATH=$base_baseline" >> "$GITHUB_ENV" + fi + - uses: ./.github/actions/install-node-dependencies # Why: every project is `composite`, so tsc already writes a .tsbuildinfo that lets diff --git a/config/scripts/native-chat-live-session-benchmark.ts b/config/scripts/native-chat-live-session-benchmark.ts index 16f3cb6f228..4fb05c48759 100644 --- a/config/scripts/native-chat-live-session-benchmark.ts +++ b/config/scripts/native-chat-live-session-benchmark.ts @@ -33,12 +33,13 @@ let checksum = 0 let expectedChecksum = 0 let validatedCases = 0 let cappedCalibrations = 0 +let sessionSink: NativeChatSession | null = null let messageArraySink: NativeChatMessage[] | null = null let contentSink = '' let pendingMatchSink: Set | null = null let userRowSink: readonly NativeChatUserRow[] | null = null const benchmarkStartedAt = performance.now() -const getPendingSink = (): Set | null => pendingMatchSink +const getSinks = () => [sessionSink, pendingMatchSink] as const function proseFixture(count: number, bytes: number, withTurnId: boolean): NativeChatMessage[] { const payload = 'Ab Cd '.repeat(Math.ceil(bytes / 7)).slice(0, bytes) @@ -127,6 +128,7 @@ function messageWeight(message: NativeChatMessage, content: string): number { } function consumeSession(session: NativeChatSession, index: number): number { + sessionSink = session messageArraySink = session.messages const message = session.messages[index % session.messages.length] if (!message) { @@ -325,9 +327,10 @@ benchmark( ) strictEqual(checksum, expectedChecksum, 'benchmark checksum accounting drifted') +strictEqual(getSinks()[0]?.sessionId, 'benchmark', 'session outputs did not escape') strictEqual(Array.isArray(messageArraySink), true, 'message arrays did not escape') strictEqual(contentSink.length > 0, true, 'message content did not escape') -strictEqual(getPendingSink() instanceof Set, true, 'pending baseline scan did not escape') +strictEqual(getSinks()[1] instanceof Set, true, 'pending baseline scan did not escape') strictEqual(Array.isArray(userRowSink), true, 'user row scan did not escape') console.log( `validated=${validatedCases} cases, checksum=${checksum}, capped calibrations=${cappedCalibrations}, runtime=${( diff --git a/config/scripts/typecheck-diagnostic-baseline.mjs b/config/scripts/typecheck-diagnostic-baseline.mjs index 024f4a0810f..ae7cc1bcdd3 100644 --- a/config/scripts/typecheck-diagnostic-baseline.mjs +++ b/config/scripts/typecheck-diagnostic-baseline.mjs @@ -21,6 +21,8 @@ function parseArgs(argv) { options.project = next } else if (value === '--baseline') { options.baseline = next + } else if (value === '--base-baseline') { + options.baseBaseline = next } else if (value === '--tsc') { options.tsc = next } else { @@ -29,7 +31,9 @@ function parseArgs(argv) { index += 1 } if (!options.project || !options.baseline) { - throw new Error('Usage: --project --baseline [--tsc ] [--write]') + throw new Error( + 'Usage: --project --baseline [--base-baseline ] [--tsc ] [--write]' + ) } return options } @@ -102,7 +106,11 @@ function collectDiagnostics(projectPath, tscPath) { function diagnosticCounts(diagnostics) { const counts = new Map() for (const diagnostic of diagnostics) { - const key = JSON.stringify(diagnostic) + const key = JSON.stringify({ + file: diagnostic.file, + code: diagnostic.code, + message: diagnostic.message + }) counts.set(key, (counts.get(key) ?? 0) + 1) } return counts @@ -111,7 +119,11 @@ function diagnosticCounts(diagnostics) { function subtractDiagnostics(left, right) { const remaining = diagnosticCounts(right) return left.filter((diagnostic) => { - const key = JSON.stringify(diagnostic) + const key = JSON.stringify({ + file: diagnostic.file, + code: diagnostic.code, + message: diagnostic.message + }) const count = remaining.get(key) ?? 0 if (count === 0) { return true @@ -128,23 +140,51 @@ function formatDiagnostic(diagnostic) { return `${location} TS${diagnostic.code}: ${diagnostic.message}` } +async function readBaseline(baselinePath) { + const baseline = JSON.parse(await readFile(baselinePath, 'utf8')) + if (baseline.version !== 1 || !Array.isArray(baseline.diagnostics)) { + throw new Error(`Invalid diagnostic baseline: ${relative(repoRoot, baselinePath)}`) + } + return baseline +} + +function reportBaselineGrowth(diagnostics, baseDiagnostics) { + const added = subtractDiagnostics(diagnostics, baseDiagnostics) + if (added.length === 0) { + return false + } + console.error(`Diagnostic baseline grew (${added.length}); only removals are allowed:`) + for (const diagnostic of added) { + console.error(formatDiagnostic(diagnostic)) + } + process.exitCode = 1 + return true +} + async function main() { const options = parseArgs(process.argv.slice(2)) const projectPath = resolve(repoRoot, options.project) const baselinePath = resolve(repoRoot, options.baseline) + const baseBaselinePath = options.baseBaseline + ? resolve(repoRoot, options.baseBaseline) + : process.env.TYPECHECK_BASELINE_BASE_PATH const tscPath = resolve(repoRoot, options.tsc ?? 'node_modules/typescript/bin/tsc') const diagnostics = collectDiagnostics(projectPath, tscPath) + const baseBaseline = baseBaselinePath ? await readBaseline(baseBaselinePath) : null if (options.write) { + if (baseBaseline && reportBaselineGrowth(diagnostics, baseBaseline.diagnostics)) { + return + } const baseline = { version: 1, diagnostics } await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`) console.log(`Wrote ${diagnostics.length} diagnostics to ${relative(repoRoot, baselinePath)}`) return } - const baseline = JSON.parse(await readFile(baselinePath, 'utf8')) - if (baseline.version !== 1 || !Array.isArray(baseline.diagnostics)) { - throw new Error(`Invalid diagnostic baseline: ${relative(repoRoot, baselinePath)}`) + const baseline = await readBaseline(baselinePath) + if (baseBaseline && reportBaselineGrowth(baseline.diagnostics, baseBaseline.diagnostics)) { + return } const unexpected = subtractDiagnostics(diagnostics, baseline.diagnostics) const stale = subtractDiagnostics(baseline.diagnostics, diagnostics) diff --git a/config/scripts/typecheck-diagnostic-baseline.test.mjs b/config/scripts/typecheck-diagnostic-baseline.test.mjs index c5aa3be3a97..fc4dc426ce0 100644 --- a/config/scripts/typecheck-diagnostic-baseline.test.mjs +++ b/config/scripts/typecheck-diagnostic-baseline.test.mjs @@ -26,6 +26,7 @@ function createFixture(source) { writeFileSync(join(fixtureDir, 'source.ts'), source) return { baseline: join(fixtureDir, 'baseline.json'), + baseBaseline: join(fixtureDir, 'base-baseline.json'), project: join(fixtureDir, 'tsconfig.json'), source: join(fixtureDir, 'source.ts') } @@ -40,7 +41,17 @@ function runBaseline(paths, extraArgs = []) { } describe('typecheck diagnostic baseline', () => { - it('accepts only the exact recorded diagnostics', () => { + it('loads PR-base baselines in desktop and mobile CI', () => { + const desktopWorkflow = readFileSync(join(repoRoot, '.github/workflows/pr.yml'), 'utf8') + const mobileWorkflow = readFileSync(join(repoRoot, '.github/workflows/mobile.yml'), 'utf8') + + expect(desktopWorkflow).toContain('$BASE_SHA:config/typecheck-e2e-diagnostics.json') + expect(mobileWorkflow).toContain('$BASE_SHA:mobile/typecheck-test-diagnostics.json') + expect(desktopWorkflow).toContain('TYPECHECK_BASELINE_BASE_PATH=') + expect(mobileWorkflow).toContain('TYPECHECK_BASELINE_BASE_PATH=') + }) + + it('requires recorded diagnostics to remain present', () => { const paths = createFixture("const value: number = 'bad'\n") expect(runBaseline(paths, ['--write']).status).toBe(0) expect(JSON.parse(readFileSync(paths.baseline, 'utf8')).diagnostics).toHaveLength(1) @@ -61,4 +72,34 @@ describe('typecheck diagnostic baseline', () => { expect(unexpected.status).toBe(1) expect(unexpected.stderr).toContain('Unexpected diagnostics (1)') }) + + it('rejects another occurrence of an allowlisted diagnostic', () => { + const paths = createFixture("const first: number = 'bad'\n") + expect(runBaseline(paths, ['--write']).status).toBe(0) + + writeFileSync(paths.source, "const first: number = 'bad'\nconst second: number = 'also-bad'\n") + const unexpected = runBaseline(paths) + expect(unexpected.status).toBe(1) + expect(unexpected.stderr).toContain('Unexpected diagnostics (1)') + }) + + it('does not drift when only diagnostic positions move', () => { + const paths = createFixture("const value: number = 'bad'\n") + expect(runBaseline(paths, ['--write']).status).toBe(0) + + writeFileSync(paths.source, "\nconst value: number = 'bad'\n") + expect(runBaseline(paths).status).toBe(0) + }) + + it('refuses to regenerate a baseline that grows from the PR base', () => { + const paths = createFixture("const first: number = 'bad'\n") + expect(runBaseline(paths, ['--write']).status).toBe(0) + writeFileSync(paths.baseBaseline, readFileSync(paths.baseline, 'utf8')) + + writeFileSync(paths.source, "const first: number = 'bad'\nconst second: boolean = 'also-bad'\n") + const growth = runBaseline(paths, ['--write', '--base-baseline', paths.baseBaseline]) + expect(growth.status).toBe(1) + expect(growth.stderr).toContain('Diagnostic baseline grew (1)') + expect(JSON.parse(readFileSync(paths.baseline, 'utf8')).diagnostics).toHaveLength(1) + }) }) diff --git a/config/tsconfig.e2e.json b/config/tsconfig.e2e.json index 32df2cee220..7c2535b8338 100644 --- a/config/tsconfig.e2e.json +++ b/config/tsconfig.e2e.json @@ -1,6 +1,6 @@ { - // Enforced by `pnpm typecheck`; the exact pre-existing diagnostics are ratcheted in - // typecheck-e2e-diagnostics.json so new errors fail while the backlog shrinks. + // Enforced by `pnpm typecheck`; pre-existing diagnostic identities and counts are ratcheted + // in typecheck-e2e-diagnostics.json so new errors fail while the backlog shrinks. "extends": "@electron-toolkit/tsconfig/tsconfig.node.json", // Why: specs import renderer and main modules directly, so the ambient declarations those // modules rely on (vite client types, build-time defines) have to be in the program too.