diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4888dd6f3b3..1b17bcec075 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -273,6 +273,9 @@ jobs: - name: Smoke packaged CLI run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/linux-unpacked + - name: Smoke packaged hang watchdog worker + run: xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked + # Why: regression specs under tests/e2e/** used to merge green without ever # running — e2e.yml only fired on schedule/release (#10518). Path-filter so # ordinary PRs stay light; any E2E suite change still gets a full shard run. diff --git a/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs b/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs new file mode 100644 index 00000000000..b084f986eb3 --- /dev/null +++ b/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs @@ -0,0 +1,23 @@ +import { readFileSync } from 'node:fs' +import { parse } from 'yaml' +import { describe, expect, it } from 'vitest' + +describe('packaged hang watchdog worker contract', () => { + it('boots the worker from app.asar in PR checks', () => { + const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8')) + const smokeSource = readFileSync( + 'config/scripts/smoke-packaged-hang-watchdog-worker.mjs', + 'utf8' + ) + const smokeStep = workflow.jobs.package.steps.find( + (step) => step.name === 'Smoke packaged hang watchdog worker' + ) + + expect(smokeStep.run).toBe( + 'xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked' + ) + expect(smokeSource).toContain( + "process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir]" + ) + }) +}) diff --git a/config/scripts/smoke-packaged-hang-watchdog-worker.mjs b/config/scripts/smoke-packaged-hang-watchdog-worker.mjs new file mode 100644 index 00000000000..e4b7af33ab9 --- /dev/null +++ b/config/scripts/smoke-packaged-hang-watchdog-worker.mjs @@ -0,0 +1,165 @@ +import { spawnSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { Worker } from 'node:worker_threads' + +const INTERNAL_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_INTERNAL' +const ASAR_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_ASAR' +const TIMEOUT_MS = 100 +const CHECK_INTERVAL_MS = 20 +const POLL_TIMEOUT_MS = 5_000 + +function sleep(ms) { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)) +} + +function readAppDirArg(argv) { + const explicit = argv.find((arg) => arg.startsWith('--app-dir=')) + if (explicit) { + return explicit.slice('--app-dir='.length) + } + if (process.platform === 'darwin') { + return 'dist/mac-arm64/Orca.app' + } + if (process.platform === 'win32') { + return 'dist/win-unpacked' + } + return 'dist/linux-unpacked' +} + +function getResourcesDir(appDir) { + return process.platform === 'darwin' || appDir.endsWith('.app') + ? join(appDir, 'Contents', 'Resources') + : join(appDir, 'resources') +} + +function readMarker(markerPath) { + try { + return JSON.parse(readFileSync(markerPath, 'utf8')) + } catch { + return null + } +} + +async function waitForMarker(markerPath, predicate, workerError) { + const deadline = Date.now() + POLL_TIMEOUT_MS + while (Date.now() < deadline) { + if (workerError.current) { + throw workerError.current + } + const marker = readMarker(markerPath) + if (predicate(marker)) { + return marker + } + await sleep(CHECK_INTERVAL_MS) + } + throw new Error(`Timed out waiting for packaged watchdog marker at ${markerPath}`) +} + +async function waitForExit(worker, workerError) { + const exitCode = await Promise.race([ + new Promise((resolveExit) => worker.once('exit', resolveExit)), + sleep(POLL_TIMEOUT_MS).then(() => { + throw new Error('Timed out waiting for packaged watchdog worker to exit') + }) + ]) + if (workerError.current) { + throw workerError.current + } + if (exitCode !== 0) { + throw new Error(`Packaged watchdog worker exited with code ${exitCode}`) + } +} + +async function runInternal() { + const appAsar = process.env[ASAR_ENV] + if (!appAsar) { + throw new Error(`Missing ${ASAR_ENV}`) + } + const { app } = await import('electron') + const tempRoot = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-smoke-')) + const markerPath = join(tempRoot, 'main-thread-hang.json') + const entryPath = join(appAsar, 'out', 'main', 'main-thread-hang-watchdog-entry.js') + let worker + try { + await app.whenReady() + if (!existsSync(entryPath)) { + throw new Error(`Packaged watchdog entry is missing from app.asar: ${entryPath}`) + } + const workerError = { current: null } + worker = new Worker(entryPath, { + workerData: { + parentPid: process.pid, + markerPath, + timeoutMs: TIMEOUT_MS, + checkIntervalMs: CHECK_INTERVAL_MS + } + }) + worker.once('error', (error) => { + workerError.current = error + }) + await waitForMarker(markerPath, (marker) => marker?.selfRecovered === false, workerError) + worker.postMessage({ type: 'heartbeat' }) + await waitForMarker(markerPath, (marker) => marker?.selfRecovered === true, workerError) + worker.postMessage({ type: 'shutdown' }) + await waitForExit(worker, workerError) + worker = undefined + console.log('[packaged-watchdog-smoke] app.asar worker detected and recovered a stall') + } finally { + await worker?.terminate() + app.quit() + rmSync(tempRoot, { recursive: true, force: true }) + } +} + +function runSmoke() { + const appDir = resolve(readAppDirArg(process.argv.slice(2))) + const appAsar = join(getResourcesDir(appDir), 'app.asar') + if (!existsSync(appAsar)) { + throw new Error(`Packaged app archive is missing: ${appAsar}`) + } + const require = createRequire(import.meta.url) + const executable = require('electron') + const launcherDir = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-launcher-')) + const launcherPath = join(launcherDir, 'main.cjs') + writeFileSync( + join(launcherDir, 'package.json'), + JSON.stringify({ name: 'orca-packaged-watchdog-smoke', main: 'main.cjs' }) + ) + writeFileSync( + launcherPath, + `import(${JSON.stringify(pathToFileURL(import.meta.filename).href)}).catch((error) => { + console.error(error) + process.exitCode = 1 +})\n` + ) + const env = { ...process.env, [INTERNAL_ENV]: '1', [ASAR_ENV]: appAsar, NODE_PATH: '' } + delete env.ELECTRON_RUN_AS_NODE + try { + const electronArgs = + process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir] + const result = spawnSync(executable, electronArgs, { + env, + encoding: 'utf8', + timeout: 15_000 + }) + if (result.status !== 0) { + throw new Error( + `Packaged watchdog smoke failed (${result.error?.message ?? result.signal ?? result.status}):\n` + + `${result.stderr || result.stdout}` + ) + } + process.stdout.write(result.stdout) + } finally { + rmSync(launcherDir, { recursive: true, force: true }) + } +} + +if (process.env[INTERNAL_ENV] === '1') { + await runInternal() +} else { + runSmoke() +} diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts index 5f4d75b4ad2..a1ececdadb4 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts @@ -12,7 +12,11 @@ vi.mock('node:child_process', () => ({ spawn: spawnMock })) -import { recordHangObservation } from './main-thread-hang-watchdog-entry' +import { + isHangWatchdogWorkerData, + recordHangObservation, + runWatchdog +} from './main-thread-hang-watchdog-entry' describe('recordHangObservation', () => { let dir: string @@ -102,3 +106,75 @@ describe('recordHangObservation', () => { ).not.toThrow() }) }) + +describe('watchdog worker entry', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('accepts only complete positive worker data', () => { + expect( + isHangWatchdogWorkerData({ + parentPid: 42, + markerPath: '/tmp/marker', + timeoutMs: 100, + checkIntervalMs: 25 + }) + ).toBe(true) + for (const value of [ + null, + {}, + { parentPid: 0, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: 7, timeoutMs: 100, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 0, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: Infinity } + ]) { + expect(isHangWatchdogWorkerData(value)).toBe(false) + } + }) + + it('routes heartbeats and shuts down its timer and port', () => { + const markerPath = join(tmpdir(), `hang-watchdog-entry-${process.pid}.json`) + let onMessage: ((message: { type: 'heartbeat' | 'shutdown' }) => void) | undefined + const port = { + on: vi.fn( + (_event: 'message', listener: (message: { type: 'heartbeat' | 'shutdown' }) => void) => { + onMessage = listener + } + ), + close: vi.fn() + } + try { + runWatchdog( + { + parentPid: process.pid, + markerPath, + timeoutMs: 100, + checkIntervalMs: 25 + }, + port + ) + vi.advanceTimersByTime(75) + onMessage?.({ type: 'heartbeat' }) + vi.advanceTimersByTime(75) + expect(consumeHangDetectionMarker(markerPath)).toBeNull() + vi.advanceTimersByTime(50) + expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: false }) + + onMessage?.({ type: 'heartbeat' }) + expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: true }) + onMessage?.({ type: 'shutdown' }) + expect(port.close).toHaveBeenCalledOnce() + vi.advanceTimersByTime(1_000) + expect(port.close).toHaveBeenCalledOnce() + expect(consumeHangDetectionMarker(markerPath)).toBeNull() + } finally { + rmSync(markerPath, { force: true }) + } + }) +}) diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts index 1b100b73566..2c60b17e9ff 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts @@ -6,6 +6,11 @@ import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol' +type HangWatchdogPort = { + on: (event: 'message', listener: (message: MainToHangWatchdogWorkerMessage) => void) => unknown + close: () => void +} + // Observation only: a false positive must never kill a live main thread mid-write. export function recordHangObservation(options: { parentPid: number @@ -28,8 +33,10 @@ export function recordHangObservation(options: { } } -function runWatchdog(config: HangWatchdogWorkerData): void { - const port = parentPort +export function runWatchdog( + config: HangWatchdogWorkerData, + port: HangWatchdogPort | null = parentPort +): void { if (!port) { return } @@ -71,7 +78,7 @@ function runWatchdog(config: HangWatchdogWorkerData): void { }) } -function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData { +export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData { const data = value as Partial | null return ( !!data &&