From 20832ef36f9a2560b55b85ea273e26d27ca5f367 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:46:14 -0700 Subject: [PATCH] =?UTF-8?q?ci(daemon):=20fail=20builds,=20packaging,=20and?= =?UTF-8?q?=20CI=20when=20the=20terminal=20daemon=20cannot=20start=20?= =?UTF-8?q?=E2=80=94=20and=20stop=20failing=20silently=20(#7849)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/computer-e2e.yml | 13 ++ config/electron-builder.config.cjs | 15 ++ config/scripts/computer-e2e-workflow.test.mjs | 33 +++ config/scripts/daemon-boot-smoke.mjs | 217 ++++++++++++++++++ .../scripts/verify-packaged-daemon-entry.cjs | 46 ++++ src/main/daemon/daemon-entry.ts | 7 + src/main/daemon/daemon-init.test.ts | 117 ++++++++++ src/main/daemon/daemon-init.ts | 51 +++- src/main/index.ts | 14 +- src/shared/telemetry-events.ts | 9 + 10 files changed, 514 insertions(+), 8 deletions(-) create mode 100644 config/scripts/daemon-boot-smoke.mjs create mode 100644 config/scripts/verify-packaged-daemon-entry.cjs diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 8024d498918..218ff6a6c82 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -10,7 +10,14 @@ on: - 'config/scripts/computer-use-skill-guidance.test.mjs' - 'config/scripts/computer-use-smoke.mjs' - 'config/scripts/computer-use-smoke.test.mjs' + - 'config/scripts/daemon-boot-smoke.mjs' - 'config/scripts/verify-computer-native.mjs' + # Why: the native-smoke job boots the built terminal daemon under plain + # Node, so any change to the daemon bundle graph or the main build must + # re-run it (the v1.4.129-rc.1 daemon outage shipped with green CI). + - 'electron.vite.config.ts' + - 'build-plugins/**' + - 'src/main/daemon/**' - 'native/computer-use-macos/**' - 'native/computer-use-linux/**' - 'native/computer-use-windows/**' @@ -98,6 +105,12 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm build:electron-vite + # Why: boot the BUILT daemon-entry under plain Node the way production + # forks it. v1.4.129-rc.1 shipped a daemon that exited at module load + # (leaked electron require) while every other check passed; this fails + # the PR when the built daemon cannot start on ubuntu-22.04 / windows. + - name: Daemon boot smoke + run: node config/scripts/daemon-boot-smoke.mjs - if: runner.os == 'Linux' env: ORCA_COMPUTER_E2E: '1' diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index a5a8042def5..def7142b166 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -2,6 +2,7 @@ const { chmodSync, existsSync, readdirSync } = require('node:fs') const { execFileSync } = require('node:child_process') const { join, resolve } = require('node:path') const electronBuilderNativeRebuild = require('./scripts/electron-builder-native-rebuild.cjs') +const { verifyPackagedDaemonEntryBoots } = require('./scripts/verify-packaged-daemon-entry.cjs') const { createPackagedRuntimeNodeModuleResources, prunePackagedRuntimeNodeModules, @@ -130,6 +131,20 @@ module.exports = { } prunePackagedRuntimeNodeModules(resourcesDir, context.electronPlatformName, context.arch) verifyPackagedMainRuntimeDeps(resourcesDir) + // Why: boot the packaged daemon-entry under plain Node, but only for the + // slice matching the packaging host's arch — daemon-entry.js is JS, yet it + // require()s the native (N-API) node-pty for the TARGET arch, which the host + // Node cannot load cross-arch. `Arch` enum: ia32=0, x64=1, armv7l=2, + // arm64=3, universal=4 (universal contains the host slice, so run it). + const archEnumByNodeArch = { ia32: 0, x64: 1, armv7l: 2, arm64: 3 } + const hostArchEnum = archEnumByNodeArch[process.arch] + if (context.arch === hostArchEnum || context.arch === 4) { + verifyPackagedDaemonEntryBoots(resourcesDir) + } else { + console.log( + `[verify-packaged-daemon-entry] skipped cross-arch slice (target ${context.arch}, host ${process.arch})` + ) + } chmodUnixCliLaunchers(resourcesDir, context.electronPlatformName) chmodMacServeSimHelpers(resourcesDir, context.electronPlatformName) for (const filename of readdirSync(resourcesDir)) { diff --git a/config/scripts/computer-e2e-workflow.test.mjs b/config/scripts/computer-e2e-workflow.test.mjs index 92480315b18..3d4fbaecddd 100644 --- a/config/scripts/computer-e2e-workflow.test.mjs +++ b/config/scripts/computer-e2e-workflow.test.mjs @@ -110,6 +110,39 @@ describe('computer-use e2e workflow', () => { } }) + it('boots the built daemon under plain Node in the PR native-smoke job after the main build', () => { + const workflow = parse( + readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') + ) + const steps = workflow.jobs['native-smoke'].steps + const runs = steps.map((step) => step.run).filter((run) => typeof run === 'string') + const buildIndex = runs.indexOf('pnpm build:electron-vite') + const daemonSmokeIndex = runs.indexOf('node config/scripts/daemon-boot-smoke.mjs') + + expect(daemonSmokeIndex, 'native-smoke must boot the built daemon').toBeGreaterThanOrEqual(0) + expect( + buildIndex, + 'daemon boot smoke must run after the main bundle is built' + ).toBeGreaterThanOrEqual(0) + expect(daemonSmokeIndex).toBeGreaterThan(buildIndex) + }) + + it('re-runs the native-smoke job when the daemon bundle graph changes', () => { + const workflow = parse( + readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') + ) + const triggerPaths = workflow.on.pull_request.paths + + expect(triggerPaths).toEqual( + expect.arrayContaining([ + 'config/scripts/daemon-boot-smoke.mjs', + 'electron.vite.config.ts', + 'build-plugins/**', + 'src/main/daemon/**' + ]) + ) + }) + it('runs Linux computer-use e2e in the PR native-smoke job under Xvfb', () => { const workflow = parse( readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') diff --git a/config/scripts/daemon-boot-smoke.mjs b/config/scripts/daemon-boot-smoke.mjs new file mode 100644 index 00000000000..dc593b650b2 --- /dev/null +++ b/config/scripts/daemon-boot-smoke.mjs @@ -0,0 +1,217 @@ +/** + * Boots the BUILT terminal daemon (out/main/daemon-entry.js) under plain Node — + * the exact way production forks it (ELECTRON_RUN_AS_NODE = a plain-Node + * process) — and asserts it starts, serves a real PTY, and stops. + * + * Why this exists: native-smoke CI (and packaging) went green while + * v1.4.129-rc.1 shipped a daemon that exited code 1 at module load because an + * electron `require` leaked into its bundle graph. Nothing executed the built + * entry under plain Node, so the outage was invisible until an adopted old + * daemon died in the field. This runs on every PR that touches the daemon. + * + * Hard assertions (fail the job): + * - the daemon signals `{ type: 'ready' }` over IPC within the timeout, and + * - it terminates when asked (no hang / zombie). + * Best-effort (logged skip, never fails): an end-to-end `ptySpawnHealth` RPC, + * because node-pty spawn can be flaky on constrained CI runners. + */ +import { fork } from 'node:child_process' +import { connect } from 'node:net' +import { randomUUID } from 'node:crypto' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const projectDir = resolve(import.meta.dirname, '../..') +const entryPath = join(projectDir, 'out', 'main', 'daemon-entry.js') + +const READY_TIMEOUT_MS = 30_000 +const PTY_HEALTH_TIMEOUT_MS = 10_000 +const SHUTDOWN_TIMEOUT_MS = 10_000 + +function log(message) { + process.stdout.write(`[daemon-boot-smoke] ${message}\n`) +} + +// Why: the daemon rejects a hello whose protocol version differs, so read the +// current version from source rather than hardcoding a number that can drift. +function readProtocolVersion() { + const source = readFileSync(join(projectDir, 'src/main/daemon/types.ts'), 'utf8') + const match = source.match(/PROTOCOL_VERSION\s*=\s*(\d+)/) + if (!match) { + throw new Error('could not read PROTOCOL_VERSION from src/main/daemon/types.ts') + } + return Number(match[1]) +} + +function makeSocketPath(userDataDir) { + // Why: Windows AF_UNIX-style IPC uses named pipes; POSIX uses a filesystem + // socket kept under the scratch userData dir so cleanup removes it. + if (process.platform === 'win32') { + return `\\\\.\\pipe\\orca-daemon-smoke-${process.pid}-${randomUUID()}` + } + return join(userDataDir, 'daemon.sock') +} + +// Best-effort end-to-end PTY check over the daemon's own control-socket RPC. +// Connects a single control socket, completes the hello handshake, and calls +// `ptySpawnHealth` (the daemon spawns a throwaway PTY internally). Resolves +// true on success, false on any failure — never throws. +function runPtySpawnHealthCheck(socketPath, tokenPath, protocolVersion) { + return new Promise((resolveCheck) => { + let settled = false + let buffer = '' + const socket = connect(socketPath) + const finish = (ok, reason) => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + socket.destroy() + if (!ok && reason) { + log(`PTY spawn health check skipped (best-effort): ${reason}`) + } + resolveCheck(ok) + } + const timer = setTimeout(() => finish(false, 'timed out'), PTY_HEALTH_TIMEOUT_MS) + + socket.on('error', (err) => finish(false, err.message)) + socket.on('connect', () => { + const token = readFileSync(tokenPath, 'utf8').trim() + socket.write( + `${JSON.stringify({ + type: 'hello', + version: protocolVersion, + token, + clientId: randomUUID(), + role: 'control' + })}\n` + ) + }) + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8') + let newlineIdx = buffer.indexOf('\n') + while (newlineIdx !== -1) { + const line = buffer.slice(0, newlineIdx) + buffer = buffer.slice(newlineIdx + 1) + let msg + try { + msg = JSON.parse(line) + } catch { + finish(false, 'invalid response line') + return + } + if (msg.type === 'hello') { + if (!msg.ok) { + finish(false, `hello rejected: ${msg.error ?? 'unknown'}`) + return + } + socket.write(`${JSON.stringify({ id: 'health-1', type: 'ptySpawnHealth' })}\n`) + } else if (msg.id === 'health-1') { + finish( + msg.ok === true, + msg.ok === true ? undefined : (msg.error ?? 'ptySpawnHealth failed') + ) + return + } + newlineIdx = buffer.indexOf('\n') + } + }) + }) +} + +async function main() { + const userDataDir = mkdtempSync(join(tmpdir(), 'orca-daemon-boot-smoke-')) + const socketPath = makeSocketPath(userDataDir) + const tokenPath = join(userDataDir, 'daemon.token') + const protocolVersion = readProtocolVersion() + + log(`forking ${entryPath} under plain Node (${process.execPath})`) + const child = fork(entryPath, ['--socket', socketPath, '--token', tokenPath], { + // Plain Node: no ELECTRON_RUN_AS_NODE. process.execPath is already node in + // CI, and this is exactly the runtime where a leaked `require("electron")` + // throws MODULE_NOT_FOUND — the failure this smoke exists to catch. + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + env: { ...process.env, ORCA_USER_DATA_PATH: userDataDir } + }) + + let stderr = '' + child.stderr?.on('data', (chunk) => { + stderr += chunk.toString('utf8') + }) + child.stdout?.on('data', (chunk) => { + process.stdout.write(chunk) + }) + + const cleanup = () => { + if (child.exitCode === null && child.signalCode === null && child.pid) { + try { + child.kill('SIGKILL') + } catch { + // already gone + } + } + rmSync(userDataDir, { recursive: true, force: true }) + } + + try { + await new Promise((resolveReady, rejectReady) => { + const timer = setTimeout(() => { + rejectReady( + new Error( + `daemon did not signal 'ready' within ${READY_TIMEOUT_MS}ms.\nstderr:\n${stderr}` + ) + ) + }, READY_TIMEOUT_MS) + child.on('message', (msg) => { + if (msg && typeof msg === 'object' && msg.type === 'ready') { + clearTimeout(timer) + resolveReady() + } + }) + child.on('error', (err) => { + clearTimeout(timer) + rejectReady(new Error(`daemon fork errored: ${err.message}\nstderr:\n${stderr}`)) + }) + child.on('exit', (code, signal) => { + clearTimeout(timer) + rejectReady( + new Error( + `daemon exited before 'ready' (code=${code}, signal=${signal}).\nstderr:\n${stderr}` + ) + ) + }) + }) + log('daemon signaled ready') + + const ptyHealthy = await runPtySpawnHealthCheck(socketPath, tokenPath, protocolVersion) + if (ptyHealthy) { + log('ptySpawnHealth OK — daemon spawned a real PTY end-to-end') + } + + await new Promise((resolveExit, rejectExit) => { + const timer = setTimeout(() => { + rejectExit(new Error(`daemon did not exit within ${SHUTDOWN_TIMEOUT_MS}ms of SIGTERM`)) + }, SHUTDOWN_TIMEOUT_MS) + child.on('exit', (code, signal) => { + clearTimeout(timer) + log(`daemon exited after signal (code=${code}, signal=${signal})`) + resolveExit() + }) + // Why: SIGTERM is the graceful stop on POSIX (the daemon handles it); + // Windows has no POSIX signal delivery, so Node maps this to process + // termination. Either way the hard assertion is "it stops, no hang". + child.kill('SIGTERM') + }) + + log('PASS: daemon booted, served, and shut down under plain Node') + } finally { + cleanup() + } +} + +main().catch((error) => { + process.stderr.write(`[daemon-boot-smoke] FAIL: ${error.message}\n`) + process.exitCode = 1 +}) diff --git a/config/scripts/verify-packaged-daemon-entry.cjs b/config/scripts/verify-packaged-daemon-entry.cjs new file mode 100644 index 00000000000..6ad91285ca3 --- /dev/null +++ b/config/scripts/verify-packaged-daemon-entry.cjs @@ -0,0 +1,46 @@ +const { existsSync } = require('node:fs') +const { spawnSync } = require('node:child_process') +const { join } = require('node:path') + +// Why: v1.4.129-rc.1 shipped a terminal daemon that could not load (an electron +// `require` leaked into its bundle) while every build check passed. This boots +// the PACKAGED daemon-entry under plain Node against the asar-unpacked layout, +// so a bundling / asar-unpack regression fails packaging instead of reaching +// users. Module-load proof only: with no args the entry must reach argv parsing +// and print its "Usage: daemon-entry" error — a MODULE_NOT_FOUND or a missing +// usage line means the packaged graph does not load and the build must fail. +// +// resourcesDir is the packaged Resources dir (Contents/Resources on macOS, +// /resources elsewhere). execPath defaults to the packaging Node. +function verifyPackagedDaemonEntryBoots(resourcesDir, options = {}) { + const execPath = options.execPath || process.execPath + const entryPath = join(resourcesDir, 'app.asar.unpacked', 'out', 'main', 'daemon-entry.js') + if (!existsSync(entryPath)) { + // Why: some targets/layouts do not unpack here; skip rather than fail so + // the hook stays safe across platforms it has not verified. + console.log(`[verify-packaged-daemon-entry] skipped — no unpacked entry at ${entryPath}`) + return + } + + const result = spawnSync(execPath, [entryPath], { encoding: 'utf8', timeout: 10_000 }) + if (result.error) { + throw new Error( + `[verify-packaged-daemon-entry] could not launch daemon-entry.js: ${result.error.message}` + ) + } + const stderr = result.stderr || '' + if (/Cannot find module|MODULE_NOT_FOUND/.test(stderr)) { + throw new Error( + `[verify-packaged-daemon-entry] packaged daemon-entry.js failed to load under plain Node:\n${stderr}` + ) + } + if (!stderr.includes('Usage: daemon-entry')) { + throw new Error( + `[verify-packaged-daemon-entry] packaged daemon-entry.js did not reach argv parsing ` + + `(expected the "Usage: daemon-entry" error). stderr:\n${stderr}` + ) + } + console.log('[verify-packaged-daemon-entry] OK — packaged daemon-entry loads under plain Node') +} + +module.exports = { verifyPackagedDaemonEntryBoots } diff --git a/src/main/daemon/daemon-entry.ts b/src/main/daemon/daemon-entry.ts index 713cc840278..9dfc97532fa 100644 --- a/src/main/daemon/daemon-entry.ts +++ b/src/main/daemon/daemon-entry.ts @@ -46,6 +46,13 @@ export function parseArgs(argv: string[]): ParsedDaemonArgs { } async function main(): Promise { + // Why: the parent captures daemon startup stderr then destroys its end of the + // pipe once the daemon is ready. A later write here (e.g. the uncaughtException + // console.error below) would then hit a broken pipe and emit 'error' on + // process.stderr — with no listener that becomes an unhandled error that kills + // an otherwise healthy detached daemon. Swallow it: stderr is diagnostic only. + process.stderr.on('error', () => {}) + const { socketPath, tokenPath, logFilePath } = parseArgs(process.argv.slice(2)) // Fail-open: a broken log path must never block daemon startup. const daemonLog = logFilePath ? createDaemonFileLog(logFilePath) : createNoopDaemonFileLog() diff --git a/src/main/daemon/daemon-init.test.ts b/src/main/daemon/daemon-init.test.ts index 536d3cc9b24..39f05c641ee 100644 --- a/src/main/daemon/daemon-init.test.ts +++ b/src/main/daemon/daemon-init.test.ts @@ -1365,6 +1365,123 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => { expect(child.unref).not.toHaveBeenCalled() }) + it('captures daemon startup stderr into the failure error', async () => { + const mod = await importFresh() + checkDaemonHealthMock.mockResolvedValue('unreachable') + await mod.initDaemonPtyProvider() + + const launcher = spawnerInstances[0].launcher as ( + socketPath: string, + tokenPath: string + ) => Promise<{ shutdown(): Promise }> + const handlers: Record void)[]> = { + message: [], + error: [], + exit: [] + } + const stderrDataCbs: ((chunk: Buffer) => void)[] = [] + const stderrDestroy = vi.fn() + const stderr = { + on(event: string, cb: (chunk: Buffer) => void) { + if (event === 'data') { + stderrDataCbs.push(cb) + } + return this + }, + off(event: string, cb: (chunk: Buffer) => void) { + if (event === 'data') { + const idx = stderrDataCbs.indexOf(cb) + if (idx !== -1) { + stderrDataCbs.splice(idx, 1) + } + } + return this + }, + destroy: stderrDestroy + } + const child = { + pid: 4321, + stderr, + on(event: string, cb: (arg?: unknown) => void) { + handlers[event]?.push(cb) + if (event === 'exit') { + // Why: deliver the stderr tail before the exit so the failure path + // sees the captured crash reason, mirroring a module-load crash. + queueMicrotask(() => { + for (const dataCb of stderrDataCbs.slice()) { + dataCb(Buffer.from("Error: Cannot find module 'electron'\n")) + } + cb(1) + }) + } + return this + }, + off: vi.fn((event: string, cb: (arg?: unknown) => void) => { + handlers[event] = handlers[event]?.filter((handler) => handler !== cb) ?? [] + return child + }), + disconnect: vi.fn(), + unref: vi.fn() + } + forkMock.mockReturnValueOnce(child) + + const error = await launcher('/fake/socket', '/fake/token').catch((err: Error) => err) + + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toMatch(/Cannot find module 'electron'/) + expect((error as Error).message).toMatch(/Daemon stderr \(tail\)/) + // Why: the piped stderr must be released so the detached daemon does not + // keep the parent event loop alive after the failure. + expect(stderrDestroy).toHaveBeenCalled() + }) + + it('destroys the daemon stderr pipe once the daemon signals ready', async () => { + const mod = await importFresh() + checkDaemonHealthMock.mockResolvedValue('unreachable') + await mod.initDaemonPtyProvider() + + const launcher = spawnerInstances[0].launcher as ( + socketPath: string, + tokenPath: string + ) => Promise<{ shutdown(): Promise }> + const handlers: Record void)[]> = { + message: [], + error: [], + exit: [] + } + const stderrOff = vi.fn() + const stderrDestroy = vi.fn() + const stderr = { + on() { + return this + }, + off: stderrOff, + destroy: stderrDestroy + } + const child = { + pid: 12345, + stderr, + on(event: string, cb: (arg?: unknown) => void) { + handlers[event]?.push(cb) + if (event === 'message') { + queueMicrotask(() => cb({ type: 'ready' })) + } + return this + }, + off: vi.fn(() => child), + disconnect: vi.fn(), + unref: vi.fn() + } + forkMock.mockReturnValueOnce(child) + + await launcher('/fake/socket', '/fake/token') + + expect(stderrOff).toHaveBeenCalledWith('data', expect.any(Function)) + expect(stderrDestroy).toHaveBeenCalledOnce() + expect(child.disconnect).toHaveBeenCalledOnce() + expect(child.unref).toHaveBeenCalledOnce() + }) + it('preserves a health-check-failing daemon when it owns live sessions', async () => { const mod = await importFresh() await mod.initDaemonPtyProvider() diff --git a/src/main/daemon/daemon-init.ts b/src/main/daemon/daemon-init.ts index 0517f275975..4de553cde16 100644 --- a/src/main/daemon/daemon-init.ts +++ b/src/main/daemon/daemon-init.ts @@ -298,10 +298,13 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher { // userData keeps process.cwd() valid after a repo/worktree is deleted. cwd: userDataPath, // Why: detached + unref lets the daemon outlive the Electron process. - // stdio 'ignore' prevents the child from holding the parent's stdout - // open, which would prevent Electron from exiting cleanly. + // stdout stays 'ignore' so the child never holds the parent's stdout + // open (which would block Electron exit); stderr is 'pipe' so a + // module-load crash during startup is captured instead of discarded + // (v1.4.129-rc.1 shipped a daemon that only logged "exited with code 1" + // because stderr was thrown away). The pipe is destroyed on readiness. detached: true, - stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], // Why: run the relocated Orca.exe copy instead of the install-dir one. // It is byte-identical, so run-as-node behavior is unchanged; only the // image path moves out of the updater's kill zone. @@ -320,6 +323,30 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher { } ) + // Why: keep only the startup-window stderr tail so a crash cause is + // visible without unbounded memory if the daemon spews before dying. + const STARTUP_STDERR_MAX_BYTES = 8192 + let startupStderr = '' + let collectingStderr = true + const onStartupStderr = (chunk: Buffer): void => { + if (!collectingStderr) { + return + } + startupStderr += chunk.toString('utf8') + if (startupStderr.length > STARTUP_STDERR_MAX_BYTES) { + startupStderr = startupStderr.slice(-STARTUP_STDERR_MAX_BYTES) + } + } + child.stderr?.on('data', onStartupStderr) + // Why: once the daemon is up (or has failed) the parent must not keep a + // live handle on the detached daemon's stderr — a piped stream would ref + // the parent event loop and prevent Electron from exiting cleanly. + const releaseStderr = (): void => { + collectingStderr = false + child.stderr?.off('data', onStartupStderr) + child.stderr?.destroy() + } + // Wait for the daemon to signal readiness via IPC await new Promise((resolve, reject) => { let timer: ReturnType | undefined @@ -338,6 +365,14 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher { } settled = true cleanupStartupListeners() + // Why: stderr was previously discarded, so a startup crash surfaced only + // as "exited with code 1". Attach the captured tail to the thrown error + // (which the fallback path reports) and log it so the real cause shows. + const stderrTail = startupStderr.trim() + if (stderrTail) { + console.warn(`[daemon] startup failed; captured stderr tail:\n${stderrTail}`) + } + releaseStderr() if (child.pid) { try { process.kill(child.pid, 'SIGTERM') @@ -345,7 +380,9 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher { // Already dead } } - reject(error) + reject( + stderrTail ? new Error(`${error.message}\nDaemon stderr (tail):\n${stderrTail}`) : error + ) } function onReadyMessage(msg: unknown): void { if (msg && typeof msg === 'object' && (msg as { type?: string }).type === 'ready') { @@ -372,8 +409,10 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher { { mode: 0o600 } ) } - // Why: disconnect IPC channel and unref so Electron can exit - // without waiting for the daemon. The daemon keeps running. + // Why: disconnect IPC channel, release the stderr pipe, and unref so + // Electron can exit without waiting for the daemon. The daemon keeps + // running detached. + releaseStderr() child.disconnect() child.unref() resolve() diff --git a/src/main/index.ts b/src/main/index.ts index e6b0323b4fd..70cf0475c2e 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -27,7 +27,8 @@ import { registerCoreHandlers } from './ipc/register-core-handlers' import { initObservability, shutdownObservability } from './observability' import { startSpan } from './observability/tracer' import { registerMobileHandlers } from './ipc/mobile' -import { initTelemetry, shutdownTelemetry, trackAppOpenedOnce } from './telemetry/client' +import { initTelemetry, shutdownTelemetry, trackAppOpenedOnce, track } from './telemetry/client' +import { classifyError } from './telemetry/classify-error' import { runManagedHookInstallers } from './agent-hooks/install-telemetry' import { isAgentStatusHooksEnabled, @@ -644,7 +645,16 @@ function startDesktopFirstWindowStartupServices(): Promise { logStartupMilestone('startup-service-done', { service: 'agent-hook-server' }) }, onDaemonError: (error) => { - console.error('[daemon] Failed to start daemon PTY provider, falling back to local:', error) + // Why: daemon startup failure silently dropped terminals onto the local + // provider (killed on quit, no persistence) — the v1.4.129-rc.1 outage was + // invisible in the field. Log loudly (error.message carries the captured + // daemon stderr tail from the fork) and emit a low-cardinality telemetry + // signal so a fleet-wide daemon failure is observable without a bug report. + const reason = error instanceof Error ? error.message : String(error) + console.error( + `[daemon] STARTUP FAILED — falling back to local PTYs; terminals will not persist across quit. Reason: ${reason}` + ) + track('daemon_start_failed', classifyError(error)) }, onAgentHookServerError: (error) => { // Why: Claude/Codex/Gemini/OpenCode/Cursor hook callbacks are sidebar diff --git a/src/shared/telemetry-events.ts b/src/shared/telemetry-events.ts index cb4169cb1dd..1a77eb76d44 100644 --- a/src/shared/telemetry-events.ts +++ b/src/shared/telemetry-events.ts @@ -436,6 +436,13 @@ const agentErrorSchema = z }) .strict() +// Why: emitted when the terminal daemon cannot start and terminals fall back to +// the (non-persistent) local provider. Enum-only `error_class` — the raw daemon +// stderr tail stays in local logs and never reaches the wire (paths/usernames). +// A spike in this event is the fleet-wide signal for a daemon outage like +// v1.4.129-rc.1, which was otherwise invisible until users filed bug reports. +const daemonStartFailedSchema = z.object({ error_class: errorClassSchema }).strict() + const settingsChangedSchema = z .object({ setting_key: settingsChangedKeySchema, @@ -1420,6 +1427,8 @@ export const eventSchemas = { agent_hook_install_failed: agentHookInstallFailedSchema, agent_hook_unattributed: agentHookUnattributedSchema, + daemon_start_failed: daemonStartFailedSchema, + settings_changed: settingsChangedSchema, native_chat_toggled: nativeChatToggledSchema,