diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8826fd393ed..58d84eaefeb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -869,6 +869,14 @@ jobs: src/shared/startup-shell-portability.live-shell.test.ts \ src/shared/posix-command-path-lookup.test.ts + # Own serial step: RLIMIT_NPROC counts every task of the runner user, so nothing else may run. + - name: Prove a PTY spawn at the process limit fails alone (#40326) + env: + ORCA_REQUIRE_PROCESS_LIMIT_PROOF: '1' + run: | + pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 \ + src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts + # Shards wait only for the plan, not preflight, which moves them about two minutes earlier. unit_plan: needs: [code_paths] diff --git a/config/patches/node-pty@1.1.0.patch b/config/patches/node-pty@1.1.0.patch index d6ee94ff78d..c95cf3c653f 100644 --- a/config/patches/node-pty@1.1.0.patch +++ b/config/patches/node-pty@1.1.0.patch @@ -2215,3 +2215,89 @@ diff --git a/src/native.d.ts b/src/native.d.ts fork(file: string, args: string[], parsedEnv: string[], cwd: string, cols: number, rows: number, uid: number, gid: number, useUtf8: boolean, helperPath: string, onExitCallback: (code: number, signal: number) => void): IUnixProcess; open(cols: number, rows: number): IUnixOpenProcess; process(fd: number, pty?: string): string; +diff --git a/src/unix/pty.cc b/src/unix/pty.cc +index 297b471..f7ef598 100644 +--- a/src/unix/pty.cc ++++ b/src/unix/pty.cc +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -159,9 +160,17 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pid_t pid) { + 0, // Unlimited queue + 1, // Only one thread will use this initially + [th](Napi::Env) { // Finalizer used to clean threads up +- th->join(); ++ // Orca: a watcher that never started has nothing to join. ++ if (th->joinable()) { ++ th->join(); ++ } + delete th; + }); ++ // Orca: at a process or thread limit (for example a cgroup pids.max) the thread cannot start and ++ // std::thread throws std::system_error. Only Napi::Error is caught for JS, so let PtyFork turn it ++ // into one; release the function here so its finalizer still runs. ++ Napi::ThreadSafeFunction owner = tsfn; ++ try { + *th = std::thread([tsfn = std::move(tsfn), pid] { + auto callback = [](Napi::Env env, Napi::Function cb, ExitEvent *exit_event) { + cb.Call({Napi::Number::New(env, exit_event->exit_code), +@@ -250,6 +259,10 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pid_t pid) { + Napi::Error::Fatal("SetupExitCallback", "ThreadSafeFunction.BlockingCall() failed"); + } + }); ++ } catch (const std::exception&) { ++ owner.Release(); ++ throw; ++ } + } + + /** +@@ -454,6 +467,7 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) { + pthread_sigmask(SIG_SETMASK, &newmask, &oldmask); + + pid = forkpty(&master, nullptr, static_cast(term), static_cast(&winp)); ++ int fork_errno = errno; + + if (!pid) { + // remove all signal handler from child +@@ -470,7 +484,9 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) { + + switch (pid) { + case -1: +- throw Napi::Error::New(napiEnv, "forkpty(3) failed."); ++ // Orca: keep errno so a process limit (EAGAIN) reads as one. ++ throw Napi::Error::New(napiEnv, std::string("node-pty: forkpty failed: ") + strerror(fork_errno) + ++ " (errno " + std::to_string(fork_errno) + ")"); + case 0: + if (strlen(cwd_.c_str())) { + if (chdir(cwd_.c_str()) == -1) { +@@ -515,7 +531,23 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) { + + // Set up process exit callback. + Napi::Function cb = info[10].As(); +- SetupExitCallback(napiEnv, cb, pid); ++ // Orca: nothing would ever reap this child or report its exit if its watcher cannot start, so end ++ // it instead of leaking it, and report why. A failed allocation carries no errno: report ENOMEM. ++ auto fail_exit_watcher = [&](const char *what, int error_code) -> Napi::Error { ++ kill(pid, SIGKILL); ++ while (waitpid(pid, nullptr, 0) == -1 && errno == EINTR) { ++ } ++ close(master); ++ return Napi::Error::New(napiEnv, std::string("node-pty: exit_watcher_thread failed: ") + what + ++ " (errno " + std::to_string(error_code) + ")"); ++ }; ++ try { ++ SetupExitCallback(napiEnv, cb, pid); ++ } catch (const std::system_error& error) { ++ throw fail_exit_watcher(error.what(), error.code().value()); ++ } catch (const std::exception& error) { ++ throw fail_exit_watcher(error.what(), ENOMEM); ++ } + return obj; + } + diff --git a/config/scripts/workflow/source-routing/pr-shell-contract-prefixes.mjs b/config/scripts/workflow/source-routing/pr-shell-contract-prefixes.mjs index 7fc2b972fec..4681c23702b 100644 --- a/config/scripts/workflow/source-routing/pr-shell-contract-prefixes.mjs +++ b/config/scripts/workflow/source-routing/pr-shell-contract-prefixes.mjs @@ -7,6 +7,8 @@ export const SHELL_PREFIXES = [ 'src/main/daemon/shell-readiness/daemon-bash-shell-ready-rcfile.ts', 'src/main/daemon/shell-readiness/daemon-shell-ready-wrapper-fileset.ts', 'src/main/daemon/pty-subprocess/testing/node-pty-fd-leak.test.ts', + 'src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts', + 'src/main/daemon/pty-subprocess/node-pty-error-hints', 'src/main/providers/local-pty/local-pty-shell-ready', 'src/main/pty/', 'src/main/runtime/agent-session/environment/structured-session-cli-login-shell', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 59e5770f157..7234cf6d7dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -214,7 +214,7 @@ patchedDependencies: minimatch@10.2.6: 4324868e550a69eee9addc4801001bee415e98339c9b2b88d6ab4fc6028741c6 minimatch@9.0.9: c356feccb9147b4d6a1cc72b10270defcb95d14cb42ee74ddfe7219341417161 monaco-editor@0.55.1: 53b71f7d8a7c00b76ffdb73ce83d18e31692f193f5fafa6a390e48f5b3fdd636 - node-pty@1.1.0: 37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5 + node-pty@1.1.0: ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405 oxlint-plugin-anti-slop@0.1.2: c10c871702365a0c2c59e9f1c5fb92df97cae5755d534c0a3d83c6368f6feafe pdfjs-dist@6.3.289: 424fde648e16c965ce45c55bdda49ecb3a4aa992c856900714c841c3f13f8415 prosemirror-view@1.42.3: 3340ccbcc4e3302f3ae1c0eda3911015966add2e7e0df76abd89fbef4f3902f7 @@ -267,7 +267,7 @@ importers: version: 3.0.2 node-pty: specifier: ^1.1.0 - version: 1.1.0(patch_hash=37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5) + version: 1.1.0(patch_hash=ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405) posthog-node: specifier: ^5.52.5 version: 5.52.5 @@ -14264,7 +14264,7 @@ snapshots: node-int64@0.4.0: {} - node-pty@1.1.0(patch_hash=37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5): + node-pty@1.1.0(patch_hash=ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405): dependencies: node-addon-api: 7.1.1 diff --git a/src/main/daemon/pty-subprocess/node-pty-error-hints.test.ts b/src/main/daemon/pty-subprocess/node-pty-error-hints.test.ts new file mode 100644 index 00000000000..8676fe5d0be --- /dev/null +++ b/src/main/daemon/pty-subprocess/node-pty-error-hints.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { + PTY_ALLOCATION_HINT, + TERMINAL_PROCESS_LIMIT_HINT +} from '../../../shared/terminal/pty/lifecycle/terminal-spawn-error-copy' +import { addNodePtyRecoveryHint } from './node-pty-error-hints' + +// #40326: at a cgroup pids.max limit a spawn fails with EAGAIN; the user needs the process-limit +// hint, not one about running out of pseudo-terminals. +describe('a terminal spawn refused at a process limit', () => { + it.each([ + 'node-pty: forkpty failed: Resource temporarily unavailable (errno 11)', + 'node-pty: exit_watcher_thread failed: Resource temporarily unavailable (errno 11)', + 'node-pty: exit_watcher_thread failed: thread constructor failed: Resource temporarily unavailable (errno 35)' + ])('names the process limit for %s', (message) => { + const hinted = addNodePtyRecoveryHint(message) + expect(hinted.startsWith(TERMINAL_PROCESS_LIMIT_HINT)).toBe(true) + expect(hinted.startsWith(PTY_ALLOCATION_HINT)).toBe(false) + expect(hinted.endsWith(message)).toBe(true) + }) + + it.each([ + 'node-pty: forkpty failed: No space left on device (errno 28)', + 'node-pty: forkpty failed: Too many open files (errno 24)' + ])('still names the pseudo-terminal limit when forkpty ran out of devices: %s', (message) => { + expect(addNodePtyRecoveryHint(message).startsWith(PTY_ALLOCATION_HINT)).toBe(true) + }) +}) diff --git a/src/main/daemon/pty-subprocess/node-pty-error-hints.ts b/src/main/daemon/pty-subprocess/node-pty-error-hints.ts index 946530df810..4d365428db8 100644 --- a/src/main/daemon/pty-subprocess/node-pty-error-hints.ts +++ b/src/main/daemon/pty-subprocess/node-pty-error-hints.ts @@ -23,6 +23,12 @@ const PTY_ALLOCATION_STEPS = new Set([ 'open_slave' ]) +// Starting the shell, or the thread that waits for it to exit, fails at a process or thread limit. +const PROCESS_START_STEPS = new Set(['posix_spawn', 'exit_watcher_thread']) +// Linux forkpty opens a pseudo-terminal and forks, so its errno says which one ran out. +const FORK_LIMIT_ERRNOS = new Set([11, 12]) +const PTY_DEVICE_LIMIT_ERRNOS = new Set([6, 23, 24, 28]) + const RESOURCE_EXHAUSTION_ERRNOS = new Set([ 6, // ENXIO on macOS: posix_openpt could not provide a usable pty device 11, // EAGAIN on Linux @@ -55,7 +61,17 @@ export function getNodePtyRecoveryHint(diagnostic: NodePtyDiagnostic): string | ) { return PTY_ALLOCATION_HINT } - if (diagnostic.step === 'posix_spawn' && RESOURCE_EXHAUSTION_ERRNOS.has(diagnostic.errno)) { + if (diagnostic.step === 'forkpty') { + return FORK_LIMIT_ERRNOS.has(diagnostic.errno) + ? TERMINAL_PROCESS_LIMIT_HINT + : PTY_DEVICE_LIMIT_ERRNOS.has(diagnostic.errno) + ? PTY_ALLOCATION_HINT + : null + } + if ( + PROCESS_START_STEPS.has(diagnostic.step) && + RESOURCE_EXHAUSTION_ERRNOS.has(diagnostic.errno) + ) { return TERMINAL_PROCESS_LIMIT_HINT } return null diff --git a/src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts b/src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts new file mode 100644 index 00000000000..033bfeaf946 --- /dev/null +++ b/src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts @@ -0,0 +1,133 @@ +import { execFileSync, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { createRequire } from 'node:module' +import { createInterface } from 'node:readline' +import { afterEach, describe, expect, it } from 'vitest' +import { TERMINAL_PROCESS_LIMIT_HINT } from '../../../../shared/terminal/pty/lifecycle/terminal-spawn-error-copy' +import { addNodePtyRecoveryHint } from '../node-pty-error-hints' + +// #40326: at a process or thread limit (a container's pids.max), starting a PTY's exit watcher +// thread threw a C++ error node-pty does not catch, and the daemon aborted with every terminal. +// RLIMIT_NPROC counts every task (process or thread) of this user. Capping a child at the user's +// current count (which still includes the exiting `ps`) leaves one slot: forkpty takes it and the +// watcher thread finds none. Other tasks of this user make that racy, so the CI "shell contracts" +// job runs this file in its own serial step with ORCA_REQUIRE_PROCESS_LIMIT_PROOF=1; elsewhere it +// is skipped. Root ignores RLIMIT_NPROC, and only Linux counts threads against it. + +const REQUIRED = process.env.ORCA_REQUIRE_PROCESS_LIMIT_PROOF === '1' +const WATCHER_REFUSAL = /^node-pty: exit_watcher_thread failed: .*\(errno 11\)/ +const FORK_REFUSAL = /^node-pty: forkpty failed: .*\(errno 11\)/ + +function succeeds(command: string, args: string[]): boolean { + try { + execFileSync(command, args, { stdio: 'ignore' }) + return true + } catch { + return false + } +} + +function canCapTasks(): boolean { + return ( + process.platform === 'linux' && + process.getuid?.() !== 0 && + succeeds('prlimit', ['--version']) && + succeeds('ps', ['-L', '-p', String(process.pid), '--no-headers']) + ) +} + +function isRefusal(value: unknown): value is { spawned: false; message: string } { + return ( + typeof value === 'object' && + value !== null && + 'spawned' in value && + value.spawned === false && + 'message' in value && + typeof value.message === 'string' + ) +} + +/** Every task this user runs right now; RLIMIT_NPROC compares new tasks against this count. */ +function userTaskCount(): number { + const listing = execFileSync('ps', ['-L', '-u', String(process.getuid?.()), '--no-headers'], { + encoding: 'utf8' + }) + return listing.split('\n').filter((line) => line.trim().length > 0).length +} + +// The child warms libuv's thread pool before any cap, so only the PTY spawn needs a new task. +const CHILD = ` +const pty = require(process.argv[1]) +const fs = require('node:fs') +const readline = require('node:readline') +fs.readFile('/proc/self/status', () => { + process.stdout.write('ready\\n') + readline.createInterface({ input: process.stdin }).on('line', () => { + let result + try { + const term = pty.spawn('/bin/sh', ['-c', 'exit 0'], { cols: 80, rows: 24, cwd: '/' }) + term.kill() + result = { spawned: true } + } catch (error) { + result = { spawned: false, message: String(error && error.message) } + } + process.stdout.write(JSON.stringify(result) + '\\n') + }) +}) +` + +describe.skipIf(!REQUIRED)('a PTY spawn at the process limit (Linux)', () => { + let child: ChildProcessWithoutNullStreams | null = null + afterEach(() => { + child?.kill('SIGKILL') + child = null + }) + + it( + 'fails that spawn with the process-limit hint and keeps the process alive', + async () => { + expect(canCapTasks(), 'needs Linux, a non-root user, prlimit and procps ps').toBe(true) + const ptyPath = createRequire(import.meta.url).resolve('node-pty') + // The Node binary this node-pty build was made for; vitest itself may run under Bun. + const node = process.versions.bun ? 'node' : process.execPath + child = spawn(node, ['-e', CHILD, ptyPath], { stdio: ['pipe', 'pipe', 'pipe'] }) + const running = child + const lines = createInterface({ input: running.stdout })[Symbol.asyncIterator]() + const nextLine = async (): Promise => { + const { value, done } = await lines.next() + if (done) { + throw new Error(`child exited (${running.exitCode ?? running.signalCode})`) + } + return value + } + expect(await nextLine()).toBe('ready') + + const refusals: string[] = [] + // A rising task count refuses the fork instead; keep trying until the watcher thread is refused. + for ( + let attempt = 0; + attempt < 40 && !refusals.some((m) => WATCHER_REFUSAL.test(m)); + attempt++ + ) { + const cap = userTaskCount() + // Soft limit only: a non-root user cannot raise a lowered hard limit for the next attempt. + execFileSync('prlimit', ['--pid', String(running.pid), `--nproc=${cap}:`]) + running.stdin.write('spawn\n') + const result: unknown = JSON.parse(await nextLine()) + if (isRefusal(result)) { + refusals.push(result.message) + } + } + + expect(refusals.some((message) => WATCHER_REFUSAL.test(message))).toBe(true) + for (const message of refusals) { + expect(WATCHER_REFUSAL.test(message) || FORK_REFUSAL.test(message)).toBe(true) + expect(addNodePtyRecoveryHint(message).startsWith(TERMINAL_PROCESS_LIMIT_HINT)).toBe(true) + } + // An uncaught native error would have aborted the child before it could answer. + expect(running.exitCode).toBeNull() + expect(running.signalCode).toBeNull() + }, + // Up to 40 spawn rounds, each with a `ps` and a `prlimit`, on a loaded CI runner. + 60_000 + ) +})