mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(terminal): fail one spawn instead of the whole daemon at a process limit
At a process or thread limit (for example a container's cgroup pids.max),
starting the thread that waits for a new PTY's exit throws std::system_error.
node-pty is built to catch only Napi::Error, so that error escaped to
std::terminate and the daemon aborted, taking every terminal with it and
logging nothing.
The node-pty patch now catches the failed watcher start (or a failed
allocation), releases its thread-safe function, kills and reaps the child,
closes the master fd and throws a JS error ("node-pty: exit_watcher_thread
failed: ... (errno N)"). A failed forkpty also reports its errno. Process
limits map to the process-limit hint; running out of pseudo-terminals keeps
the pseudo-terminal hint.
A Linux test caps a child's RLIMIT_NPROC so the watcher thread is refused and
checks that the spawn fails with the hint while the process lives. It runs in
its own serial step of the "shell contracts" CI job.
Fixes #40326
This commit is contained in:
@@ -869,6 +869,14 @@ jobs:
|
||||
src/shared/startup-shell-portability.live-shell.test.ts \
|
||||
src/shared/posix-command-path-lookup.test.ts
|
||||
|
||||
# Own serial step: RLIMIT_NPROC counts every task of the runner user, so nothing else may run.
|
||||
- name: Prove a PTY spawn at the process limit fails alone (#40326)
|
||||
env:
|
||||
ORCA_REQUIRE_PROCESS_LIMIT_PROOF: '1'
|
||||
run: |
|
||||
pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 \
|
||||
src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts
|
||||
|
||||
# Shards wait only for the plan, not preflight, which moves them about two minutes earlier.
|
||||
unit_plan:
|
||||
needs: [code_paths]
|
||||
|
||||
@@ -2215,3 +2215,89 @@ diff --git a/src/native.d.ts b/src/native.d.ts
|
||||
fork(file: string, args: string[], parsedEnv: string[], cwd: string, cols: number, rows: number, uid: number, gid: number, useUtf8: boolean, helperPath: string, onExitCallback: (code: number, signal: number) => void): IUnixProcess;
|
||||
open(cols: number, rows: number): IUnixOpenProcess;
|
||||
process(fd: number, pty?: string): string;
|
||||
diff --git a/src/unix/pty.cc b/src/unix/pty.cc
|
||||
index 297b471..f7ef598 100644
|
||||
--- a/src/unix/pty.cc
|
||||
+++ b/src/unix/pty.cc
|
||||
@@ -26,6 +26,7 @@
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <string>
|
||||
+#include <system_error>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
@@ -159,9 +160,17 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pid_t pid) {
|
||||
0, // Unlimited queue
|
||||
1, // Only one thread will use this initially
|
||||
[th](Napi::Env) { // Finalizer used to clean threads up
|
||||
- th->join();
|
||||
+ // Orca: a watcher that never started has nothing to join.
|
||||
+ if (th->joinable()) {
|
||||
+ th->join();
|
||||
+ }
|
||||
delete th;
|
||||
});
|
||||
+ // Orca: at a process or thread limit (for example a cgroup pids.max) the thread cannot start and
|
||||
+ // std::thread throws std::system_error. Only Napi::Error is caught for JS, so let PtyFork turn it
|
||||
+ // into one; release the function here so its finalizer still runs.
|
||||
+ Napi::ThreadSafeFunction owner = tsfn;
|
||||
+ try {
|
||||
*th = std::thread([tsfn = std::move(tsfn), pid] {
|
||||
auto callback = [](Napi::Env env, Napi::Function cb, ExitEvent *exit_event) {
|
||||
cb.Call({Napi::Number::New(env, exit_event->exit_code),
|
||||
@@ -250,6 +259,10 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pid_t pid) {
|
||||
Napi::Error::Fatal("SetupExitCallback", "ThreadSafeFunction.BlockingCall() failed");
|
||||
}
|
||||
});
|
||||
+ } catch (const std::exception&) {
|
||||
+ owner.Release();
|
||||
+ throw;
|
||||
+ }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -454,6 +467,7 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) {
|
||||
pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
|
||||
|
||||
pid = forkpty(&master, nullptr, static_cast<termios*>(term), static_cast<winsize*>(&winp));
|
||||
+ int fork_errno = errno;
|
||||
|
||||
if (!pid) {
|
||||
// remove all signal handler from child
|
||||
@@ -470,7 +484,9 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) {
|
||||
|
||||
switch (pid) {
|
||||
case -1:
|
||||
- throw Napi::Error::New(napiEnv, "forkpty(3) failed.");
|
||||
+ // Orca: keep errno so a process limit (EAGAIN) reads as one.
|
||||
+ throw Napi::Error::New(napiEnv, std::string("node-pty: forkpty failed: ") + strerror(fork_errno) +
|
||||
+ " (errno " + std::to_string(fork_errno) + ")");
|
||||
case 0:
|
||||
if (strlen(cwd_.c_str())) {
|
||||
if (chdir(cwd_.c_str()) == -1) {
|
||||
@@ -515,7 +531,23 @@ Napi::Value PtyFork(const Napi::CallbackInfo& info) {
|
||||
|
||||
// Set up process exit callback.
|
||||
Napi::Function cb = info[10].As<Napi::Function>();
|
||||
- SetupExitCallback(napiEnv, cb, pid);
|
||||
+ // Orca: nothing would ever reap this child or report its exit if its watcher cannot start, so end
|
||||
+ // it instead of leaking it, and report why. A failed allocation carries no errno: report ENOMEM.
|
||||
+ auto fail_exit_watcher = [&](const char *what, int error_code) -> Napi::Error {
|
||||
+ kill(pid, SIGKILL);
|
||||
+ while (waitpid(pid, nullptr, 0) == -1 && errno == EINTR) {
|
||||
+ }
|
||||
+ close(master);
|
||||
+ return Napi::Error::New(napiEnv, std::string("node-pty: exit_watcher_thread failed: ") + what +
|
||||
+ " (errno " + std::to_string(error_code) + ")");
|
||||
+ };
|
||||
+ try {
|
||||
+ SetupExitCallback(napiEnv, cb, pid);
|
||||
+ } catch (const std::system_error& error) {
|
||||
+ throw fail_exit_watcher(error.what(), error.code().value());
|
||||
+ } catch (const std::exception& error) {
|
||||
+ throw fail_exit_watcher(error.what(), ENOMEM);
|
||||
+ }
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ export const SHELL_PREFIXES = [
|
||||
'src/main/daemon/shell-readiness/daemon-bash-shell-ready-rcfile.ts',
|
||||
'src/main/daemon/shell-readiness/daemon-shell-ready-wrapper-fileset.ts',
|
||||
'src/main/daemon/pty-subprocess/testing/node-pty-fd-leak.test.ts',
|
||||
'src/main/daemon/pty-subprocess/testing/node-pty-process-limit.test.ts',
|
||||
'src/main/daemon/pty-subprocess/node-pty-error-hints',
|
||||
'src/main/providers/local-pty/local-pty-shell-ready',
|
||||
'src/main/pty/',
|
||||
'src/main/runtime/agent-session/environment/structured-session-cli-login-shell',
|
||||
|
||||
Generated
+3
-3
@@ -214,7 +214,7 @@ patchedDependencies:
|
||||
minimatch@10.2.6: 4324868e550a69eee9addc4801001bee415e98339c9b2b88d6ab4fc6028741c6
|
||||
minimatch@9.0.9: c356feccb9147b4d6a1cc72b10270defcb95d14cb42ee74ddfe7219341417161
|
||||
monaco-editor@0.55.1: 53b71f7d8a7c00b76ffdb73ce83d18e31692f193f5fafa6a390e48f5b3fdd636
|
||||
node-pty@1.1.0: 37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5
|
||||
node-pty@1.1.0: ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405
|
||||
oxlint-plugin-anti-slop@0.1.2: c10c871702365a0c2c59e9f1c5fb92df97cae5755d534c0a3d83c6368f6feafe
|
||||
pdfjs-dist@6.3.289: 424fde648e16c965ce45c55bdda49ecb3a4aa992c856900714c841c3f13f8415
|
||||
prosemirror-view@1.42.3: 3340ccbcc4e3302f3ae1c0eda3911015966add2e7e0df76abd89fbef4f3902f7
|
||||
@@ -267,7 +267,7 @@ importers:
|
||||
version: 3.0.2
|
||||
node-pty:
|
||||
specifier: ^1.1.0
|
||||
version: 1.1.0(patch_hash=37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5)
|
||||
version: 1.1.0(patch_hash=ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405)
|
||||
posthog-node:
|
||||
specifier: ^5.52.5
|
||||
version: 5.52.5
|
||||
@@ -14264,7 +14264,7 @@ snapshots:
|
||||
|
||||
node-int64@0.4.0: {}
|
||||
|
||||
node-pty@1.1.0(patch_hash=37dc98d9a4bcfc67153fa2fbf30689563ff49dcf8e6f0578484cbacdf1e38af5):
|
||||
node-pty@1.1.0(patch_hash=ae0868e485269a2b25662c4849013cfc62a30b2f20fc4a70cbd0a5a667fae405):
|
||||
dependencies:
|
||||
node-addon-api: 7.1.1
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
PTY_ALLOCATION_HINT,
|
||||
TERMINAL_PROCESS_LIMIT_HINT
|
||||
} from '../../../shared/terminal/pty/lifecycle/terminal-spawn-error-copy'
|
||||
import { addNodePtyRecoveryHint } from './node-pty-error-hints'
|
||||
|
||||
// #40326: at a cgroup pids.max limit a spawn fails with EAGAIN; the user needs the process-limit
|
||||
// hint, not one about running out of pseudo-terminals.
|
||||
describe('a terminal spawn refused at a process limit', () => {
|
||||
it.each([
|
||||
'node-pty: forkpty failed: Resource temporarily unavailable (errno 11)',
|
||||
'node-pty: exit_watcher_thread failed: Resource temporarily unavailable (errno 11)',
|
||||
'node-pty: exit_watcher_thread failed: thread constructor failed: Resource temporarily unavailable (errno 35)'
|
||||
])('names the process limit for %s', (message) => {
|
||||
const hinted = addNodePtyRecoveryHint(message)
|
||||
expect(hinted.startsWith(TERMINAL_PROCESS_LIMIT_HINT)).toBe(true)
|
||||
expect(hinted.startsWith(PTY_ALLOCATION_HINT)).toBe(false)
|
||||
expect(hinted.endsWith(message)).toBe(true)
|
||||
})
|
||||
|
||||
it.each([
|
||||
'node-pty: forkpty failed: No space left on device (errno 28)',
|
||||
'node-pty: forkpty failed: Too many open files (errno 24)'
|
||||
])('still names the pseudo-terminal limit when forkpty ran out of devices: %s', (message) => {
|
||||
expect(addNodePtyRecoveryHint(message).startsWith(PTY_ALLOCATION_HINT)).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -23,6 +23,12 @@ const PTY_ALLOCATION_STEPS = new Set([
|
||||
'open_slave'
|
||||
])
|
||||
|
||||
// Starting the shell, or the thread that waits for it to exit, fails at a process or thread limit.
|
||||
const PROCESS_START_STEPS = new Set(['posix_spawn', 'exit_watcher_thread'])
|
||||
// Linux forkpty opens a pseudo-terminal and forks, so its errno says which one ran out.
|
||||
const FORK_LIMIT_ERRNOS = new Set([11, 12])
|
||||
const PTY_DEVICE_LIMIT_ERRNOS = new Set([6, 23, 24, 28])
|
||||
|
||||
const RESOURCE_EXHAUSTION_ERRNOS = new Set([
|
||||
6, // ENXIO on macOS: posix_openpt could not provide a usable pty device
|
||||
11, // EAGAIN on Linux
|
||||
@@ -55,7 +61,17 @@ export function getNodePtyRecoveryHint(diagnostic: NodePtyDiagnostic): string |
|
||||
) {
|
||||
return PTY_ALLOCATION_HINT
|
||||
}
|
||||
if (diagnostic.step === 'posix_spawn' && RESOURCE_EXHAUSTION_ERRNOS.has(diagnostic.errno)) {
|
||||
if (diagnostic.step === 'forkpty') {
|
||||
return FORK_LIMIT_ERRNOS.has(diagnostic.errno)
|
||||
? TERMINAL_PROCESS_LIMIT_HINT
|
||||
: PTY_DEVICE_LIMIT_ERRNOS.has(diagnostic.errno)
|
||||
? PTY_ALLOCATION_HINT
|
||||
: null
|
||||
}
|
||||
if (
|
||||
PROCESS_START_STEPS.has(diagnostic.step) &&
|
||||
RESOURCE_EXHAUSTION_ERRNOS.has(diagnostic.errno)
|
||||
) {
|
||||
return TERMINAL_PROCESS_LIMIT_HINT
|
||||
}
|
||||
return null
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { execFileSync, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { createRequire } from 'node:module'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { TERMINAL_PROCESS_LIMIT_HINT } from '../../../../shared/terminal/pty/lifecycle/terminal-spawn-error-copy'
|
||||
import { addNodePtyRecoveryHint } from '../node-pty-error-hints'
|
||||
|
||||
// #40326: at a process or thread limit (a container's pids.max), starting a PTY's exit watcher
|
||||
// thread threw a C++ error node-pty does not catch, and the daemon aborted with every terminal.
|
||||
// RLIMIT_NPROC counts every task (process or thread) of this user. Capping a child at the user's
|
||||
// current count (which still includes the exiting `ps`) leaves one slot: forkpty takes it and the
|
||||
// watcher thread finds none. Other tasks of this user make that racy, so the CI "shell contracts"
|
||||
// job runs this file in its own serial step with ORCA_REQUIRE_PROCESS_LIMIT_PROOF=1; elsewhere it
|
||||
// is skipped. Root ignores RLIMIT_NPROC, and only Linux counts threads against it.
|
||||
|
||||
const REQUIRED = process.env.ORCA_REQUIRE_PROCESS_LIMIT_PROOF === '1'
|
||||
const WATCHER_REFUSAL = /^node-pty: exit_watcher_thread failed: .*\(errno 11\)/
|
||||
const FORK_REFUSAL = /^node-pty: forkpty failed: .*\(errno 11\)/
|
||||
|
||||
function succeeds(command: string, args: string[]): boolean {
|
||||
try {
|
||||
execFileSync(command, args, { stdio: 'ignore' })
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function canCapTasks(): boolean {
|
||||
return (
|
||||
process.platform === 'linux' &&
|
||||
process.getuid?.() !== 0 &&
|
||||
succeeds('prlimit', ['--version']) &&
|
||||
succeeds('ps', ['-L', '-p', String(process.pid), '--no-headers'])
|
||||
)
|
||||
}
|
||||
|
||||
function isRefusal(value: unknown): value is { spawned: false; message: string } {
|
||||
return (
|
||||
typeof value === 'object' &&
|
||||
value !== null &&
|
||||
'spawned' in value &&
|
||||
value.spawned === false &&
|
||||
'message' in value &&
|
||||
typeof value.message === 'string'
|
||||
)
|
||||
}
|
||||
|
||||
/** Every task this user runs right now; RLIMIT_NPROC compares new tasks against this count. */
|
||||
function userTaskCount(): number {
|
||||
const listing = execFileSync('ps', ['-L', '-u', String(process.getuid?.()), '--no-headers'], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
return listing.split('\n').filter((line) => line.trim().length > 0).length
|
||||
}
|
||||
|
||||
// The child warms libuv's thread pool before any cap, so only the PTY spawn needs a new task.
|
||||
const CHILD = `
|
||||
const pty = require(process.argv[1])
|
||||
const fs = require('node:fs')
|
||||
const readline = require('node:readline')
|
||||
fs.readFile('/proc/self/status', () => {
|
||||
process.stdout.write('ready\\n')
|
||||
readline.createInterface({ input: process.stdin }).on('line', () => {
|
||||
let result
|
||||
try {
|
||||
const term = pty.spawn('/bin/sh', ['-c', 'exit 0'], { cols: 80, rows: 24, cwd: '/' })
|
||||
term.kill()
|
||||
result = { spawned: true }
|
||||
} catch (error) {
|
||||
result = { spawned: false, message: String(error && error.message) }
|
||||
}
|
||||
process.stdout.write(JSON.stringify(result) + '\\n')
|
||||
})
|
||||
})
|
||||
`
|
||||
|
||||
describe.skipIf(!REQUIRED)('a PTY spawn at the process limit (Linux)', () => {
|
||||
let child: ChildProcessWithoutNullStreams | null = null
|
||||
afterEach(() => {
|
||||
child?.kill('SIGKILL')
|
||||
child = null
|
||||
})
|
||||
|
||||
it(
|
||||
'fails that spawn with the process-limit hint and keeps the process alive',
|
||||
async () => {
|
||||
expect(canCapTasks(), 'needs Linux, a non-root user, prlimit and procps ps').toBe(true)
|
||||
const ptyPath = createRequire(import.meta.url).resolve('node-pty')
|
||||
// The Node binary this node-pty build was made for; vitest itself may run under Bun.
|
||||
const node = process.versions.bun ? 'node' : process.execPath
|
||||
child = spawn(node, ['-e', CHILD, ptyPath], { stdio: ['pipe', 'pipe', 'pipe'] })
|
||||
const running = child
|
||||
const lines = createInterface({ input: running.stdout })[Symbol.asyncIterator]()
|
||||
const nextLine = async (): Promise<string> => {
|
||||
const { value, done } = await lines.next()
|
||||
if (done) {
|
||||
throw new Error(`child exited (${running.exitCode ?? running.signalCode})`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
expect(await nextLine()).toBe('ready')
|
||||
|
||||
const refusals: string[] = []
|
||||
// A rising task count refuses the fork instead; keep trying until the watcher thread is refused.
|
||||
for (
|
||||
let attempt = 0;
|
||||
attempt < 40 && !refusals.some((m) => WATCHER_REFUSAL.test(m));
|
||||
attempt++
|
||||
) {
|
||||
const cap = userTaskCount()
|
||||
// Soft limit only: a non-root user cannot raise a lowered hard limit for the next attempt.
|
||||
execFileSync('prlimit', ['--pid', String(running.pid), `--nproc=${cap}:`])
|
||||
running.stdin.write('spawn\n')
|
||||
const result: unknown = JSON.parse(await nextLine())
|
||||
if (isRefusal(result)) {
|
||||
refusals.push(result.message)
|
||||
}
|
||||
}
|
||||
|
||||
expect(refusals.some((message) => WATCHER_REFUSAL.test(message))).toBe(true)
|
||||
for (const message of refusals) {
|
||||
expect(WATCHER_REFUSAL.test(message) || FORK_REFUSAL.test(message)).toBe(true)
|
||||
expect(addNodePtyRecoveryHint(message).startsWith(TERMINAL_PROCESS_LIMIT_HINT)).toBe(true)
|
||||
}
|
||||
// An uncaught native error would have aborted the child before it could answer.
|
||||
expect(running.exitCode).toBeNull()
|
||||
expect(running.signalCode).toBeNull()
|
||||
},
|
||||
// Up to 40 spawn rounds, each with a `ps` and a `prlimit`, on a loaded CI runner.
|
||||
60_000
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user