diff --git a/src/main/orcad/native-host-abi.test.ts b/src/main/orcad/native-host-abi.test.ts index 44f469f0faa..dfa2f7d6a7e 100644 --- a/src/main/orcad/native-host-abi.test.ts +++ b/src/main/orcad/native-host-abi.test.ts @@ -6,6 +6,8 @@ import { GLIBC_FLOOR, isBelowGlibcFloor, nativeSlotName, + parseIncompatibleArchitecture, + parseMissingSharedLibrary, parseNodeAbiMismatch, parseUnmetGlibcVersion } from './native-host-abi' @@ -97,6 +99,37 @@ describe('loader error parsing', () => { ) ).toEqual({ built: '115', host: '127' }) }) + + it('names both architectures on mach-o, and admits ELF names none', () => { + expect( + parseIncompatibleArchitecture( + "dlopen(/opt/pty.node, 0x0001): tried: '/opt/pty.node' (mach-o file, but is an incompatible architecture (have 'arm64', need 'x86_64'))" + ) + ).toEqual({ built: 'arm64', host: 'x86_64' }) + // The ELF loader refuses without saying what it found, so the verdict stands but the + // numbers do not exist to report. + expect(parseIncompatibleArchitecture('invalid ELF header')).toEqual({ + built: null, + host: null + }) + expect(parseIncompatibleArchitecture('wrong ELF class: ELFCLASS32')).not.toBeNull() + // A wrong-libc binary is not a wrong-arch binary; conflating them sends the operator + // to rebuild for an architecture that was never wrong. + expect(parseIncompatibleArchitecture("version `GLIBC_2.34' not found")).toBeNull() + }) + + it('names the shared object the loader could not open, on either loader', () => { + expect( + parseMissingSharedLibrary( + 'libstdc++.so.6: cannot open shared object file: No such file or directory' + ) + ).toBe('libstdc++.so.6') + expect(parseMissingSharedLibrary('Library not loaded: /usr/local/lib/libfoo.dylib')).toBe( + '/usr/local/lib/libfoo.dylib' + ) + // A symbol version that is absent is a rebuild, not an install: must not match here. + expect(parseMissingSharedLibrary("/lib/libc.so.6: version `GLIBC_2.34' not found")).toBeNull() + }) }) describe('detectNativeHostAbi', () => { diff --git a/src/main/orcad/native-host-abi.ts b/src/main/orcad/native-host-abi.ts index 2890bb07a15..2335ad3274b 100644 --- a/src/main/orcad/native-host-abi.ts +++ b/src/main/orcad/native-host-abi.ts @@ -121,3 +121,40 @@ export function parseNodeAbiMismatch(loaderError: string): { built: string; host const match = loaderError.match(/NODE_MODULE_VERSION\s+(\d+)\D+NODE_MODULE_VERSION\s+(\d+)/) return match ? { built: match[1], host: match[2] } : null } + +/** + * The architecture the loader refused, e.g. `incompatible architecture (have 'arm64', + * need 'x86_64')` -> { built: 'arm64', host: 'x86_64' }. ELF hosts name no architecture + * (`invalid ELF header`, `wrong ELF class: ELFCLASS32`), so both sides read null there — + * the verdict still holds, only the numbers are missing. + */ +export function parseIncompatibleArchitecture( + loaderError: string +): { built: string | null; host: string | null } | null { + const machO = loaderError.match( + /incompatible architecture \(have '?([\w.]+)'?,?\s*need '?([\w.]+)'?/ + ) + if (machO) { + return { built: machO[1], host: machO[2] } + } + if (/invalid ELF header|wrong ELF class|Exec format error|ELFCLASS(?:32|64)/.test(loaderError)) { + return { built: null, host: null } + } + return null +} + +/** + * The shared object the loader could not find, e.g. + * `libstdc++.so.6: cannot open shared object file` -> 'libstdc++.so.6'. + * + * Kept apart from the glibc floor deliberately: a missing library can be installed, + * whereas a symbol version that does not exist can only be fixed by rebuilding. + */ +export function parseMissingSharedLibrary(loaderError: string): string | null { + const elf = loaderError.match(/([\w.+-]+\.so[\w.]*): cannot open shared object file/) + if (elf) { + return elf[1] + } + const machO = loaderError.match(/Library not loaded:\s*(\S+)/) + return machO ? machO[1] : null +} diff --git a/src/main/orcad/node-pty-loader-diagnosis.ts b/src/main/orcad/node-pty-loader-diagnosis.ts new file mode 100644 index 00000000000..8024a8e0ecb --- /dev/null +++ b/src/main/orcad/node-pty-loader-diagnosis.ts @@ -0,0 +1,85 @@ +/** + * Read a dynamic-loader message and name the one thing that has to change. + * + * Pure on purpose: every shape that matters here belongs to a host we are not — Alpine, + * Ubuntu 20.04, an arm64 box handed an x64 binary — so the classification has to be + * testable from a machine that cannot reproduce any of them. + * + * Shared by the two places a node-pty load can fail: `orcad`'s boot precondition + * (out of process, before anything requires node-pty) and the SSH relay's spawn path. + */ +import type { RuntimeTerminalUnavailableReason } from '../../shared/runtime-types' +import { + parseIncompatibleArchitecture, + parseMissingSharedLibrary, + parseNodeAbiMismatch, + parseUnmetGlibcVersion +} from './native-host-abi' + +export type NodePtyLoadCause = { + reason: RuntimeTerminalUnavailableReason + /** Short human phrase naming the actual values found, not a remedy. */ + detail: string +} + +/** + * node-pty's own loader walks several directories and rethrows only the LAST failure, + * wrapped in this sentence. The tail is therefore the `prebuilds/-` + * miss — `Cannot find module` — even when the real failure was the dynamic loader + * refusing `build/Release/pty.node`. Anything acting on that tail sends the operator to + * install a module that is already installed. + */ +export function isFlattenedNodePtyLoaderMessage(message: string): boolean { + return /Failed to load native module: (?:conpty|pty)\.node(?:,|:|$)/.test(message) +} + +/** The real cause a flattened message still carries, when the last attempt was the telling one. */ +export function classifyNodePtyLoaderMessage(message: string): NodePtyLoadCause { + const abiMismatch = parseNodeAbiMismatch(message) + if (abiMismatch) { + return { + reason: 'abi_mismatch', + detail: `built for Node ABI ${abiMismatch.built}, this host runs ABI ${abiMismatch.host}` + } + } + const unmetGlibc = parseUnmetGlibcVersion(message) + if (unmetGlibc) { + return { reason: 'libc_floor', detail: `the binary requires GLIBC_${unmetGlibc}` } + } + const unmetCxx = message.match(/((?:GLIBCXX_|CXXABI_)[0-9.]+)'? not found/) + if (unmetCxx) { + return { reason: 'libc_floor', detail: `the binary requires ${unmetCxx[1]}` } + } + const arch = parseIncompatibleArchitecture(message) + if (arch) { + return { + reason: 'arch_mismatch', + detail: + arch.built && arch.host + ? `built for ${arch.built}, this host needs ${arch.host}` + : `the loader rejected the binary's format (${firstErrorLine(message)})` + } + } + const missingLibrary = parseMissingSharedLibrary(message) + if (missingLibrary) { + return { + reason: 'shared_library_missing', + detail: `${missingLibrary} is not installed on this host` + } + } + if (/MODULE_NOT_FOUND|Cannot find module/.test(message)) { + return { reason: 'dependency_missing', detail: firstErrorLine(message) } + } + return { reason: 'load_failed', detail: firstErrorLine(message) } +} + +/** + * Why not simply the first non-empty line: when a child dies without catching, node + * prints the offending source line and a caret before the error, so line one is the + * script rather than the diagnosis. Prefer the first line that reads as an error. + */ +export function firstErrorLine(text: string): string { + const lines = text.split('\n').filter((candidate) => candidate.trim().length > 0) + const errorLine = lines.find((candidate) => /^[A-Za-z]*(Error|Exception):/.test(candidate.trim())) + return (errorLine ?? lines[0] ?? text).trim().slice(0, 400) +} diff --git a/src/main/orcad/node-pty-precondition.ts b/src/main/orcad/node-pty-precondition.ts index 2857449c736..7d633bd0f9a 100644 --- a/src/main/orcad/node-pty-precondition.ts +++ b/src/main/orcad/node-pty-precondition.ts @@ -25,13 +25,8 @@ import { parseBuildToolchainProbe, toolchainInstallHintLines } from '../ssh/build-toolchain-diagnosis' -import { - detectNativeHostAbi, - nativeSlotName, - parseNodeAbiMismatch, - parseUnmetGlibcVersion, - type NativeHostAbi -} from './native-host-abi' +import { detectNativeHostAbi, nativeSlotName, type NativeHostAbi } from './native-host-abi' +import { classifyNodePtyLoaderMessage, firstErrorLine } from './node-pty-loader-diagnosis' import { installPrebuiltSlot, type PrebuiltSlotOutcome } from './node-pty-prebuilt-slot' // Why every verdict travels on STDOUT: node echoes the whole `-e` source into stderr @@ -72,21 +67,35 @@ export type NodePtyProbeFailure = { } /** - * Read the child's exit into a cause. Pure, so every failure shape is testable from a - * host that cannot reproduce it — the whole point, since the shapes that matter belong - * to Alpine and Ubuntu 20.04. + * What the child actually reported, before any judgement is made about it. + * + * Split from the classification so callers that need the loader's own words — the relay, + * which quotes them back when nothing recognizes the shape — do not have to re-derive + * them from a formatted verdict. */ -export function classifyNodePtyProbeResult( +export type NodePtyProbeOutcome = + | { kind: 'loaded'; loadedDir: string | null } + | { kind: 'noBinary' } + | { kind: 'loaderError'; message: string } + | { kind: 'signalled'; signal: NodeJS.Signals } + /** The probe never answered. Not evidence about node-pty either way. */ + | { kind: 'unanswered'; detail: string } + /** It answered, but with nothing that names a cause. */ + | { kind: 'unexplained'; detail: string } + +export function readNodePtyProbeOutcome( result: Pick -): NodePtyProbeFailure | null { +): NodePtyProbeOutcome { const stdout = result.stdout if (result.code === 0 && stdout.includes(PROBE_OK_TOKEN)) { - return null + return { + kind: 'loaded', + loadedDir: stdout.split(PROBE_OK_TOKEN)[1]?.trim().split('\n')[0]?.trim() || null + } } if (result.timedOut) { return { - status: 'unverifiable', - reason: 'unknown', + kind: 'unanswered', detail: 'the node-pty load probe did not finish in time, so nothing was established' } } @@ -94,27 +103,51 @@ export function classifyNodePtyProbeResult( // the loader never reaches the catch, and often prints nothing at all. That silence is // exactly the uncatchable case this probe is a separate process for. if (result.signal) { - return { - status: 'blocked', - reason: 'load_crashed', - detail: `the load probe was killed by ${result.signal}` - } + return { kind: 'signalled', signal: result.signal } } if (stdout.includes(NO_BINARY_TOKEN)) { - return { - status: 'blocked', - reason: 'dependency_missing', - detail: 'node-pty is installed but has no compiled binary for this platform' - } + return { kind: 'noBinary' } } const reported = readReportedLoadError(stdout) if (reported !== null) { - return classifyLoaderMessage(reported) + return { kind: 'loaderError', message: reported } } return { - status: 'blocked', - reason: 'load_failed', - detail: firstLine(result.stderr) || `the load probe exited with code ${result.code}` + kind: 'unexplained', + detail: firstErrorLine(result.stderr) || `the load probe exited with code ${result.code}` + } +} + +/** + * Read the child's exit into a cause. Pure, so every failure shape is testable from a + * host that cannot reproduce it — the whole point, since the shapes that matter belong + * to Alpine and Ubuntu 20.04. + */ +export function classifyNodePtyProbeResult( + result: Pick +): NodePtyProbeFailure | null { + const outcome = readNodePtyProbeOutcome(result) + switch (outcome.kind) { + case 'loaded': + return null + case 'unanswered': + return { status: 'unverifiable', reason: 'unknown', detail: outcome.detail } + case 'signalled': + return { + status: 'blocked', + reason: 'load_crashed', + detail: `the load probe was killed by ${outcome.signal}` + } + case 'noBinary': + return { + status: 'blocked', + reason: 'dependency_missing', + detail: 'node-pty is installed but has no compiled binary for this platform' + } + case 'loaderError': + return classifyLoaderMessage(outcome.message) + case 'unexplained': + return { status: 'blocked', reason: 'load_failed', detail: outcome.detail } } } @@ -133,40 +166,7 @@ function readReportedLoadError(stdout: string): string | null { /** Read a dynamic-loader message. Pure, so shapes this host cannot reproduce are testable. */ export function classifyLoaderMessage(message: string): NodePtyProbeFailure { - const abiMismatch = parseNodeAbiMismatch(message) - if (abiMismatch) { - return { - status: 'blocked', - reason: 'abi_mismatch', - detail: `built for Node ABI ${abiMismatch.built}, this host runs ABI ${abiMismatch.host}` - } - } - const unmetGlibc = parseUnmetGlibcVersion(message) - if (unmetGlibc) { - return { - status: 'blocked', - reason: 'libc_floor', - detail: `the binary requires GLIBC_${unmetGlibc}` - } - } - if (/(GLIBCXX_|CXXABI_)[0-9.]+'? not found/.test(message)) { - return { status: 'blocked', reason: 'libc_floor', detail: firstLine(message) } - } - if (/MODULE_NOT_FOUND|Cannot find module/.test(message)) { - return { status: 'blocked', reason: 'dependency_missing', detail: firstLine(message) } - } - return { status: 'blocked', reason: 'load_failed', detail: firstLine(message) } -} - -/** - * Why not simply the first non-empty line: when the child dies without catching, node - * prints the offending source line and a caret before the error, so line one is the - * script rather than the diagnosis. Prefer the first line that reads as an error. - */ -function firstLine(text: string): string { - const lines = text.split('\n').filter((candidate) => candidate.trim().length > 0) - const errorLine = lines.find((candidate) => /^[A-Za-z]*(Error|Exception):/.test(candidate.trim())) - return (errorLine ?? lines[0] ?? text).trim().slice(0, 400) + return { status: 'blocked', ...classifyNodePtyLoaderMessage(message) } } /** @@ -305,7 +305,8 @@ export function checkNodePtyPrecondition( // Loaded. The remaining way terminals fail is spawn-time: node-pty posix_spawns // build/Release/spawn-helper, and a missing one turns every terminal.create into ENOENT // on a host that otherwise looks healthy. That is a degradation, not a boot blocker. - const loadedDir = result.stdout.split(PROBE_OK_TOKEN)[1]?.trim().split('\n')[0]?.trim() + const outcome = readNodePtyProbeOutcome(result) + const loadedDir = outcome.kind === 'loaded' ? outcome.loadedDir : null if (abi.platform !== 'win32') { const helper = join(loadedDir || join(nodePtyDir, 'build', 'Release'), 'spawn-helper') if (!isExecutableFile(helper)) { diff --git a/src/relay/dispatcher-rpc-routing.ts b/src/relay/dispatcher-rpc-routing.ts index c30d222ff3d..164a375e1c6 100644 --- a/src/relay/dispatcher-rpc-routing.ts +++ b/src/relay/dispatcher-rpc-routing.ts @@ -3,6 +3,10 @@ import { SKILL_INSTALL_RPC_ERROR_CODE, SkillInstallFailureSchema } from '../shared/skill-install-failure' +import { + TERMINAL_UNAVAILABLE_RPC_ERROR_CODE, + TerminalUnavailableCauseSchema +} from '../shared/terminal-unavailable-cause' import { RelayErrorCode, type JsonRpcNotification, @@ -135,11 +139,16 @@ export abstract class RelayDispatcherRpcRouting extends RelayDispatcherFrameCode const message = err instanceof Error ? err.message : String(err) const errorCode = (err as { code?: unknown }).code const code = typeof errorCode === 'number' ? errorCode : -32000 - const skillFailure = + // Why an allowlist keyed on the error code: error `data` is otherwise dropped, so a + // handler cannot leak internals by attaching them. Each published shape is validated + // against its own schema before it crosses. + const structured = errorCode === SKILL_INSTALL_RPC_ERROR_CODE ? SkillInstallFailureSchema.safeParse((err as { data?: unknown }).data) - : null - const data = skillFailure?.success === true ? skillFailure.data : undefined + : errorCode === TERMINAL_UNAVAILABLE_RPC_ERROR_CODE + ? TerminalUnavailableCauseSchema.safeParse((err as { data?: unknown }).data) + : null + const data = structured?.success === true ? structured.data : undefined const accepted = this.sendResponse( client, req.id, diff --git a/src/relay/dispatcher-structured-error.test.ts b/src/relay/dispatcher-structured-error.test.ts index 0ba8e164216..3ce0a79f19f 100644 --- a/src/relay/dispatcher-structured-error.test.ts +++ b/src/relay/dispatcher-structured-error.test.ts @@ -1,6 +1,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { RelayDispatcher } from './dispatcher' import { encodeJsonRpcFrame, MessageType, type JsonRpcResponse } from './protocol' +import { + TERMINAL_UNAVAILABLE_RPC_ERROR_CODE, + type TerminalUnavailableCause +} from '../shared/terminal-unavailable-cause' function decodeResponse(frame: Buffer): JsonRpcResponse | null { if (frame[0] !== MessageType.Regular) { @@ -52,4 +56,54 @@ describe('RelayDispatcher structured errors', () => { message: 'boom' }) }) + + it('carries a terminal-unavailable cause across the wire, and rejects a malformed one', async () => { + // Why this must cross: the fault is proved on the relay at spawn time, and the only + // machinery that can repair it runs on the client. Prose cannot be acted on. + vi.useFakeTimers() + const written: Buffer[] = [] + const dispatcher = new RelayDispatcher((data) => { + written.push(Buffer.from(data)) + }) + dispatchers.push(dispatcher) + const cause: TerminalUnavailableCause = { + status: 'blocked', + reason: 'abi_mismatch', + detail: 'built for Node ABI 127, this host runs ABI 115', + repairable: true, + host: { + platform: 'linux', + arch: 'x64', + libc: 'glibc', + glibcVersion: '2.31', + nodeAbi: '115', + nodeVersion: 'v20.11.0' + } + } + dispatcher.onRequest('pty.spawn', async () => { + throw Object.assign(new Error('Remote terminals are unavailable'), { + code: TERMINAL_UNAVAILABLE_RPC_ERROR_CODE, + data: cause + }) + }) + dispatcher.onRequest('pty.spawnBogus', async () => { + throw Object.assign(new Error('Remote terminals are unavailable'), { + code: TERMINAL_UNAVAILABLE_RPC_ERROR_CODE, + data: { status: 'blocked', repairable: true } + }) + }) + + dispatcher.feed(encodeJsonRpcFrame({ jsonrpc: '2.0', id: 8, method: 'pty.spawn' }, 1, 0)) + dispatcher.feed(encodeJsonRpcFrame({ jsonrpc: '2.0', id: 9, method: 'pty.spawnBogus' }, 2, 0)) + await vi.advanceTimersByTimeAsync(0) + + const responses = written.map(decodeResponse) + expect(responses.find((message) => message?.id === 8)?.error?.data).toEqual(cause) + // A cause that does not validate is dropped entirely; a half-read cause must never + // authorize a repair. + expect(responses.find((message) => message?.id === 9)?.error).toEqual({ + code: -32000, + message: 'Remote terminals are unavailable' + }) + }) }) diff --git a/src/relay/node-pty-binding-survey.test.ts b/src/relay/node-pty-binding-survey.test.ts new file mode 100644 index 00000000000..b51a57fd4a8 --- /dev/null +++ b/src/relay/node-pty-binding-survey.test.ts @@ -0,0 +1,138 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { afterEach, describe, expect, it } from 'vitest' +import { isFlattenedNodePtyLoaderMessage } from '../main/orcad/node-pty-loader-diagnosis' +import { + collectNodePtyUnavailableDiagnosis, + readNodeGypBuildRecord, + surveyNodePtyBinding +} from './node-pty-binding-survey' +import { formatNodePtyUnavailableMessage } from './node-pty-unavailable-diagnosis' + +const HOST = { platform: process.platform, arch: process.arch } +/** What node-pty throws once its loader has replaced the real cause with its last miss. */ +const FLATTENED = + 'Failed to load native module: pty.node, checked: build/Release, build/Debug, ' + + `prebuilds/${process.platform}-${process.arch}: Error: Cannot find module './pty.node'` + +const roots: string[] = [] + +function fixture(options: { binding?: boolean; configGypi?: string } = {}): string { + const root = mkdtempSync(join(tmpdir(), 'orca-node-pty-')) + roots.push(root) + const dir = join(root, 'node-pty') + mkdirSync(join(dir, 'lib'), { recursive: true }) + writeFileSync(join(dir, 'lib', 'index.js'), 'module.exports = {}\n') + writeFileSync(join(dir, 'lib', 'utils.js'), 'exports.loadNativeModule = () => ({})\n') + if (options.binding) { + mkdirSync(join(dir, 'build', 'Release'), { recursive: true }) + // Deliberately not a valid addon: the point is to make the dynamic loader talk. + for (const name of ['pty.node', 'conpty.node']) { + writeFileSync(join(dir, 'build', 'Release', name), 'not an addon\n') + } + } + if (options.configGypi !== undefined) { + mkdirSync(join(dir, 'build'), { recursive: true }) + writeFileSync(join(dir, 'build', 'config.gypi'), options.configGypi) + } + return dir +} + +afterEach(() => { + while (roots.length > 0) { + rmSync(roots.pop()!, { recursive: true, force: true }) + } +}) + +describe('surveyNodePtyBinding', () => { + it('finds the file node-pty itself would open, and lists where it looked when there is none', () => { + const withBinding = surveyNodePtyBinding(fixture({ binding: true }), HOST) + expect(withBinding?.bindingPath).toMatch(/build[/\\]Release[/\\](con)?pty\.node$/) + + const without = surveyNodePtyBinding(fixture(), HOST) + expect(without?.bindingPath).toBeNull() + expect(without?.searched).toHaveLength(3) + expect(without?.searched.join(' ')).toContain(`prebuilds/${process.platform}-${process.arch}`) + }) +}) + +describe('readNodeGypBuildRecord', () => { + it('reads what node-gyp configured for, past its leading comment lines', () => { + const dir = fixture({ + configGypi: + '# Do not edit. File was generated by node-gyp\'s "configure" step\n' + + '{ "variables": { "node_module_version": 127, "target_arch": "arm64" } }\n' + }) + expect(readNodeGypBuildRecord(dir)).toEqual({ nodeAbi: '127', arch: 'arm64' }) + }) + + it('answers nothing rather than guessing when there is no build record', () => { + expect(readNodeGypBuildRecord(fixture())).toEqual({ nodeAbi: null, arch: null }) + }) +}) + +describe('collectNodePtyUnavailableDiagnosis', () => { + it("recovers the dynamic loader's own words that node-pty threw away", async () => { + // The whole defect in one assertion: what reaches the relay is FLATTENED, which names + // no cause; the diagnosis must carry what the loader actually said about the file. + const diagnosis = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: fixture({ binding: true }), + error: new Error(FLATTENED) + }) + expect(diagnosis.status).toBe('blocked') + expect(diagnosis.rawError).toBeTruthy() + expect(isFlattenedNodePtyLoaderMessage(diagnosis.rawError!)).toBe(false) + expect(diagnosis.rawError).not.toBe(FLATTENED) + expect(diagnosis.rawError).toMatch(/pty\.node/) + }, 20_000) + + it("prefers node-gyp's build record over a loader message that named no fault", async () => { + const diagnosis = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: fixture({ + binding: true, + configGypi: '{ "variables": { "node_module_version": 4242, "target_arch": "x64" } }' + }), + error: new Error(FLATTENED) + }) + // A garbage binary reads differently per platform (mach-o vs ELF), so only assert the + // build record is consulted when the loader message did not name the fault itself. + if (diagnosis.reason === 'abi_mismatch') { + expect(formatNodePtyUnavailableMessage(diagnosis)).toContain( + `built for Node ABI 4242, this host runs ABI ${process.versions.modules}` + ) + } else { + expect(['arch_mismatch', 'load_failed', 'load_crashed']).toContain(diagnosis.reason) + } + }, 20_000) + + it('reports an unlocatable install as unverifiable, not as a diagnosis', async () => { + const diagnosis = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: null, + error: new Error(FLATTENED) + }) + expect(diagnosis.status).toBe('unverifiable') + const text = formatNodePtyUnavailableMessage(diagnosis) + expect(text).toContain('could not establish why') + // It still has to be reportable: the raw error is the only thing an issue can quote. + expect(text).toContain(FLATTENED) + }) + + it('probes the host toolchain only when nothing was compiled', async () => { + const diagnosis = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: fixture(), + error: new Error(FLATTENED) + }) + expect(diagnosis.survey?.bindingPath).toBeNull() + expect(['toolchain_missing', 'dependency_missing']).toContain(diagnosis.reason) + // Non-Linux hosts ship a node-pty prebuild, so a toolchain answer there would be noise. + expect(diagnosis.toolchain === null).toBe(process.platform !== 'linux') + + const compiled = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: fixture({ binding: true }), + error: new Error(FLATTENED) + }) + expect(compiled.toolchain).toBeNull() + }, 20_000) +}) diff --git a/src/relay/node-pty-binding-survey.ts b/src/relay/node-pty-binding-survey.ts new file mode 100644 index 00000000000..805527f1aad --- /dev/null +++ b/src/relay/node-pty-binding-survey.ts @@ -0,0 +1,215 @@ +/** + * Gather the evidence a node-pty spawn failure needs, on the host that failed. + * + * Three sources, because no single one is sufficient: + * + * 1. What is on disk where node-pty's loader looks, and what node-gyp recorded it was + * configured for (`build/config.gypi`). This answers "wrong ABI / wrong arch" even + * when the loader said nothing useful, and it is the only source available when the + * binding is absent entirely. + * 2. The dynamic loader's own words, recovered by dlopen'ing the file node-pty would + * have opened. node-pty's loader rethrows only its LAST attempt, so the real message + * is otherwise destroyed before the relay sees it. This runs in a CHILD process: a + * binding that aborts inside the loader would take the relay down with it, and a + * relay that dies is a reconnect loop rather than an error message. + * 3. The host's C/C++ toolchain, but only when nothing was compiled — "install + * build-essential" is the right answer for a compile that never ran, and noise for a + * binary that exists and is simply wrong. + * + * Every step is best-effort and failure-tolerant: whatever cannot be established is + * reported as unestablished rather than guessed (docs/reference/ssh-execution-boundary.md). + */ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { release } from 'node:os' +import process from 'node:process' +import { runProcess } from '../shared/child-process/run-process' +import { + buildToolchainProbeCommand, + parseBuildToolchainProbe, + type BuildToolchainStatus +} from '../main/ssh/build-toolchain-diagnosis' +import { detectNativeHostAbi } from '../main/orcad/native-host-abi' +import { + buildNodePtyLoadProbeScript, + readNodePtyProbeOutcome +} from '../main/orcad/node-pty-precondition' +import { + diagnoseNodePtyUnavailable, + type NodePtyBindingSurvey, + type NodePtyDiagnosisInput, + type NodePtyUnavailableDiagnosis, + type NodePtyUnavailableHost +} from './node-pty-unavailable-diagnosis' + +/** Bounded so a wedged loader delays one spawn rejection, not the relay. */ +const LOAD_PROBE_TIMEOUT_MS = 10_000 +const TOOLCHAIN_PROBE_TIMEOUT_MS = 5_000 + +/** node-pty's own search order, so the file surveyed is the file it would have opened. */ +function bindingSearchDirs(platform: NodeJS.Platform, arch: string): string[] { + return ['build/Release', 'build/Debug', `prebuilds/${platform}-${arch}`] +} + +/** Windows defers to conpty.node on builds that have ConPTY, exactly as node-pty picks it. */ +function bindingBaseName(platform: NodeJS.Platform): string { + if (platform !== 'win32') { + return 'pty' + } + return Number(release().split('.')[2]) >= 18309 ? 'conpty' : 'pty' +} + +export function surveyNodePtyBinding( + nodePtyDir: string, + host: Pick +): NodePtyBindingSurvey | null { + const name = bindingBaseName(host.platform) + const searched = bindingSearchDirs(host.platform, host.arch) + let bindingPath: string | null = null + try { + for (const dir of searched) { + for (const root of [nodePtyDir, join(nodePtyDir, 'lib')]) { + const candidate = join(root, dir, `${name}.node`) + if (existsSync(candidate)) { + bindingPath = candidate + break + } + } + if (bindingPath) { + break + } + } + } catch { + return null + } + const built = readNodeGypBuildRecord(nodePtyDir) + return { + moduleDir: nodePtyDir, + bindingPath, + searched, + builtNodeAbi: built.nodeAbi, + builtArch: built.arch + } +} + +/** + * What node-gyp configured this build for. + * + * Why this file and not the binary: `build/config.gypi` is written by `node-gyp + * configure` from the headers it downloaded, so it names the ABI and architecture the + * `.node` was compiled against without parsing ELF. It survives a build that later + * failed, which is the case where the loader has nothing to say. + */ +export function readNodeGypBuildRecord(nodePtyDir: string): { + nodeAbi: string | null + arch: string | null +} { + try { + const raw = readFileSync(join(nodePtyDir, 'build', 'config.gypi'), 'utf8') + // node-gyp prefixes the JSON with `# Do not edit…` comment lines. + const body = raw + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + .join('\n') + const variables = (JSON.parse(body) as { variables?: Record }).variables + const nodeAbi = variables?.node_module_version + const arch = variables?.target_arch + return { + nodeAbi: nodeAbi === undefined || nodeAbi === null ? null : String(nodeAbi), + arch: typeof arch === 'string' && arch.length > 0 ? arch : null + } + } catch { + return { nodeAbi: null, arch: null } + } +} + +/** + * The loader's verdict on the binding, recovered out of process. + * + * Returns the pieces `diagnoseNodePtyUnavailable` reads; a probe that could not run + * answers `unverifiableBecause` rather than a cause, because it established nothing. + */ +async function probeNodePtyLoader( + nodePtyDir: string +): Promise> { + let result + try { + result = await runProcess({ + program: process.execPath, + args: ['-e', buildNodePtyLoadProbeScript(nodePtyDir)], + timeoutMs: LOAD_PROBE_TIMEOUT_MS + }) + } catch (error) { + return { + unverifiableBecause: `the node-pty load probe could not be started (${(error as Error).message})` + } + } + const outcome = readNodePtyProbeOutcome(result) + switch (outcome.kind) { + case 'loaderError': + return { loaderError: outcome.message } + case 'signalled': + return { probeSignal: outcome.signal } + case 'unanswered': + return { unverifiableBecause: outcome.detail } + // `loaded` here means the binding is fine under plain Node while the relay's own + // require failed — real, and not something the loader can explain. `noBinary` and + // `unexplained` are both better answered by the on-disk survey than by the probe. + case 'loaded': + case 'noBinary': + case 'unexplained': + return {} + } +} + +/** node-pty has no Linux prebuild, so only there does a missing toolchain explain anything. */ +async function probeRelayBuildToolchain( + platform: NodeJS.Platform +): Promise { + if (platform !== 'linux') { + return null + } + try { + const result = await runProcess({ + program: '/bin/sh', + args: ['-c', buildToolchainProbeCommand()], + timeoutMs: TOOLCHAIN_PROBE_TIMEOUT_MS + }) + return result.timedOut ? null : parseBuildToolchainProbe(result.stdout) + } catch { + return null + } +} + +function readErrorMessage(error: unknown): string | null { + if (error instanceof Error) { + return error.message + } + return typeof error === 'string' && error.length > 0 ? error : null +} + +/** + * Everything above, in the order that makes each step's cost conditional on the previous + * one's answer. Called only on the failure path, so a spawn that works pays nothing. + */ +export async function collectNodePtyUnavailableDiagnosis(options: { + nodePtyDir: string | null + error?: unknown +}): Promise { + const abi = detectNativeHostAbi() + const host: NodePtyUnavailableHost = { ...abi, nodeVersion: process.version } + const requireError = readErrorMessage(options.error) + if (!options.nodePtyDir) { + return diagnoseNodePtyUnavailable({ + host, + survey: null, + requireError, + unverifiableBecause: 'the relay could not locate its node-pty install directory' + }) + } + const survey = surveyNodePtyBinding(options.nodePtyDir, host) + const probed = survey?.bindingPath ? await probeNodePtyLoader(options.nodePtyDir) : {} + const toolchain = + survey && !survey.bindingPath ? await probeRelayBuildToolchain(host.platform) : null + return diagnoseNodePtyUnavailable({ ...probed, host, survey, requireError, toolchain }) +} diff --git a/src/relay/node-pty-unavailable-diagnosis.test.ts b/src/relay/node-pty-unavailable-diagnosis.test.ts new file mode 100644 index 00000000000..7bed6ef256e --- /dev/null +++ b/src/relay/node-pty-unavailable-diagnosis.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from 'vitest' +import { + diagnoseNodePtyUnavailable, + formatNodePtyUnavailableMessage, + type NodePtyBindingSurvey, + type NodePtyDiagnosisInput, + toTerminalUnavailableCause, + type NodePtyUnavailableHost +} from './node-pty-unavailable-diagnosis' +import { parseBuildToolchainProbe } from '../main/ssh/build-toolchain-diagnosis' +import { + mayRepairFromCause, + parseTerminalUnavailableCause +} from '../shared/terminal-unavailable-cause' + +const UBUNTU_2004: NodePtyUnavailableHost = { + platform: 'linux', + arch: 'x64', + libc: 'glibc', + glibcVersion: '2.31', + nodeAbi: '115', + nodeVersion: 'v20.11.0' +} + +const MODULE_DIR = '/opt/orca/relay/node_modules/node-pty' +const SEARCHED = ['build/Release', 'build/Debug', 'prebuilds/linux-x64'] + +const INSTALLED: NodePtyBindingSurvey = { + moduleDir: MODULE_DIR, + bindingPath: `${MODULE_DIR}/build/Release/pty.node`, + searched: SEARCHED, + builtNodeAbi: null, + builtArch: null +} + +const NOTHING_INSTALLED: NodePtyBindingSurvey = { ...INSTALLED, bindingPath: null } + +/** What node-pty itself throws: the real cause replaced by its LAST directory miss. */ +const FLATTENED = + 'Failed to load native module: pty.node, checked: build/Release, build/Debug, ' + + "prebuilds/linux-x64: Error: Cannot find module '../prebuilds/linux-x64//pty.node'" + +const diagnose = (overrides: Partial = {}) => + diagnoseNodePtyUnavailable({ + host: UBUNTU_2004, + survey: INSTALLED, + requireError: FLATTENED, + ...overrides + }) + +const message = (overrides: Partial = {}) => + formatNodePtyUnavailableMessage(diagnose(overrides)) + +const toolchain = (present: readonly string[]) => + parseBuildToolchainProbe([...present.map((tool) => `HAVE ${tool}`), 'PKG apt-get'].join('\n')) + +describe('diagnoseNodePtyUnavailable', () => { + it("never treats node-pty's flattened wrapper as the cause", () => { + // node-pty rethrows only its last directory miss, so acting on that text sends the + // user to install a module that is already installed. + expect( + diagnose({ survey: NOTHING_INSTALLED, toolchain: toolchain(['make', 'g++', 'python3']) }) + ).toMatchObject({ reason: 'dependency_missing' }) + expect(diagnose().reason).not.toBe('dependency_missing') + }) + + it('names the glibc the host actually has next to the one the binary needs', () => { + const verdict = diagnose({ + loaderError: + "/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /opt/orca/node_modules/node-pty/build/Release/pty.node)" + }) + expect(verdict).toMatchObject({ status: 'blocked', reason: 'libc_floor' }) + const text = formatNodePtyUnavailableMessage(verdict) + expect(text).toContain('GLIBC_2.34') + expect(text).toContain('glibc 2.31') + // The remedy is a rebuild; offering "install build-essential" here is a wrong answer. + expect(text).not.toContain('build tools') + }) + + it('names both ABI numbers from the loader message', () => { + const text = message({ + loaderError: + 'The module was compiled against a different Node.js version using NODE_MODULE_VERSION 115. ' + + 'This version of Node.js requires NODE_MODULE_VERSION 127.' + }) + expect(text).toContain('built for Node ABI 115, this host runs ABI 127') + expect(text).toContain('v20.11.0') + }) + + it("reads the ABI mismatch off node-gyp's build record when the loader said nothing", () => { + // The case the old message could only hedge about: the binding is present, node-pty + // destroyed the loader error, and the only evidence left is what node-gyp configured. + const text = message({ + survey: { ...INSTALLED, builtNodeAbi: '127' } + }) + expect(text).toContain('built for Node ABI 127, this host runs ABI 115') + }) + + it('separates an architecture mismatch from an ABI mismatch', () => { + expect(diagnose({ loaderError: 'invalid ELF header' }).reason).toBe('arch_mismatch') + expect(message({ survey: { ...INSTALLED, builtArch: 'arm64' } })).toContain( + 'built for arm64, this host runs x64' + ) + expect( + message({ + loaderError: + "dlopen(/opt/pty.node, 0x0001): tried: '/opt/pty.node' (mach-o file, but is an " + + "incompatible architecture (have 'arm64', need 'x86_64'))" + }) + ).toContain('built for arm64, this host needs x86_64') + }) + + it('separates a missing shared library from a libc floor break', () => { + // Different remedies: install a package, versus rebuild against an older toolchain. + const verdict = diagnose({ + loaderError: 'libstdc++.so.6: cannot open shared object file: No such file or directory' + }) + expect(verdict.reason).toBe('shared_library_missing') + const text = formatNodePtyUnavailableMessage(verdict) + expect(text).toContain('libstdc++.so.6 is not installed on this host') + expect(text).toContain('Install that library') + }) + + it('offers the build-tools remedy only when it probed the toolchain and found it missing', () => { + const missing = diagnose({ + survey: NOTHING_INSTALLED, + toolchain: toolchain(['python3']) + }) + expect(missing.reason).toBe('toolchain_missing') + const text = formatNodePtyUnavailableMessage(missing) + expect(text).toContain('make and a C++ compiler are not installed') + expect(text).toContain(`checked ${SEARCHED.join(', ')} under ${MODULE_DIR}`) + expect(text).toContain('sudo apt-get install -y build-essential python3') + + // Toolchain present and nothing compiled: the install failed for another reason, and + // "install make/g++/python3" would send the user chasing tools they already have. + const present = diagnose({ + survey: NOTHING_INSTALLED, + toolchain: toolchain(['make', 'g++', 'python3']) + }) + expect(present.reason).toBe('dependency_missing') + expect(formatNodePtyUnavailableMessage(present)).not.toContain('apt-get') + }) + + it('reports a binding that killed the probe as a crash rather than a miss', () => { + const text = message({ probeSignal: 'SIGSEGV' }) + expect(text).toContain('SIGSEGV') + expect(text).toContain('incompatible with this host rather than missing') + }) + + it('quotes the loader verbatim when nothing recognizes it', () => { + const raw = 'dlopen(/opt/pty.node): unexpected relocation kind 0x9f' + const verdict = diagnose({ loaderError: raw }) + expect(verdict).toMatchObject({ status: 'blocked', reason: 'load_failed', rawError: raw }) + const text = formatNodePtyUnavailableMessage(verdict) + expect(text).toContain(`Loader error: ${raw}`) + expect(text).toContain('file an issue') + }) + + it('reports a probe that never answered as unverifiable and diagnoses nothing', () => { + // docs/reference/ssh-execution-boundary.md: loss of contact is not a verdict. + const verdict = diagnose({ + unverifiableBecause: 'the node-pty load probe did not finish in time' + }) + expect(verdict.status).toBe('unverifiable') + const text = formatNodePtyUnavailableMessage(verdict) + expect(text).toContain('could not establish why') + expect(text).toContain('not evidence node-pty is broken') + expect(text).not.toContain('Reconnect to rebuild') + expect(text).not.toContain('apt-get') + }) + + it('marks rebuildable faults repairable and everything else not', () => { + // The relay's node-pty is compiled ON the remote, so a binding that no longer matches + // the machine is fixed by recompiling there. A missing compiler or a missing library + // is not: the rebuild would need the very thing that is absent. + const repairable = (overrides: Partial) => + toTerminalUnavailableCause(diagnose(overrides)).repairable + + expect(repairable({ survey: { ...INSTALLED, builtNodeAbi: '127' } })).toBe(true) + expect(repairable({ survey: { ...INSTALLED, builtArch: 'arm64' } })).toBe(true) + expect(repairable({ loaderError: "version `GLIBC_2.34' not found" })).toBe(true) + expect(repairable({ probeSignal: 'SIGSEGV' })).toBe(true) + expect( + repairable({ survey: NOTHING_INSTALLED, toolchain: toolchain(['make', 'g++', 'python3']) }) + ).toBe(true) + + expect(repairable({ survey: NOTHING_INSTALLED, toolchain: toolchain(['python3']) })).toBe(false) + expect(repairable({ loaderError: 'libstdc++.so.6: cannot open shared object file' })).toBe( + false + ) + // Nothing was established, so nothing may be rewritten on the host (#14830). + expect(repairable({ unverifiableBecause: 'probe timed out' })).toBe(false) + }) + + it('publishes a cause that survives its own wire schema', () => { + const cause = toTerminalUnavailableCause( + diagnose({ loaderError: "version `GLIBC_2.34' not found" }) + ) + expect(parseTerminalUnavailableCause(cause)).toEqual(cause) + expect(mayRepairFromCause(cause)).toBe(true) + expect(cause.host).toMatchObject({ arch: 'x64', nodeAbi: '115', glibcVersion: '2.31' }) + + // A peer claiming repairable on an unverifiable status must not be believed. + expect(mayRepairFromCause({ ...cause, status: 'unverifiable' })).toBe(false) + expect(parseTerminalUnavailableCause({ ...cause, host: undefined })).toBeNull() + // A reason this client has never heard of must not discard the whole cause; the + // relay may name faults added after the client shipped. + expect(parseTerminalUnavailableCause({ ...cause, reason: 'invented_later' })).not.toBeNull() + }) + + it('puts the host on every message so a bug report needs no follow-up question', () => { + for (const overrides of [ + {}, + { loaderError: 'invalid ELF header' }, + { unverifiableBecause: 'probe timed out' } + ]) { + expect(message(overrides)).toContain( + 'linux/x64, glibc 2.31, Node v20.11.0 (ABI 115), prebuild slot linux-x64-glibc' + ) + } + }) +}) diff --git a/src/relay/node-pty-unavailable-diagnosis.ts b/src/relay/node-pty-unavailable-diagnosis.ts new file mode 100644 index 00000000000..590eedcc375 --- /dev/null +++ b/src/relay/node-pty-unavailable-diagnosis.ts @@ -0,0 +1,362 @@ +/** + * Why the remote host cannot spawn terminals, in terms the user can act on and check. + * + * The relay used to answer this with one hedged paragraph — "install build tools, or + * else reconnect, or else check your Node version" — because the only thing it looked at + * was that `require('node-pty')` threw. That paragraph names three different remedies for + * four different faults and lets the user verify none of them. + * + * node-pty's own loader is why the raw cause went missing: it walks build/Release, + * build/Debug and prebuilds/-, then rethrows only the LAST error. So a + * `pty.node` the dynamic loader refused arrives as `Cannot find module '../prebuilds/…'`, + * and the GLIBC/ABI/arch sentence that actually says what is wrong is discarded before + * the relay ever sees it. Recovering it needs a separate dlopen of the file the loader + * would have opened — see node-pty-binding-survey.ts. + * + * Everything here is pure so every verdict is testable from a host that is none of the + * hosts that break. `unverifiable` is a first-class outcome: a probe that did not answer + * is not a diagnosis (docs/reference/ssh-execution-boundary.md). + */ +import { GLIBC_FLOOR, nativeSlotName, type NativeHostAbi } from '../main/orcad/native-host-abi' +import { + classifyNodePtyLoaderMessage, + isFlattenedNodePtyLoaderMessage +} from '../main/orcad/node-pty-loader-diagnosis' +import { + toolchainInstallHintLines, + type BuildToolchainStatus +} from '../main/ssh/build-toolchain-diagnosis' +import type { RuntimeTerminalUnavailableReason } from '../shared/runtime-types' +import type { TerminalUnavailableCause } from '../shared/terminal-unavailable-cause' + +/** What is actually on disk where node-pty's loader looks, and what it was built for. */ +export type NodePtyBindingSurvey = { + /** The node-pty install the relay would load from. */ + moduleDir: string + /** The compiled binding the loader would open, or null when no directory holds one. */ + bindingPath: string | null + /** Directories checked, so "nothing is installed" is a statement with evidence. */ + searched: string[] + /** `node_module_version` from node-gyp's build/config.gypi, when it is readable. */ + builtNodeAbi: string | null + /** `target_arch` from node-gyp's build/config.gypi, when it is readable. */ + builtArch: string | null +} + +export type NodePtyUnavailableHost = NativeHostAbi & { nodeVersion: string } + +export type NodePtyUnavailableDiagnosis = { + /** `blocked` — proved. `unverifiable` — nothing answered, which is not evidence. */ + status: 'blocked' | 'unverifiable' + reason: RuntimeTerminalUnavailableReason + host: NodePtyUnavailableHost + /** Short phrase naming the values found. */ + detail: string + /** The loader's own words, kept verbatim so an unclassified verdict is still reportable. */ + rawError: string | null + survey: NodePtyBindingSurvey | null + toolchain: BuildToolchainStatus | null +} + +export type NodePtyDiagnosisInput = { + /** What the recovered dlopen said, when one ran. Preferred over `requireError`. */ + loaderError?: string | null + /** What `require('node-pty')`/`pty.spawn` threw. Usually flattened by node-pty. */ + requireError?: string | null + /** A load probe that was killed rather than answering. */ + probeSignal?: NodeJS.Signals | null + /** Set when the load probe never answered at all; forces `unverifiable`. */ + unverifiableBecause?: string | null + host: NodePtyUnavailableHost + survey: NodePtyBindingSurvey | null + toolchain?: BuildToolchainStatus | null +} + +/** Reasons a loader message can establish on its own, and which nothing else outranks. */ +const LOADER_NAMED_FAULTS: ReadonlySet = new Set([ + 'abi_mismatch', + 'arch_mismatch', + 'libc_floor', + 'shared_library_missing' +]) + +export function diagnoseNodePtyUnavailable( + input: NodePtyDiagnosisInput +): NodePtyUnavailableDiagnosis { + const { host, survey } = input + const toolchain = input.toolchain ?? null + // Why the require error is only a fallback: node-pty flattens the real cause away, so + // its text is evidence of "did not load", never of why. + const usableRequireError = + input.requireError && !isFlattenedNodePtyLoaderMessage(input.requireError) + ? input.requireError + : null + // Capped because a macOS dlopen error lists every path it tried; the message quotes this + // verbatim when nothing classifies it, and a toast is not a log file. + const rawError = truncate(input.loaderError ?? usableRequireError ?? input.requireError ?? null) + const base = { host, rawError, survey, toolchain } as const + + if (input.unverifiableBecause) { + return { + ...base, + status: 'unverifiable', + reason: 'unknown', + detail: input.unverifiableBecause + } + } + // Before anything the loader said: a binary that aborts inside the loader never reaches + // a catch and often prints nothing, so the signal is the only evidence there is. + if (input.probeSignal) { + return { + ...base, + status: 'blocked', + reason: 'load_crashed', + detail: `loading the binding killed the probe with ${input.probeSignal}` + } + } + + const classifiable = input.loaderError ?? usableRequireError + const classified = classifiable ? classifyNodePtyLoaderMessage(classifiable) : null + // Only a loader message that named the fault outranks the build record. `load_failed` + // and `dependency_missing` do not: the first named nothing, and the second is what + // node-pty says about a binding it never reached. + if (classified && LOADER_NAMED_FAULTS.has(classified.reason)) { + return { ...base, status: 'blocked', ...classified } + } + + // The loader said nothing usable. The binding's own build record still can: node-gyp + // records the ABI and arch it configured for, and either differing from this runtime is + // a fault the user can check without reproducing the load. + if (survey?.bindingPath) { + if (survey.builtNodeAbi && survey.builtNodeAbi !== host.nodeAbi) { + return { + ...base, + status: 'blocked', + reason: 'abi_mismatch', + detail: `built for Node ABI ${survey.builtNodeAbi}, this host runs ABI ${host.nodeAbi}` + } + } + if (survey.builtArch && survey.builtArch !== host.arch) { + return { + ...base, + status: 'blocked', + reason: 'arch_mismatch', + detail: `built for ${survey.builtArch}, this host runs ${host.arch}` + } + } + return { + ...base, + status: 'blocked', + reason: classified?.reason ?? 'load_failed', + detail: classified?.detail ?? 'the binding is present but the loader refused it' + } + } + + if (!survey) { + return { + ...base, + status: 'unverifiable', + reason: 'unknown', + detail: "the relay could not read node-pty's install directory" + } + } + // Nothing compiled anywhere. On Linux that is either a compile that never ran for want + // of a toolchain, or an install that failed for some other reason — different remedies. + if (toolchain?.toolchainMissing) { + return { + ...base, + status: 'blocked', + reason: 'toolchain_missing', + detail: `no compiled binding exists and ${missingToolSummary(toolchain)} missing` + } + } + return { + ...base, + status: 'blocked', + reason: 'dependency_missing', + detail: 'no compiled node-pty binding exists on this host' + } +} + +const RAW_ERROR_MAX = 600 + +function truncate(message: string | null): string | null { + if (message === null || message.length <= RAW_ERROR_MAX) { + return message + } + return `${message.slice(0, RAW_ERROR_MAX)}…` +} + +function missingToolSummary(toolchain: BuildToolchainStatus): string { + const present = new Set(toolchain.present) + const missing: string[] = [] + if (!present.has('make')) { + missing.push('make') + } + if (!present.has('g++') && !present.has('c++') && !present.has('clang++')) { + missing.push('a C++ compiler') + } + if (!present.has('python3') && !present.has('python')) { + missing.push('python3') + } + if (missing.length <= 1) { + return `${missing[0] ?? 'the build tools'} is` + } + return `${missing.slice(0, -1).join(', ')} and ${missing.at(-1)} are` +} + +/** + * Faults a rebuild on the host actually fixes. + * + * The relay's node-pty is compiled ON the remote by `npm install`, so a binding that is + * absent, built for another Node ABI, built for another architecture, or linked against a + * newer libc than the host provides is all one thing: the compiled artifact no longer + * matches the machine, and recompiling here produces one that does. That is different + * from the packaged desktop app, where the binary is built elsewhere and the glibc floor + * in docs/reference/linux-glibc-compatibility.md is the binding constraint. + * + * Excluded on purpose: `toolchain_missing` (no compiler to rebuild with) and + * `shared_library_missing` (the compile would need the same absent library). + */ +const REBUILD_FIXES: ReadonlySet = new Set([ + 'abi_mismatch', + 'arch_mismatch', + 'libc_floor', + 'load_crashed', + 'dependency_missing' +]) + +/** + * The machine-readable cause, for a client that can act instead of printing. + * + * `repairable` requires a proved status AND a toolchain that is not known-missing: a + * rebuild the host cannot perform is not a repair, it is a wasted `npm install` — which + * is the shape of #14830. + */ +export function toTerminalUnavailableCause( + diagnosis: NodePtyUnavailableDiagnosis +): TerminalUnavailableCause { + const { host } = diagnosis + return { + status: diagnosis.status, + reason: diagnosis.reason, + detail: diagnosis.detail.slice(0, 400), + repairable: + diagnosis.status === 'blocked' && + REBUILD_FIXES.has(diagnosis.reason) && + diagnosis.toolchain?.toolchainMissing !== true, + host: { + platform: host.platform, + arch: host.arch, + libc: host.libc, + ...(host.glibcVersion ? { glibcVersion: host.glibcVersion } : {}), + nodeAbi: host.nodeAbi, + nodeVersion: host.nodeVersion + }, + ...(diagnosis.rawError ? { rawError: diagnosis.rawError.slice(0, 1000) } : {}) + } +} + +/** `linux/x64, glibc 2.31, Node v20.11.0 (ABI 115), prebuild slot linux-x64-glibc`. */ +function formatNodePtyHostLine(host: NodePtyUnavailableHost): string { + const libc = + host.libc === 'none' ? null : `${host.libc}${host.glibcVersion ? ` ${host.glibcVersion}` : ''}` + return [ + `${host.platform}/${host.arch}`, + libc, + `Node ${host.nodeVersion} (ABI ${host.nodeAbi})`, + `prebuild slot ${nativeSlotName(host)}` + ] + .filter((part): part is string => part !== null) + .join(', ') +} + +/** + * One remedy per fault, each naming a value the user can go and check. + * + * `unverifiable` deliberately prescribes nothing: the relay proved only that it could not + * establish a cause, and dressing that up as a diagnosis is the bug this replaces. + */ +export function formatNodePtyUnavailableMessage(diagnosis: NodePtyUnavailableDiagnosis): string { + const { host } = diagnosis + // Unverifiable deliberately prescribes nothing beyond a retry: nothing was established, + // and dressing that up as a diagnosis is the bug this replaces. + const opening = + diagnosis.status === 'unverifiable' + ? `Remote terminals are unavailable, and the relay could not establish why: ${diagnosis.detail}. ` + + `That is not evidence node-pty is broken — reconnect to retry.` + : `Remote terminals are unavailable: ${remedyFor(diagnosis)}` + const lines = [opening, `Host: ${formatNodePtyHostLine(host)}.`] + // Quoted only where nothing else named the fault: elsewhere the remedy already carries + // the numbers, and a dlopen dump would bury them. + const quoteRaw = + diagnosis.status === 'unverifiable' || + diagnosis.reason === 'load_failed' || + diagnosis.reason === 'unknown' + if (diagnosis.rawError && quoteRaw) { + lines.push(`Loader error: ${diagnosis.rawError}`) + } + return lines.join('\n') +} + +function remedyFor(diagnosis: NodePtyUnavailableDiagnosis): string { + const { host, survey, toolchain } = diagnosis + switch (diagnosis.reason) { + case 'toolchain_missing': + return ( + `node-pty ships no prebuilt binary for Linux and this host has no compiled one ` + + `(${searchedPhrase(survey)}), because ${toolchain ? missingToolSummary(toolchain) : 'the build tools are'} not installed. ` + + `Install them on the remote host, then reconnect:\n` + + `${(toolchain ? toolchainInstallHintLines(toolchain) : []).join('\n')}` + ) + case 'dependency_missing': + return ( + `node-pty has no compiled binary on this host (${searchedPhrase(survey)}). ` + + `The C/C++ build tools needed to compile it are present, so reconnect to reinstall ` + + `the relay's native modules.` + ) + case 'abi_mismatch': + return ( + `the installed node-pty binding was built for a different Node ABI than the remote's ` + + `Node — ${diagnosis.detail}. Reconnect to rebuild node-pty against ${host.nodeVersion}, ` + + `or run the relay on the Node version the binding was built for.` + ) + case 'arch_mismatch': + return ( + `the installed node-pty binding does not match this host's CPU architecture — ` + + `${diagnosis.detail}. Reconnect to rebuild node-pty on the remote host; a binding ` + + `copied from a machine of another architecture can never load here.` + ) + case 'libc_floor': + return ( + `${diagnosis.detail}, which this host's C library does not provide ` + + `(${host.glibcVersion ? `glibc ${host.glibcVersion}` : 'this host reports no glibc version'}). ` + + `The binding was compiled on a newer system than this one. Reconnect to rebuild ` + + `node-pty here; Orca's own Linux floor is glibc ${GLIBC_FLOOR}.` + ) + case 'shared_library_missing': + return ( + `node-pty's native binding cannot be opened because ${diagnosis.detail}. ` + + `Install that library on the remote host, then reconnect.` + ) + case 'load_crashed': + return ( + `${diagnosis.detail}, which means the binding is incompatible with this host rather ` + + `than missing. Reconnect to rebuild the relay's native modules.` + ) + case 'load_failed': + case 'spawn_helper_missing': + case 'unknown': + return ( + `this host refused to load node-pty's native binding and the cause was not recognized. ` + + `Reconnect to rebuild the relay's native modules; if that does not help, please file an ` + + `issue quoting the loader error below.` + ) + } +} + +function searchedPhrase(survey: NodePtyBindingSurvey | null): string { + return survey && survey.searched.length > 0 + ? `checked ${survey.searched.join(', ')} under ${survey.moduleDir}` + : 'nothing was found where node-pty looks' +} diff --git a/src/relay/pty-handler-spawn-admission.test.ts b/src/relay/pty-handler-spawn-admission.test.ts index 2df29b95420..728f883d3b1 100644 --- a/src/relay/pty-handler-spawn-admission.test.ts +++ b/src/relay/pty-handler-spawn-admission.test.ts @@ -32,7 +32,7 @@ vi.mock('../main/shell-prompt-readiness-probe', () => ({ createShellPromptReadinessProbe: mockCreateShellPromptReadinessProbe })) -import { MAX_RELAY_PTY_SESSIONS, PtyHandler, formatNodePtyUnavailableMessage } from './pty-handler' +import { MAX_RELAY_PTY_SESSIONS, PtyHandler } from './pty-handler' import type { RelayDispatcher } from './dispatcher' import { beginPtyHandlerTest, @@ -222,24 +222,6 @@ describe('PtyHandler', () => { expect(mockPtySpawn).toHaveBeenCalledOnce() }) - it('hedges both causes on Linux and offers the build-tools remedy nowhere else', () => { - const linux = formatNodePtyUnavailableMessage('linux') - expect(linux).toContain('Remote terminals are unavailable') - // Conditional, not asserted: a host with build-essential can still hit an ABI/Node-version flip. - expect(linux).toMatch(/If it is missing the C\/C\+\+ build tools/) - expect(linux).toContain('python3') - expect(linux).toContain('version and architecture match the installed binding') - - // Windows/macOS ship node-pty prebuilds, so "install make/g++/python3" sends the user chasing nothing. - for (const platform of ['win32', 'darwin'] as const) { - const message = formatNodePtyUnavailableMessage(platform) - expect(message).toContain('Remote terminals are unavailable') - expect(message).not.toContain('build tools') - expect(message).not.toContain('python3') - expect(message).toMatch(/reconnect/i) - } - }) - it('normalizes a missing native binding as degraded node-pty availability', async () => { mockPtySpawn.mockImplementationOnce(() => { throw new Error( @@ -253,6 +235,30 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(0) }) + it('keeps the load error it was handed instead of replacing it with guesses', async () => { + // #17830: the user got three remedies for four possible faults and could verify none. + // The relay must carry what it was actually told, and must not prescribe a toolchain + // install it never probed for. + const thrown = + 'Failed to load native module: conpty.node, checked: build/Release, prebuilds/win32-x64' + mockPtySpawn.mockImplementationOnce(() => { + throw new Error(thrown) + }) + + const message = await dispatcher.callRequest('pty.spawn', {}).then( + () => '', + (error: Error) => error.message + ) + + expect(message).toContain(thrown) + expect(message).not.toContain('install make, a C++ compiler, and python3') + // Nothing here established a cause — the relay's node-pty directory is not on disk in + // this harness — so per docs/reference/ssh-execution-boundary.md it must say so rather + // than pick a diagnosis. Every message still names the host, for the bug report. + expect(message).toContain('could not establish why') + expect(message).toMatch(/Host: linux\/\w+, .*Node v[\d.]+ \(ABI \d+\)/) + }) + it('preserves unrelated node-pty spawn failures', async () => { mockPtySpawn.mockImplementationOnce(() => { throw new Error('File not found: missing-shell.exe') diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 82f0b19b9ff..f8bd2351cf2 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -102,23 +102,16 @@ import { injectRelayFishHistoryEnv, injectRelayHistoryEnv } from './terminal-history' - -// Why: only Linux compiles node-pty (no prebuilt), so the build-tools remedy is a closable setup gap -// there and wrong advice anywhere node-pty ships one. The relay only sees an unloadable binding, never -// why — a skipped compile and a later Node/ABI flip look identical here — so Linux hedges both causes. -export function formatNodePtyUnavailableMessage(platform: NodeJS.Platform): string { - const remedy = - platform === 'linux' - ? "node-pty's native binding is not loadable on this host. If it is missing the C/C++ build tools needed to compile node-pty, install make, a C++ compiler, and python3 on the remote host, then reconnect. Otherwise reconnect to reinstall the relay's native modules, and check that the remote Node.js version and architecture match the installed binding." - : "node-pty's native binding failed to load on this host. Reconnect to reinstall the relay's native modules; if it persists, check that the remote Node.js version and architecture match the installed binding." - return `Remote terminals are unavailable: ${remedy}` -} +import { isFlattenedNodePtyLoaderMessage } from '../main/orcad/node-pty-loader-diagnosis' +import { collectNodePtyUnavailableDiagnosis } from './node-pty-binding-survey' +import { + formatNodePtyUnavailableMessage, + toTerminalUnavailableCause +} from './node-pty-unavailable-diagnosis' +import { TERMINAL_UNAVAILABLE_RPC_ERROR_CODE } from '../shared/terminal-unavailable-cause' function isMissingNodePtyNativeBinding(error: unknown): boolean { - return ( - error instanceof Error && - /Failed to load native module: (?:conpty|pty)\.node(?:,|$)/.test(error.message) - ) + return error instanceof Error && isFlattenedNodePtyLoaderMessage(error.message) } function parseSourceRecoveryRequest(value: unknown): PtySourceRecoveryRequest | undefined { @@ -474,6 +467,8 @@ export class PtyHandler { private ptyModule: typeof NodePty | null = null private ptyModuleLoadPromise: Promise | null = null private reloadPtyModuleFromDisk = false + /** The last thing `require('node-pty')` threw, kept because it is the only cause anyone has. */ + private lastPtyLoadError: unknown = null // Why: single optional slot is intentional — callers compose externally; a throw is swallowed so it can't block cleanup. private exitListener: PtyExitListener | null = null private surfaceRetiredListener: PtySurfaceRetiredListener | null = null @@ -547,27 +542,56 @@ export class PtyHandler { try { this.ptyModule = await import('node-pty') return this.ptyModule - } catch { + } catch (error) { + // Why keep it: this is the only place the load error exists. Discarding it here is + // what left the relay able to say "unavailable" and never why. + this.lastPtyLoadError = error this.reloadPtyModuleFromDisk = true } } // Why: tie module resolution to the deployed bundle dir, not cwd. - const moduleEntry = join(__dirname, 'node_modules', 'node-pty', 'lib', 'index.js') + const moduleEntry = join(this.relayNodePtyDir(), 'lib', 'index.js') if (!existsSync(moduleEntry)) { + this.lastPtyLoadError = this.lastPtyLoadError ?? new Error(`no node-pty at ${moduleEntry}`) return null } try { this.ptyModule = require(moduleEntry) as typeof NodePty return this.ptyModule - } catch { + } catch (error) { + this.lastPtyLoadError = error return null } } + /** Where the relay's own node-pty lives — the deployed bundle dir, never cwd. */ + private relayNodePtyDir(): string { + return join(__dirname, 'node_modules', 'node-pty') + } + + /** + * The rejection for a spawn that cannot happen: prose for a human, and the structured + * cause for a client that can repair the host instead of printing a paragraph. + * + * Runs the survey and out-of-process load probe only here, on the failure path, so a + * healthy relay never pays for them. + */ + private async nodePtyUnavailableError(spawnError?: unknown): Promise { + const nodePtyDir = this.relayNodePtyDir() + const diagnosis = await collectNodePtyUnavailableDiagnosis({ + nodePtyDir: existsSync(nodePtyDir) ? nodePtyDir : null, + error: spawnError ?? this.lastPtyLoadError + }) + return Object.assign(new Error(formatNodePtyUnavailableMessage(diagnosis)), { + code: TERMINAL_UNAVAILABLE_RPC_ERROR_CODE, + data: toTerminalUnavailableCause(diagnosis) + }) + } + private invalidatePtyModuleAfterBindingFailure(): void { this.ptyModule = null this.reloadPtyModuleFromDisk = true - const moduleRoot = join(__dirname, 'node_modules', 'node-pty') + const moduleRoot = this.relayNodePtyDir() for (const cachedPath of Object.keys(require.cache)) { if (isPathInsideOrEqual(moduleRoot, cachedPath)) { delete require.cache[cachedPath] @@ -1718,7 +1742,7 @@ export class PtyHandler { }> { const pty = await this.loadPty() if (!pty) { - throw new Error(formatNodePtyUnavailableMessage(process.platform)) + throw await this.nodePtyUnavailableError() } const cols = (params.cols as number) || 80 @@ -1831,7 +1855,7 @@ export class PtyHandler { // Why: Windows loads conpty.node only on first spawn, so handle that late binding failure here. if (isMissingNodePtyNativeBinding(error)) { this.invalidatePtyModuleAfterBindingFailure() - throw new Error(formatNodePtyUnavailableMessage(process.platform)) + throw await this.nodePtyUnavailableError(error) } throw error } diff --git a/src/shared/runtime-capability-degradation.ts b/src/shared/runtime-capability-degradation.ts index 05620ac9942..977cd68737f 100644 --- a/src/shared/runtime-capability-degradation.ts +++ b/src/shared/runtime-capability-degradation.ts @@ -20,8 +20,11 @@ export type RuntimeBrowserUnavailableReason = */ export type RuntimeTerminalUnavailableReason = | 'dependency_missing' + | 'toolchain_missing' | 'libc_floor' + | 'shared_library_missing' | 'abi_mismatch' + | 'arch_mismatch' | 'load_failed' | 'load_crashed' | 'spawn_helper_missing' @@ -43,10 +46,16 @@ export type RuntimeDegradation = { const TERMINAL_UNAVAILABLE_MESSAGES: Record = { dependency_missing: 'Terminals are unavailable on this host: node-pty has no native binary for this platform. Install or rebuild it, or deploy a build that ships a prebuilt binary for this platform.', + toolchain_missing: + 'Terminals are unavailable on this host: node-pty has no prebuilt binary for Linux and this host is missing the C/C++ build tools needed to compile one. Install them, then reconnect.', libc_floor: "This host's node-pty binary was built against a newer C library than the host provides, so the dynamic loader refuses it. Rebuild node-pty on this host, or deploy a build whose prebuilt binary matches this platform's libc.", + shared_library_missing: + 'Terminals are unavailable on this host: a shared library that node-pty links against is not installed, so the dynamic loader cannot open the binary. Install the named library, then reconnect.', abi_mismatch: "This host's node-pty binary was built for a different Node ABI than the running Node, so it cannot be loaded. Rebuild node-pty against this Node version.", + arch_mismatch: + "This host's node-pty binary was built for a different CPU architecture than the running Node, so the dynamic loader refuses it. Rebuild node-pty on this host, or deploy a build for this architecture.", load_failed: 'Terminals are unavailable on this host: node-pty failed to load.', load_crashed: 'Terminals are unavailable on this host: loading node-pty terminated the probe process, which means the binary is incompatible with this host rather than merely missing.', diff --git a/src/shared/terminal-unavailable-cause.ts b/src/shared/terminal-unavailable-cause.ts new file mode 100644 index 00000000000..2671941da30 --- /dev/null +++ b/src/shared/terminal-unavailable-cause.ts @@ -0,0 +1,73 @@ +/** + * The machine-readable half of "remote terminals are unavailable". + * + * Why this exists: the fault is proved on the relay, at spawn time, and the machinery + * that can repair it (`repairInstalledNativeDeps`) lives on the client, at connect time. + * Until now the only thing that crossed the wire was prose, so the client could not tell + * a rebuildable ABI flip from a host whose glibc will never satisfy the binary — and the + * message had to hedge across all of them. + * + * Wire compatibility (docs/reference/remote-wire-compatibility.md): this rides as the + * optional `data` of an existing JSON-RPC error, so it is Rule 1 — additive. A client + * that does not read it still renders `error.message`, which is exactly today's + * behaviour, so no capability negotiation is needed. + * + * `repairable` is the field with teeth: it is true only for a fault that was PROVED and + * that rebuilding node-pty on the host actually fixes. An `unverifiable` cause is never + * repairable — a probe that did not answer must not trigger a destructive repair, which + * is the #14830 lesson recorded in docs/reference/ssh-execution-boundary.md. + */ +import { z } from 'zod' +import { TERMINAL_UNAVAILABLE_ERROR_CODE } from './runtime-capability-degradation' + +export const TERMINAL_UNAVAILABLE_RPC_ERROR_CODE = TERMINAL_UNAVAILABLE_ERROR_CODE + +const TerminalUnavailableHostSchema = z + .object({ + platform: z.string().min(1).max(32), + arch: z.string().min(1).max(32), + libc: z.enum(['glibc', 'musl', 'none']), + /** Absent, not null, when the host reports no version — see native-host-abi.ts. */ + glibcVersion: z.string().min(1).max(32).optional(), + /** `NODE_MODULE_VERSION` the remote runtime accepts. */ + nodeAbi: z.string().min(1).max(16), + nodeVersion: z.string().min(1).max(32) + }) + .strict() + +export const TerminalUnavailableCauseSchema = z + .object({ + /** `blocked` — proved. `unverifiable` — nothing answered; never act on it. */ + status: z.enum(['blocked', 'unverifiable']), + /** + * Open vocabulary, deliberately `string` rather than an enum: a newer relay may name a + * reason this client has never heard of, and a strict enum would drop the whole cause + * (including `repairable`) rather than the one field it cannot interpret. + */ + reason: z.string().min(1).max(64), + detail: z.string().max(400), + /** Proved, and rebuilding node-pty on the host is the fix. */ + repairable: z.boolean(), + host: TerminalUnavailableHostSchema, + /** The dynamic loader's own words, when they were recovered. */ + rawError: z.string().max(1000).optional() + }) + .strict() + +export type TerminalUnavailableCause = z.infer + +/** Null for anything that does not validate; a malformed cause must never be acted on. */ +export function parseTerminalUnavailableCause(value: unknown): TerminalUnavailableCause | null { + const parsed = TerminalUnavailableCauseSchema.safeParse(value) + return parsed.success ? parsed.data : null +} + +/** + * Whether the client may rewrite the host's `node_modules` on the strength of this cause. + * + * Deliberately re-derived here rather than trusting `repairable` alone: the flag arrives + * from a peer, and only a `blocked` status is evidence of anything. + */ +export function mayRepairFromCause(cause: TerminalUnavailableCause | null): boolean { + return cause !== null && cause.status === 'blocked' && cause.repairable +}