diff --git a/src/main/ssh/build-toolchain-diagnosis.ts b/src/main/ssh/build-toolchain-diagnosis.ts index dcec7a51237..77d20ce475a 100644 --- a/src/main/ssh/build-toolchain-diagnosis.ts +++ b/src/main/ssh/build-toolchain-diagnosis.ts @@ -174,10 +174,13 @@ const NODE_HEADERS_TARBALL_RE = /node-v[0-9.]+-headers\.tar\.gz/i * depends on what the local-headers export found first, so the formatter takes that answer. */ export function isNodeHeadersDownloadFailure(message: string): boolean { + // Why `configure error` is required: node-gyp's fetch client logs `attempt N failed with ` + // on retries it then recovers from, so a network token alone also matches a build that got its + // headers and died later for an unrelated reason. Only the configure step downloads headers. return ( - message.toLowerCase().includes('gyp') && + /gyp ERR! configure error/i.test(message) && NODE_HEADERS_TARBALL_RE.test(message) && - /\b(ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EAI_AGAIN|EHOSTUNREACH|ENETUNREACH|fetch failed|FetchError)\b/i.test( + /\b(ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN|ECONNRESET)\b/.test( message ) ) @@ -213,11 +216,11 @@ export function formatNodeHeadersDownloadError( ] : [ 'The remote host could not download the Node.js headers needed to compile node-pty, and ' + - `its Node install does not ship them locally. ${NODE_HEADERS_CONTEXT}`, + `its Node install has no local headers matching its own version. ${NODE_HEADERS_CONTEXT}`, '', 'Fix one of the following on the remote host, then reconnect:', ' - Install Node.js from an official build or a version manager (nvm, fnm, volta, n), ' + - 'which include the headers; or', + 'which ship headers for exactly the Node they run; or', ' - Allow outbound HTTPS to nodejs.org, or point npm at a mirror: ' + 'npm config set disturl https:///dist' ] diff --git a/src/main/ssh/ssh-relay-build-toolchain.test.ts b/src/main/ssh/ssh-relay-build-toolchain.test.ts index 6d491887c33..6fcc4f5321d 100644 --- a/src/main/ssh/ssh-relay-build-toolchain.test.ts +++ b/src/main/ssh/ssh-relay-build-toolchain.test.ts @@ -140,7 +140,7 @@ describe('isNodeHeadersDownloadFailure', () => { expect(isNodeHeadersDownloadFailure(HEADERS_REFUSED)).toBe(true) expect( isNodeHeadersDownloadFailure( - 'gyp http fetch GET https://nodejs.org/download/release/v20.19.0/node-v20.19.0-headers.tar.gz attempt 1 failed with ENOTFOUND' + 'gyp http fetch GET https://nodejs.org/download/release/v20.19.0/node-v20.19.0-headers.tar.gz attempt 1 failed with ENOTFOUND\ngyp ERR! configure error' ) ).toBe(true) }) @@ -159,13 +159,27 @@ describe('isNodeHeadersDownloadFailure', () => { 'gyp info using node-v24.12.0-headers.tar.gz\ngyp ERR! build error make failed with exit code: 2' ) ).toBe(false) + // A retried attempt that recovered, then a compile failure: not a download failure. + expect( + isNodeHeadersDownloadFailure( + 'gyp http fetch GET https://nodejs.org/download/release/v24.12.0/node-v24.12.0-headers.tar.gz attempt 1 failed with ECONNRESET\n' + + 'gyp http 200 https://nodejs.org/download/release/v24.12.0/node-v24.12.0-headers.tar.gz\n' + + 'gyp ERR! build error\ngyp ERR! stack Error: `make` failed with exit code: 2' + ) + ).toBe(false) + // A mirror answering non-2xx is a FetchError without a network code: a different remedy. + expect( + isNodeHeadersDownloadFailure( + 'gyp ERR! configure error\ngyp ERR! stack FetchError: 404 Not Found https://mirror/dist/v24.12.0/node-v24.12.0-headers.tar.gz' + ) + ).toBe(false) }) }) describe('formatNodeHeadersDownloadError', () => { it('names both host remedies when the host ships no headers', () => { const msg = formatNodeHeadersDownloadError(HEADERS_REFUSED, null) - expect(msg).toContain('does not ship them locally') + expect(msg).toContain('no local headers matching its own version') expect(msg).toContain('/include/node') expect(msg).toContain('nvm, fnm, volta, n') expect(msg).toContain('disturl') @@ -176,7 +190,7 @@ describe('formatNodeHeadersDownloadError', () => { const msg = formatNodeHeadersDownloadError(HEADERS_REFUSED, '/usr/local') expect(msg).toContain('/usr/local/include/node') expect(msg).toContain('Orca defect') - expect(msg).not.toContain('does not ship them locally') + expect(msg).not.toContain('no local headers matching its own version') expect(msg).not.toContain('nvm, fnm, volta, n') expect(msg).toContain('ECONNREFUSED') }) diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index 5b7d30ec829..5d8101361c6 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -1265,8 +1265,15 @@ async function installNativeDeps( throw err } signal?.throwIfAborted() + // Same diagnosis as the install catch: this fallback is non-fatal, so the log is the only + // place the offline-headers cause can reach anyone. + const rebuildMsg = (err as Error).message console.warn( - `[ssh-relay][NATIVE-DEPS-REBUILD-FAIL] npm rebuild native deps failed at ${remoteDir} (${platform}): ${(err as Error).message}` + `[ssh-relay][NATIVE-DEPS-REBUILD-FAIL] npm rebuild native deps failed at ${remoteDir} (${platform}): ${ + platform.startsWith('linux') && isNodeHeadersDownloadFailure(rebuildMsg) + ? formatNodeHeadersDownloadError(rebuildMsg, localNodeHeadersFromOutput(rebuildMsg)) + : rebuildMsg + }` ) } signal?.throwIfAborted() diff --git a/src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts b/src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts index 5a691b69af4..d3894456d99 100644 --- a/src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts +++ b/src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts @@ -190,7 +190,7 @@ describe('installNativeDeps staged uploads', () => { const error = await deployAndLaunchRelay(conn).catch((e: Error) => e) expect((error as Error).message).toContain('could not download the Node.js headers') - expect((error as Error).message).toContain('does not ship them locally') + expect((error as Error).message).toContain('no local headers matching its own version') expect((error as Error).message).toContain('ECONNREFUSED') // A full toolchain: the toolchain probe must not run, and this is not a "build tools" error. expect((error as Error).message).not.toContain('build tools') @@ -205,7 +205,7 @@ describe('installNativeDeps staged uploads', () => { makeExecResponses({ npmInstall: { reject: - 'Command "npm install" failed (exit 1): ORCA-NODE-HEADERS:/usr/local\nnpm error gyp http fetch GET https://nodejs.org/download/release/v24.12.0/node-v24.12.0-headers.tar.gz attempt 1 failed with ECONNREFUSED' + 'Command "npm install" failed (exit 1): ORCA-NODE-HEADERS:/usr/local\nnpm error gyp http fetch GET https://nodejs.org/download/release/v24.12.0/node-v24.12.0-headers.tar.gz attempt 1 failed with ECONNREFUSED\nnpm error gyp ERR! configure error' } }) ) @@ -213,7 +213,7 @@ describe('installNativeDeps staged uploads', () => { const error = await deployAndLaunchRelay(conn).catch((e: Error) => e) expect((error as Error).message).toContain('/usr/local/include/node') expect((error as Error).message).toContain('Orca defect') - expect((error as Error).message).not.toContain('does not ship them locally') + expect((error as Error).message).not.toContain('no local headers matching its own version') }) it('promotes only after the first-install lock is acquired', async () => { diff --git a/src/main/ssh/ssh-relay-node-headers.test.ts b/src/main/ssh/ssh-relay-node-headers.test.ts index fa5b6ac00c9..81b49954a22 100644 --- a/src/main/ssh/ssh-relay-node-headers.test.ts +++ b/src/main/ssh/ssh-relay-node-headers.test.ts @@ -98,6 +98,26 @@ describe.skipIf(!POSIX)('exportLocalNodeHeadersPrefix', () => { expect(nodedir).toBe(dirname(dirname(process.execPath))) }) + it('clears an inherited nodedir when the probe finds no matching headers', () => { + // A remote profile's stale nodedir must not survive past the version check. + const root = mkdtempSync(join(tmpdir(), 'orca-node-headers-')) + roots.push(root) + const copied = join(root, 'bin', 'node') + mkdirSync(dirname(copied), { recursive: true }) + spawnSync('cp', [process.execPath, copied]) + const script = `${exportLocalNodeHeadersPrefix(copied)}printf '%s|%s' "$npm_config_nodedir" "$npm_package_config_node_gyp_nodedir"` + const result = spawnSync('/bin/sh', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + npm_config_nodedir: '/usr/stale-headers', + npm_package_config_node_gyp_nodedir: '/usr/stale-headers' + } + }) + expect(result.status).toBe(0) + expect(result.stdout.split('\n').at(-1)).toBe('|') + }) + it('does not fail the command line when node itself cannot run', () => { const script = `${exportLocalNodeHeadersPrefix('/nonexistent/node')}echo "after:$npm_config_nodedir"` const result = spawnSync('/bin/sh', ['-c', script], { encoding: 'utf8' }) diff --git a/src/main/ssh/ssh-relay-node-headers.ts b/src/main/ssh/ssh-relay-node-headers.ts index 30897ed2ef4..7f9f6721d70 100644 --- a/src/main/ssh/ssh-relay-node-headers.ts +++ b/src/main/ssh/ssh-relay-node-headers.ts @@ -49,8 +49,11 @@ export const LOCAL_NODE_HEADERS_MARKER_PREFIX = 'ORCA-NODE-HEADERS:' */ export function exportLocalNodeHeadersPrefix(nodePath: string): string { const probe = `${shellEscape(nodePath)} -e ${shellEscape(LOCAL_NODE_HEADERS_PROBE_JS)} 2>/dev/null` + // Why the unset: a remote profile can already carry a nodedir (a stale distro header dir, an old + // ~/.npmrc). Left alone it would bypass the version check above and build a wrong-ABI binding. return ( `${NODEDIR_SHELL_VAR}=$(${probe}); ` + + `unset npm_config_nodedir npm_package_config_node_gyp_nodedir; ` + `if [ -n "$${NODEDIR_SHELL_VAR}" ]; then ` + `export npm_config_nodedir="$${NODEDIR_SHELL_VAR}" npm_package_config_node_gyp_nodedir="$${NODEDIR_SHELL_VAR}"; ` + `fi; ` + diff --git a/src/main/ssh/ssh-relay-offline-node-headers.docker.test.ts b/src/main/ssh/ssh-relay-offline-node-headers.docker.test.ts index 7805cb9040c..db9e1a83502 100644 --- a/src/main/ssh/ssh-relay-offline-node-headers.docker.test.ts +++ b/src/main/ssh/ssh-relay-offline-node-headers.docker.test.ts @@ -11,6 +11,7 @@ import { execFileSync, spawnSync } from 'node:child_process' import { randomUUID } from 'node:crypto' import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { connect } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' @@ -45,7 +46,7 @@ function dockerExec(fixture: TargetFixture, command: string): string { return run('docker', ['exec', fixture.containerName, 'bash', '-lc', command], 60_000) } -function startTarget(): TargetFixture { +async function startTarget(): Promise { const image = `orca-review-offline-headers:${NODE_IMAGE.replace(/[^A-Za-z0-9_.-]/g, '-')}` run( 'docker', @@ -85,9 +86,35 @@ function startTarget(): TargetFixture { 120_000 ) const port = Number(run('docker', ['port', containerName, '22/tcp']).split(':').at(-1)) + // `docker run -d` returns before sshd binds; connect() against a closed port is a flake. + await waitForSshBanner(port) return { containerName, identityFile, port, tempDir } } +/** Resolves once sshd answers with its banner on the mapped port, or throws after the deadline. */ +async function waitForSshBanner(port: number, deadlineMs = 60_000): Promise { + const deadline = Date.now() + deadlineMs + for (;;) { + const gotBanner = await new Promise((resolve) => { + const socket = connect({ host: TARGET_HOST, port }) + const done = (value: boolean): void => { + socket.destroy() + resolve(value) + } + socket.setTimeout(2_000, () => done(false)) + socket.once('data', (chunk) => done(chunk.toString('utf8').startsWith('SSH-'))) + socket.once('error', () => done(false)) + }) + if (gotBanner) { + return + } + if (Date.now() > deadline) { + throw new Error(`sshd on port ${port} did not answer within ${deadlineMs / 1000}s`) + } + await new Promise((resolve) => setTimeout(resolve, 500)) + } +} + function stopTarget(fixture: TargetFixture | null): void { if (!fixture) { return @@ -115,8 +142,8 @@ describe.skipIf(!RUN_REVIEW_ORACLE)( () => { let fixture: TargetFixture | null = null - beforeAll(() => { - fixture = startTarget() + beforeAll(async () => { + fixture = await startTarget() }, 900_000) afterAll(() => {