diff --git a/src/main/ssh/ssh-relay-deploy-timing.ts b/src/main/ssh/ssh-relay-deploy-timing.ts new file mode 100644 index 00000000000..0bd2ca95a17 --- /dev/null +++ b/src/main/ssh/ssh-relay-deploy-timing.ts @@ -0,0 +1,13 @@ +// npm install on a cold Windows cache plus antivirus scanning can exceed the +// default 30s exec timeout. +export const NATIVE_DEPS_COMMAND_TIMEOUT_MS = 240_000 + +// Why: a missing binding can require both install and rebuild while the same +// install lock is held. Concurrent first installs must wait for that valid +// holder instead of failing halfway through its bounded work. +export const NATIVE_DEPS_REPAIR_BUDGET_MS = 2 * NATIVE_DEPS_COMMAND_TIMEOUT_MS + +// Why: individual exec commands have their own timeouts, but the full deploy +// pipeline has no overall bound. Leave headroom after the native-deps budget +// for platform detection, upload, probes, and launch. +export const RELAY_DEPLOY_TIMEOUT_MS = NATIVE_DEPS_REPAIR_BUDGET_MS + 60_000 diff --git a/src/main/ssh/ssh-relay-deploy.test.ts b/src/main/ssh/ssh-relay-deploy.test.ts index aa68f984aa6..4eb5bd57364 100644 --- a/src/main/ssh/ssh-relay-deploy.test.ts +++ b/src/main/ssh/ssh-relay-deploy.test.ts @@ -62,6 +62,10 @@ vi.mock('./ssh-relay-versioned-install', () => ({ gcOldRelayVersions: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./ssh-relay-repair-lock', () => ({ + tryAcquireRelayRepairLock: vi.fn().mockResolvedValue(true) +})) + vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index 681f24b98ff..fb0bad68c8f 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -19,6 +19,8 @@ import { abandonInstall, gcOldRelayVersions } from './ssh-relay-versioned-install' +import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock' +import { NATIVE_DEPS_COMMAND_TIMEOUT_MS, RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing' import { shellEscape } from './ssh-connection-utils' import { probeBuildToolchain, @@ -66,22 +68,6 @@ export type RelayDeployResult = { sockPath?: string } -// npm install on a cold Windows cache plus antivirus scanning can exceed the -// default 30s exec timeout. The same bound covers the post-install `npm rebuild` -// retry that repairs a native binding skipped by an ignore-scripts host. -const NATIVE_DEPS_INSTALL_TIMEOUT_MS = 240_000 - -// Why: individual exec commands have their own timeouts, but the full deploy -// pipeline (detect platform → check existing → upload → npm install → rebuild → -// launch) has no overall bound, so a hanging step or slow SFTP upload could -// block the connection indefinitely. This is a backstop, sized so the common -// native-deps case — a first install AND its follow-up rebuild, each up to -// NATIVE_DEPS_INSTALL_TIMEOUT_MS — plus headroom is not falsely timed out. -// (Adding heavy install-lock contention on top could still trip it; that stays -// a rare, self-healing-on-retry edge rather than a reason to loosen the -// backstop further.) -const RELAY_DEPLOY_TIMEOUT_MS = 2 * NATIVE_DEPS_INSTALL_TIMEOUT_MS + 60_000 - function execHostCommand( conn: SshConnection, hostPlatform: RemoteHostPlatform, @@ -443,6 +429,12 @@ const RELAY_NATIVE_DEPS = { '@parcel/watcher': '2.5.6' } as const +// Why: npm 12 blocks dependency lifecycle scripts unless each exact package +// version is approved, even when ignore-scripts is explicitly disabled. +const RELAY_NATIVE_DEP_SCRIPT_ALLOWLIST = Object.fromEntries( + Object.entries(RELAY_NATIVE_DEPS).map(([name, version]) => [`${name}@${version}`, true]) +) + const LOAD_RELAY_NATIVE_DEPS = 'require("node-pty"); require("@parcel/watcher")' async function hasRequiredNativeDeps( @@ -484,22 +476,14 @@ async function repairInstalledNativeDeps( return } - // Why: an already-installed relay can ALWAYS launch in degraded mode (fs/git/ - // preflight still work — only native-backed ops fail), so native-deps repair - // is best-effort and must never block the connection: not on lock contention, - // and not on an offline/unbuildable repair. Pre-repair this path used - // require.resolve (which passed for a present-but-unloadable binding), so a - // fatal repair here would be a straight regression. launchRelay still surfaces - // a genuinely dead transport on its own. + // Why: an already-installed relay can launch in degraded mode (fs/git/preflight + // still work; native-backed ops fail), so native-deps repair is best-effort: + // lock contention and repair failures must not abort the connection. Pre-repair + // this path used require.resolve (which passed for a present-but-unloadable + // binding), so a fatal repair here would be a straight regression. console.warn(`[ssh-relay] Repairing missing native deps at ${remoteDir}`) - try { - await acquireInstallLock(conn, remoteDir, hostPlatform) - } catch (err) { - console.warn( - `[ssh-relay] Could not lock native-deps repair at ${remoteDir}; launching degraded: ${ - err instanceof Error ? err.message : String(err) - }` - ) + if (!(await tryAcquireRelayRepairLock(conn, remoteDir, hostPlatform))) { + console.warn(`[ssh-relay] Native-deps repair lock is busy at ${remoteDir}; launching degraded`) return } try { @@ -551,7 +535,8 @@ async function installNativeDeps( version: '1.0.0', private: true, type: 'commonjs', - dependencies: RELAY_NATIVE_DEPS + dependencies: RELAY_NATIVE_DEPS, + allowScripts: RELAY_NATIVE_DEP_SCRIPT_ALLOWLIST })}\n` await writeRemoteFile( conn, @@ -582,7 +567,7 @@ async function installNativeDeps( `npm install --ignore-scripts=false --omit=dev --no-audit --no-fund ${installArgs} 2>&1` ) await execHostCommand(conn, hostPlatform, command, { - timeoutMs: NATIVE_DEPS_INSTALL_TIMEOUT_MS + timeoutMs: NATIVE_DEPS_COMMAND_TIMEOUT_MS }) } catch (err) { // Don't write .install-complete on hard fail; reconnect retries on a @@ -657,7 +642,7 @@ async function rebuildNativeDeps( `npm rebuild --ignore-scripts=false ${depNames.map(shellEscape).join(' ')} 2>&1` ) await execHostCommand(conn, hostPlatform, command, { - timeoutMs: NATIVE_DEPS_INSTALL_TIMEOUT_MS + timeoutMs: NATIVE_DEPS_COMMAND_TIMEOUT_MS }) } diff --git a/src/main/ssh/ssh-relay-install-lock-commands.ts b/src/main/ssh/ssh-relay-install-lock-commands.ts new file mode 100644 index 00000000000..dd341161ecc --- /dev/null +++ b/src/main/ssh/ssh-relay-install-lock-commands.ts @@ -0,0 +1,146 @@ +import { shellEscape } from './ssh-connection-utils' +import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +const STALE_STEAL_LOCK_SECONDS = 120 + +export function acquireInstallLockParentCommand( + host: RemoteHostPlatform, + remoteRelayDir: string +): string { + if (!isWindowsRemoteHost(host)) { + return `mkdir -p ${shellEscape(remoteRelayDir)}` + } + return powerShellCommand( + `$null = New-Item -ItemType Directory -Force -Path ${powerShellLiteral(remoteRelayDir)}` + ) +} + +export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: string): string { + if (!isWindowsRemoteHost(host)) { + return `mkdir ${shellEscape(lockDir)} 2>&1 && echo OK || echo BUSY` + } + // New-Item has no -LiteralPath parameter; using it breaks stock Windows PowerShell. + return powerShellCommand( + `$ErrorActionPreference = "Stop"; try { $null = New-Item -ItemType Directory -Path ${powerShellLiteral(lockDir)}; 'OK' } catch { 'BUSY' }` + ) +} + +export function lockAgeSecondsCommand(host: RemoteHostPlatform, lockDir: string): string { + if (!isWindowsRemoteHost(host)) { + return `${posixLockAgeSecondsAssignment(lockDir)} && echo "$age" || echo` + } + return powerShellCommand( + [ + `$item = Get-Item -LiteralPath ${powerShellLiteral(lockDir)} -ErrorAction Stop`, + '$mtime = ([DateTimeOffset]$item.LastWriteTimeUtc).ToUnixTimeSeconds()', + '$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()', + 'Write-Output ($now - $mtime)' + ].join('; ') + ) +} + +export function tryStealInstallLockCommand( + host: RemoteHostPlatform, + lockDir: string, + staleAfterSeconds: number +): string { + if (!isWindowsRemoteHost(host)) { + return posixStealInstallLockCommand(lockDir, staleAfterSeconds) + } + return windowsStealInstallLockCommand(lockDir, staleAfterSeconds) +} + +function posixStealInstallLockCommand(lockDir: string, staleAfterSeconds: number): string { + const escapedLockDir = shellEscape(lockDir) + const escapedStealLockPrefix = shellEscape(`${lockDir}.steal`) + return [ + `${posixLockIdentityAssignment(lockDir, 'lock_key')} && mtime=\${lock_key%%:*} && now=$(date +%s) && age=$((now - mtime)) || age=0;`, + `if [ "\${age:-0}" -le ${staleAfterSeconds} ] 2>/dev/null; then echo BUSY; else`, + `steal=${escapedStealLockPrefix}.$mtime;`, + 'owns_steal=0;', + 'owned_steal=;', + 'lock_tombstone=;', + 'attempts=0;', + `while [ "$attempts" -lt 8 ] && [ "$owns_steal" != 1 ]; do`, + 'if mkdir "$steal" 2>/dev/null; then owns_steal=1; owned_steal=$steal; break; fi;', + `steal_key=$(stat -c %Y:%i "$steal" 2>/dev/null || stat -f %m:%i "$steal" 2>/dev/null) && steal_mtime=\${steal_key%%:*} && steal_now=$(date +%s) && steal_age=$((steal_now - steal_mtime)) || break;`, + `if [ "\${steal_age:-0}" -le ${STALE_STEAL_LOCK_SECONDS} ] 2>/dev/null; then break; fi;`, + `steal_token=$(printf '%s' "$steal_key" | tr ':/' '__');`, + `steal="$steal.next.$steal_token";`, + 'attempts=$((attempts + 1));', + 'done;', + 'if [ "$owns_steal" = 1 ]; then', + `trap 'rmdir "$owned_steal" 2>/dev/null || true; rm -rf "$lock_tombstone" 2>/dev/null || true' EXIT;`, + `${posixLockIdentityAssignment(lockDir, 'current_key')} && current_mtime=\${current_key%%:*} && current_now=$(date +%s) && current_age=$((current_now - current_mtime)) || current_age=0;`, + `if [ "$current_key" = "$lock_key" ] && [ "\${current_age:-0}" -gt ${staleAfterSeconds} ] 2>/dev/null; then`, + `lock_tombstone=${escapedLockDir}.tombstone.$$.$(date +%s);`, + `if mv ${escapedLockDir} "$lock_tombstone" 2>/dev/null; then mkdir ${escapedLockDir} 2>&1 && echo OK || echo BUSY; else echo BUSY; fi;`, + 'else echo BUSY; fi;', + 'else echo BUSY; fi; fi' + ].join(' ') +} + +function windowsStealInstallLockCommand(lockDir: string, staleAfterSeconds: number): string { + return powerShellCommand( + [ + `$lock = ${powerShellLiteral(lockDir)}`, + 'try {', + '$item = Get-Item -LiteralPath $lock -ErrorAction Stop', + '$mtime = ([DateTimeOffset]$item.LastWriteTimeUtc).ToUnixTimeSeconds()', + '$lockIdentity = "${mtime}:$($item.CreationTimeUtc.Ticks)"', + '$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()', + `if (($now - $mtime) -le ${staleAfterSeconds}) { 'BUSY' } else {`, + '$steal = "$lock.steal.$mtime"', + '$ownsSteal = $false', + '$ownedSteal = $null', + '$lockTombstone = $null', + 'try {', + 'for ($attempt = 0; ($attempt -lt 8) -and (-not $ownsSteal); $attempt++) {', + 'try {', + '$null = New-Item -ItemType Directory -Path $steal -ErrorAction Stop', + '$ownsSteal = $true', + '$ownedSteal = $steal', + 'break', + '} catch {', + 'try {', + '$stealItem = Get-Item -LiteralPath $steal -ErrorAction Stop', + '$stealMtime = ([DateTimeOffset]$stealItem.LastWriteTimeUtc).ToUnixTimeSeconds()', + '$stealAge = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $stealMtime', + `if ($stealAge -le ${STALE_STEAL_LOCK_SECONDS}) { break }`, + '$stealIdentity = "$stealMtime-$($stealItem.CreationTimeUtc.Ticks)"', + '$steal = "$steal.next.$stealIdentity"', + '} catch { break }', + '}', + '}', + "if (-not $ownsSteal) { 'BUSY' } else {", + '$current = Get-Item -LiteralPath $lock -ErrorAction Stop', + '$currentMtime = ([DateTimeOffset]$current.LastWriteTimeUtc).ToUnixTimeSeconds()', + '$currentIdentity = "${currentMtime}:$($current.CreationTimeUtc.Ticks)"', + '$currentNow = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()', + `if (($currentIdentity -eq $lockIdentity) -and (($currentNow - $currentMtime) -gt ${staleAfterSeconds})) {`, + '$lockTombstone = "$lock.tombstone.$PID.$([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds())"', + 'Move-Item -LiteralPath $lock -Destination $lockTombstone -ErrorAction Stop', + "try { $null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop; 'OK' } catch { 'BUSY' }", + "} else { 'BUSY' }", + '}', + "} catch { 'BUSY' } finally { if ($null -ne $ownedSteal) { Remove-Item -LiteralPath $ownedSteal -Recurse -Force -ErrorAction SilentlyContinue }; if ($null -ne $lockTombstone) { Remove-Item -LiteralPath $lockTombstone -Recurse -Force -ErrorAction SilentlyContinue } }", + '}', + "} catch { 'BUSY' }" + ].join('; ') + ) +} + +function posixLockAgeSecondsAssignment(lockDir: string): string { + return `${posixLockMtimeSecondsAssignment(lockDir, 'mtime')} && now=$(date +%s) && age=$((now - mtime))` +} + +function posixLockIdentityAssignment(lockDir: string, variableName: string): string { + const escapedLockDir = shellEscape(lockDir) + return `${variableName}=$(stat -c %Y:%i ${escapedLockDir} 2>/dev/null || stat -f %m:%i ${escapedLockDir} 2>/dev/null)` +} + +function posixLockMtimeSecondsAssignment(lockDir: string, variableName: string): string { + const escapedLockDir = shellEscape(lockDir) + return `${variableName}=$(stat -c %Y ${escapedLockDir} 2>/dev/null || stat -f %m ${escapedLockDir} 2>/dev/null)` +} diff --git a/src/main/ssh/ssh-relay-native-deps-install.test.ts b/src/main/ssh/ssh-relay-native-deps-install.test.ts index d15d68b1640..5dd1b4905ca 100644 --- a/src/main/ssh/ssh-relay-native-deps-install.test.ts +++ b/src/main/ssh/ssh-relay-native-deps-install.test.ts @@ -47,6 +47,10 @@ vi.mock('./ssh-relay-versioned-install', () => ({ gcOldRelayVersions: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./ssh-relay-repair-lock', () => ({ + tryAcquireRelayRepairLock: vi.fn().mockResolvedValue(true) +})) + vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) @@ -61,6 +65,7 @@ import { finalizeInstall, isRelayAlreadyInstalled } from './ssh-relay-versioned-install' +import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock' import type { SshConnection } from './ssh-connection' type SftpWriteCapture = { @@ -270,6 +275,10 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { // break `require('node-pty')`. expect(parsed.type).toBe('commonjs') expect(parsed.dependencies).toEqual({ '@parcel/watcher': '2.5.6', 'node-pty': '1.1.0' }) + expect(parsed.allowScripts).toEqual({ + '@parcel/watcher@2.5.6': true, + 'node-pty@1.1.0': true + }) const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c) const npmInstallIdx = execCalls.findIndex( @@ -694,7 +703,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { await deployAndLaunchRelay(conn) - expect(vi.mocked(acquireInstallLock)).toHaveBeenCalledTimes(1) + expect(vi.mocked(tryAcquireRelayRepairLock)).toHaveBeenCalledTimes(1) expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1) const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c) expect( @@ -734,9 +743,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { // Why: lock contention/wedge must not block a completed relay from launching // in degraded mode — repair is best-effort and we hold no lock to release. vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true) - vi.mocked(acquireInstallLock).mockRejectedValueOnce( - new Error('Could not acquire relay install lock at /x; another install is in progress') - ) + vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce(false) const conn = makeMockConnection(sftpCapture) feed([ '__ORCA_REMOTE_PLATFORM__ Linux x86_64', @@ -753,7 +760,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c) expect(execCalls.some((c) => c.includes('npm install'))).toBe(false) const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? '')) - expect(warnMessages.some((m) => m.includes('Could not lock native-deps repair'))).toBe(true) + expect(warnMessages.some((m) => m.includes('repair lock is busy'))).toBe(true) }) it('loads native bindings when checking whether a completed relay needs repair', async () => { @@ -792,6 +799,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { await deployAndLaunchRelay(conn) expect(vi.mocked(acquireInstallLock)).not.toHaveBeenCalled() + expect(vi.mocked(tryAcquireRelayRepairLock)).not.toHaveBeenCalled() expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled() const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c) expect(execCalls.some((c) => c.includes('npm install'))).toBe(false) diff --git a/src/main/ssh/ssh-relay-repair-lock.ts b/src/main/ssh/ssh-relay-repair-lock.ts new file mode 100644 index 00000000000..ebcac8c09ab --- /dev/null +++ b/src/main/ssh/ssh-relay-repair-lock.ts @@ -0,0 +1,61 @@ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { + acquireInstallLockParentCommand, + tryCreateInstallLockCommand, + tryStealInstallLockCommand +} from './ssh-relay-install-lock-commands' +import { + getRemoteHostPlatform, + isWindowsRemoteHost, + joinRemotePath, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { INSTALL_LOCK_STALE_SECONDS, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-versioned-install' + +const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') + +function execHostCommand( + conn: SshConnection, + host: RemoteHostPlatform, + command: string +): Promise { + return execCommand(conn, command, { wrapCommand: !isWindowsRemoteHost(host) }) +} + +/** + * Try once to acquire the install lock for best-effort repair work. + * + * Why: a completed relay can launch in degraded mode, so repair must not wait + * behind another installer. An already-stale lock is still recovered now. + */ +export async function tryAcquireRelayRepairLock( + conn: SshConnection, + remoteRelayDir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST +): Promise { + const lockDir = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME) + try { + await execHostCommand(conn, host, acquireInstallLockParentCommand(host, remoteRelayDir)) + const firstAttempt = await execHostCommand( + conn, + host, + tryCreateInstallLockCommand(host, lockDir) + ) + if (firstAttempt.trim().endsWith('OK')) { + return true + } + const steal = await execHostCommand( + conn, + host, + tryStealInstallLockCommand(host, lockDir, INSTALL_LOCK_STALE_SECONDS) + ) + if (steal.trim().endsWith('OK')) { + console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`) + return true + } + return false + } catch { + return false + } +} diff --git a/src/main/ssh/ssh-relay-versioned-install.test.ts b/src/main/ssh/ssh-relay-versioned-install.test.ts index 244ae80a48c..e7e96ad7c2f 100644 --- a/src/main/ssh/ssh-relay-versioned-install.test.ts +++ b/src/main/ssh/ssh-relay-versioned-install.test.ts @@ -23,6 +23,7 @@ import { abandonInstall, gcOldRelayVersions } from './ssh-relay-versioned-install' +import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock' import { execCommand } from './ssh-relay-deploy-helpers' import { getRemoteHostPlatform } from './ssh-remote-platform' import type { SshConnection } from './ssh-connection' @@ -131,6 +132,38 @@ describe('acquireInstallLock', () => { expect(mockExec).toHaveBeenCalledTimes(2) }) + it('returns immediately when a best-effort repair lock is busy', async () => { + mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('BUSY').mockResolvedValueOnce('BUSY') + + await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe(false) + + const commands = mockExec.mock.calls.map(([, command]) => command) + expect(commands.some((command) => command.includes('.install-lock.steal'))).toBe(true) + expect(commands.filter((command) => command.startsWith('rm -rf'))).toHaveLength(0) + }) + + it('does not steal a live repair lock that exceeds the deploy backstop', async () => { + // Why: repair can run npm install plus rebuild under the same lock; a + // second reconnect must launch degraded rather than corrupt node_modules. + mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('BUSY').mockResolvedValueOnce('BUSY') + + await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe(false) + + const commands = mockExec.mock.calls.map(([, command]) => command) + expect(commands.some((command) => command.includes('.install-lock.steal'))).toBe(true) + expect(commands.filter((command) => command.startsWith('rm -rf'))).toHaveLength(0) + }) + + it('recovers a stale best-effort repair lock without polling', async () => { + mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('BUSY').mockResolvedValueOnce('OK') + + await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe(true) + + const commands = mockExec.mock.calls.map(([, command]) => command) + expect(commands.some((command) => command.includes('.install-lock.steal'))).toBe(true) + expect(commands.filter((command) => command.includes('.install-lock.steal'))).toHaveLength(1) + }) + it('polls until the lock becomes available (concurrent installer wins, then we acquire)', async () => { vi.useFakeTimers() try { @@ -159,36 +192,29 @@ describe('acquireInstallLock', () => { } }) - it('steals a stale lock and retries with a reset timeout window', async () => { + it('steals a stale lock with a serialized remote steal command', async () => { vi.useFakeTimers({ now: 1_700_000_000_000 }) try { - let mkdirCalls = 0 mockExec.mockImplementation(async (_conn: unknown, cmd: string) => { if (cmd.startsWith('mkdir -p')) { return '' } + if (cmd.includes('.install-lock.steal')) { + return 'OK' + } if (cmd.includes('mkdir') && cmd.includes('.install-lock')) { - mkdirCalls++ - return mkdirCalls > 200 ? 'OK' : 'BUSY' - } - if (cmd.includes('stat')) { - return `${Math.floor((Date.now() - 10 * 60 * 1000) / 1000)}\n` - } - if (cmd.startsWith('rm -rf')) { - mkdirCalls = 1000 - return '' + return 'BUSY' } return '' }) const promise = acquireInstallLock(conn, '/r') - // Drive through the full timeout (120s) so the stale-recovery branch - // fires, then drive a few more seconds for the post-recovery retry. - await vi.advanceTimersByTimeAsync(125_000) + // Drive through the full deploy bound so stale recovery fires. + await vi.advanceTimersByTimeAsync(545_000) await promise const cmds = mockExec.mock.calls.map(([, c]) => c) - expect(cmds.some((c) => c.includes('rm -rf') && c.includes('.install-lock'))).toBe(true) + expect(cmds.some((c) => c.includes('.install-lock.steal'))).toBe(true) } finally { vi.useRealTimers() } @@ -201,23 +227,50 @@ describe('acquireInstallLock', () => { if (cmd.startsWith('mkdir -p')) { return '' } + if (cmd.includes('.install-lock.steal')) { + return 'BUSY' + } if (cmd.includes('mkdir') && cmd.includes('.install-lock')) { return 'BUSY' } if (cmd.includes('stat')) { - return `${Math.floor(Date.now() / 1000)}\n` + return '0\n' } return '' }) const rejection = expect(acquireInstallLock(conn, '/r')).rejects.toThrow(/not yet stale/i) - await vi.advanceTimersByTimeAsync(125_000) + await vi.advanceTimersByTimeAsync(545_000) await rejection } finally { vi.useRealTimers() } }) + it('keeps waiting while a slow first installer is within the deploy bound', async () => { + vi.useFakeTimers({ now: 1_700_000_000_000 }) + try { + const availableAt = Date.now() + 500_000 + mockExec.mockImplementation(async (_conn: unknown, cmd: string) => { + if (cmd.startsWith('mkdir -p')) { + return '' + } + if (cmd.includes('mkdir') && cmd.includes('.install-lock')) { + return Date.now() >= availableAt ? 'OK' : 'BUSY' + } + return '' + }) + + const promise = acquireInstallLock(conn, '/r') + await vi.advanceTimersByTimeAsync(501_000) + + await expect(promise).resolves.toBeUndefined() + expect(mockExec.mock.calls.some(([, cmd]) => cmd.includes('.install-lock.steal'))).toBe(false) + } finally { + vi.useRealTimers() + } + }) + it('finalizeInstall writes .install-complete then removes the lock', async () => { mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('') await finalizeInstall(conn, '/r') @@ -273,8 +326,8 @@ describe('gcOldRelayVersions', () => { it('skips siblings whose .install-lock is held and fresh', async () => { mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\n') mockExec.mockResolvedValueOnce('LOCKED') - // isLockStale: mtime ~now → not stale. - mockExec.mockResolvedValueOnce(`${Math.floor(Date.now() / 1000)}\n`) + // isLockStale: age ~now → not stale. + mockExec.mockResolvedValueOnce('0\n') await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb') const cmds = mockExec.mock.calls.map(([, c]) => c) expect(cmds.some((c) => c.includes('rm -rf'))).toBe(false) @@ -283,9 +336,8 @@ describe('gcOldRelayVersions', () => { it('removes a sibling with a stale lock + .install-complete (rm-lock failed mid-finalize)', async () => { mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\n') mockExec.mockResolvedValueOnce('LOCKED') - // isLockStale: mtime well in the past → stale. - const staleSec = Math.floor((Date.now() - 10 * 60 * 1000) / 1000) - mockExec.mockResolvedValueOnce(`${staleSec}\n`) + // isLockStale: age well above the stale window → stale. + mockExec.mockResolvedValueOnce(`${21 * 60}\n`) mockExec.mockResolvedValueOnce('COMPLETE') // .install-complete present mockExec.mockResolvedValueOnce('') // socket probe → no ALIVE mockExec.mockResolvedValueOnce('') // rm -rf diff --git a/src/main/ssh/ssh-relay-versioned-install.ts b/src/main/ssh/ssh-relay-versioned-install.ts index a669f0b7875..4028bf4233b 100644 --- a/src/main/ssh/ssh-relay-versioned-install.ts +++ b/src/main/ssh/ssh-relay-versioned-install.ts @@ -15,16 +15,20 @@ import { existsSync, readFileSync } from 'node:fs' import type { SshConnection } from './ssh-connection' import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' +import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing' import { acquireInstallLockParentCommand, + lockAgeSecondsCommand, + tryCreateInstallLockCommand, + tryStealInstallLockCommand +} from './ssh-relay-install-lock-commands' +import { listRelayBaseDirsCommand, - lockMtimeEpochCommand, probeDirectoryExistsCommand, probeFileExistsCommand, probeRelayInstalledCommand, relayLivenessProbeCommand, removeRemoteTreeCommand, - tryCreateInstallLockCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' import { @@ -51,18 +55,21 @@ const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/ // living on remote disks forever. const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ -const INSTALL_LOCK_NAME = '.install-lock' +export const RELAY_INSTALL_LOCK_NAME = '.install-lock' const INSTALL_COMPLETE_NAME = '.install-complete' const INSTALL_LOCK_POLL_MS = 1_000 -const INSTALL_LOCK_TIMEOUT_MS = 120_000 +// Why: the lock holder can legitimately use the full deploy bound for upload, +// native install/rebuild, probes, and finalization. A concurrent first install +// must not fail earlier; completed-relay repair uses a separate one-shot path. +const INSTALL_LOCK_TIMEOUT_MS = RELAY_DEPLOY_TIMEOUT_MS // Why: a stale lock dir from a crashed installer must be recoverable without // user intervention. After the timeout we check the lock's mtime; if it's // older than this window the previous installer is assumed dead and we steal -// the lock. 2 minutes is well above a normal `npm install node-pty` runtime -// (10–60s on slow hosts) so a slow concurrent installer is not falsely -// declared dead. -const INSTALL_LOCK_STALE_MS = 120_000 +// the lock. Native-deps repair can keep running after the deploy backstop wins +// Promise.race, so leave enough room for its bounded under-lock exec sequence. +const INSTALL_LOCK_STALE_MS = 20 * 60_000 +export const INSTALL_LOCK_STALE_SECONDS = INSTALL_LOCK_STALE_MS / 1000 const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') type RelayInstalledProbeOptions = { @@ -166,13 +173,12 @@ export async function acquireInstallLock( remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { - const lockDir = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) + const lockDir = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME) // Why: the parent dir may not exist yet on a first install. mkdir -p is // safe to run multiple times — it's a no-op if the dir already exists. await execHostCommand(conn, host, acquireInstallLockParentCommand(host, remoteRelayDir)) let start = Date.now() - let recoveredOnce = false while (true) { try { const result = await execHostCommand(conn, host, tryCreateInstallLockCommand(host, lockDir)) @@ -183,24 +189,16 @@ export async function acquireInstallLock( /* mkdir failed with non-zero — fall through to BUSY treatment */ } if (Date.now() - start >= INSTALL_LOCK_TIMEOUT_MS) { - if (recoveredOnce) { - throw new Error( - `Could not acquire relay install lock at ${lockDir} after ${ - INSTALL_LOCK_TIMEOUT_MS / 1000 - }s; another install is in progress or the lock is wedged.` - ) - } - // Stale-lock recovery: if the lock dir's mtime is older than the stale - // window, the previous installer crashed. Steal it and retry once, - // resetting the timeout window so a single post-recovery race doesn't - // immediately exhaust the budget. - const ageOk = await isLockStale(conn, lockDir, host) - if (ageOk) { + // Stale-lock recovery is one remote command so two observers cannot both + // delete a lock that the other one just recreated. + const steal = await execHostCommand( + conn, + host, + tryStealInstallLockCommand(host, lockDir, INSTALL_LOCK_STALE_SECONDS) + ).catch(() => 'BUSY') + if (steal.trim().endsWith('OK')) { console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`) - await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {}) - recoveredOnce = true - start = Date.now() - continue + return } throw new Error( `Could not acquire relay install lock at ${lockDir} after ${ @@ -212,22 +210,20 @@ export async function acquireInstallLock( } } -async function isLockStale( +export async function isRelayInstallLockStale( conn: SshConnection, lockDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { try { - // Why: `stat` flags differ between GNU coreutils (Linux) and BSD (macOS). - // We try GNU first, then BSD; both produce a Unix epoch in seconds on - // stdout. If both fail we conservatively treat the lock as not stale. - const out = await execHostCommand(conn, host, lockMtimeEpochCommand(host, lockDir)) - const mtimeSec = Number.parseInt(out.trim(), 10) - if (!Number.isFinite(mtimeSec)) { + // Why: compute age on the remote host so clock skew between two Orca + // clients cannot make a live repair lock look older than it is. + const out = await execHostCommand(conn, host, lockAgeSecondsCommand(host, lockDir)) + const ageSec = Number.parseInt(out.trim(), 10) + if (!Number.isFinite(ageSec) || ageSec < 0) { return false } - const ageMs = Date.now() - mtimeSec * 1000 - return ageMs > INSTALL_LOCK_STALE_MS + return ageSec * 1000 > INSTALL_LOCK_STALE_MS } catch { return false } @@ -245,7 +241,7 @@ export async function finalizeInstall( host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { const sentinel = joinRemotePath(host, remoteRelayDir, INSTALL_COMPLETE_NAME) - const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) + const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME) await execHostCommand(conn, host, writeRemoteEmptyFileCommand(host, sentinel)) await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {}) } @@ -260,7 +256,7 @@ export async function abandonInstall( remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { - const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) + const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME) await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {}) } @@ -346,7 +342,7 @@ async function isCandidateSafeToRemove( ): Promise { const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name) - const lockDir = joinRemotePath(host, dir, INSTALL_LOCK_NAME) + const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) const lockProbe = await execHostCommand( conn, host, @@ -356,13 +352,13 @@ async function isCandidateSafeToRemove( if (locked) { // Why: a locked dir is normally unsafe to remove — but a STALE lock - // (mtime older than INSTALL_LOCK_STALE_MS) means the previous installer + // (remote age older than INSTALL_LOCK_STALE_MS) means the previous installer // crashed and is never coming back. If the dir also has the // .install-complete sentinel (touch succeeded but the rm-lock at the // end of finalizeInstall failed), removing the dir is safe — no // installer is racing us, and the daemon (if any) keeps running off // its already-loaded code regardless of disk state. - if (!(await isLockStale(conn, lockDir, host))) { + if (!(await isRelayInstallLockStale(conn, lockDir, host))) { return false } process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`) diff --git a/src/main/ssh/ssh-remote-commands.test.ts b/src/main/ssh/ssh-remote-commands.test.ts index 952daf9edde..ae62a46de7c 100644 --- a/src/main/ssh/ssh-remote-commands.test.ts +++ b/src/main/ssh/ssh-remote-commands.test.ts @@ -1,4 +1,21 @@ +import { execFileSync, spawn, spawnSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + statSync, + utimesSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { + lockAgeSecondsCommand, + tryCreateInstallLockCommand, + tryStealInstallLockCommand +} from './ssh-relay-install-lock-commands' import { commandInRemoteDirectory, commandWithNodePath, @@ -6,19 +23,52 @@ import { makeRemoteDirectoryCommand, probeRelayInstalledCommand, readRemoteHomeCommand, - relayLivenessProbeCommand, - tryCreateInstallLockCommand + relayLivenessProbeCommand } from './ssh-remote-commands' import { getRemoteHostPlatform } from './ssh-remote-platform' const posix = getRemoteHostPlatform('linux-x64') const windows = getRemoteHostPlatform('win32-x64') +const powerShellExecutable = ( + process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh'] +).find((candidate) => { + const result = spawnSync(candidate, ['-NoProfile', '-NonInteractive', '-Command', 'exit 0'], { + stdio: 'ignore' + }) + return result.status === 0 +}) function decodePowerShellCommand(command: string): string { const match = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/) return match ? Buffer.from(match[1], 'base64').toString('utf16le') : '' } +function runShellCommand(command: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn('/bin/sh', ['-c', command], { + stdio: ['ignore', 'pipe', 'pipe'] + }) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + child.stderr.on('data', (chunk) => { + stderr += chunk + }) + child.on('error', reject) + child.on('close', (code) => { + if (code === 0) { + resolve(stdout) + return + } + reject(new Error(`shell exited ${code}: ${stderr}`)) + }) + }) +} + describe('ssh remote command builders', () => { it('keeps POSIX deploy commands POSIX-native', () => { expect(readRemoteHomeCommand(posix)).toBe('echo $HOME') @@ -101,6 +151,121 @@ describe('ssh remote command builders', () => { expect(script).not.toContain('}; catch') }) + it('computes install-lock age on the remote host clock', () => { + const posixCommand = lockAgeSecondsCommand(posix, '/home/me/.orca-remote/relay/.install-lock') + const windowsScript = decodePowerShellCommand( + lockAgeSecondsCommand(windows, 'C:/Users/me/.orca-remote/relay/.install-lock') + ) + + expect(posixCommand).toContain('date +%s') + expect(posixCommand).toContain('echo "$age"') + expect(windowsScript).toContain('[DateTimeOffset]::UtcNow.ToUnixTimeSeconds()') + expect(windowsScript).toContain('Write-Output ($now - $mtime)') + }) + + it('serializes stale install-lock recovery under a sibling steal mutex', () => { + const posixCommand = tryStealInstallLockCommand( + posix, + '/home/me/.orca-remote/relay/.install-lock', + 20 * 60 + ) + const windowsScript = decodePowerShellCommand( + tryStealInstallLockCommand(windows, 'C:/Users/me/.orca-remote/relay/.install-lock', 20 * 60) + ) + + expect(posixCommand).toContain('.install-lock.steal') + expect(posixCommand).toContain('$mtime') + expect(posixCommand).toContain('-gt 1200') + expect(posixCommand).toContain('current_mtime') + expect(posixCommand).toContain('steal_age') + expect(posixCommand).toContain('steal="$steal.next.$steal_token"') + expect(posixCommand).toContain('trap') + expect(windowsScript).toContain('$lock.steal.$mtime') + expect(windowsScript).toContain('-gt 1200') + expect(windowsScript).toContain('$currentIdentity -eq $lockIdentity') + expect(windowsScript).toContain('$stealAge -le 120') + expect(windowsScript).toContain('$steal = "$steal.next.$stealIdentity"') + expect(windowsScript).toContain('New-Item -ItemType Directory -Path $steal') + expect(windowsScript).toContain('finally') + }) + + it.runIf(powerShellExecutable)('emits a parseable Windows stale-lock recovery command', () => { + const script = decodePowerShellCommand( + tryStealInstallLockCommand( + windows, + 'C:/Users/orca-missing/.orca-remote/relay/.install-lock', + 20 * 60 + ) + ) + const result = spawnSync( + powerShellExecutable!, + ['-NoProfile', '-NonInteractive', '-Command', script], + { encoding: 'utf8' } + ) + + expect(result.status, result.stderr).toBe(0) + expect(result.stdout.trim()).toBe('BUSY') + }) + + it.runIf(process.platform !== 'win32')( + 'recovers a stale POSIX steal mutex left by a crashed stealer', + () => { + const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-')) + try { + const lockDir = join(root, '.install-lock') + mkdirSync(lockDir) + const staleDate = new Date(Date.now() - 60 * 60_000) + utimesSync(lockDir, staleDate, staleDate) + const lockMtimeSeconds = Math.floor(statSync(lockDir).mtimeMs / 1000) + const stealDir = `${lockDir}.steal.${lockMtimeSeconds}` + mkdirSync(stealDir) + utimesSync(stealDir, staleDate, staleDate) + + const command = tryStealInstallLockCommand(posix, lockDir, 20 * 60) + const output = execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) + + expect(output.trim()).toBe('OK') + expect(existsSync(lockDir)).toBe(true) + expect(existsSync(stealDir)).toBe(true) + expect(readdirSync(root).some((name) => name.includes('.next.'))).toBe(false) + expect(Math.floor(statSync(lockDir).mtimeMs / 1000)).toBeGreaterThan(lockMtimeSeconds) + } finally { + rmSync(root, { recursive: true, force: true }) + } + } + ) + + it.runIf(process.platform !== 'win32')( + 'lets only one POSIX stealer take over a preexisting stale steal mutex', + async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-race-')) + try { + const lockDir = join(root, '.install-lock') + mkdirSync(lockDir) + const staleDate = new Date(Date.now() - 60 * 60_000) + utimesSync(lockDir, staleDate, staleDate) + const lockMtimeSeconds = Math.floor(statSync(lockDir).mtimeMs / 1000) + const stealDir = `${lockDir}.steal.${lockMtimeSeconds}` + mkdirSync(stealDir) + utimesSync(stealDir, staleDate, staleDate) + + const command = tryStealInstallLockCommand(posix, lockDir, 20 * 60) + const outputs = await Promise.all( + Array.from({ length: 64 }, () => runShellCommand(command)) + ) + const okCount = outputs.filter((output) => output.trim().endsWith('OK')).length + + expect(okCount).toBe(1) + expect(existsSync(lockDir)).toBe(true) + expect(existsSync(stealDir)).toBe(true) + expect(readdirSync(root).some((name) => name.includes('.next.'))).toBe(false) + expect(readdirSync(root).some((name) => name.includes('.tombstone'))).toBe(false) + } finally { + rmSync(root, { recursive: true, force: true }) + } + } + ) + it('makes Windows remote directory changes fail before running scoped commands', () => { const scopedCommand = decodePowerShellCommand( commandInRemoteDirectory(windows, 'C:/Users/me/.orca-remote/relay-0.1.0', "'READY'") diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 339f4aef594..751c91017da 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -81,36 +81,6 @@ export function probeRelayInstalledCommand( ) } -export function acquireInstallLockParentCommand( - host: RemoteHostPlatform, - remoteRelayDir: string -): string { - return makeRemoteDirectoryCommand(host, remoteRelayDir) -} - -export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: string): string { - if (!isWindowsRemoteHost(host)) { - return `mkdir ${shellEscape(lockDir)} 2>&1 && echo OK || echo BUSY` - } - // New-Item has no -LiteralPath parameter; using it breaks stock Windows PowerShell. - return powerShellCommand( - `$ErrorActionPreference = "Stop"; try { $null = New-Item -ItemType Directory -Path ${powerShellLiteral(lockDir)}; 'OK' } catch { 'BUSY' }` - ) -} - -export function lockMtimeEpochCommand(host: RemoteHostPlatform, lockDir: string): string { - if (!isWindowsRemoteHost(host)) { - return `stat -c %Y ${shellEscape(lockDir)} 2>/dev/null || stat -f %m ${shellEscape(lockDir)} 2>/dev/null || echo` - } - return powerShellCommand( - [ - `$item = Get-Item -LiteralPath ${powerShellLiteral(lockDir)} -ErrorAction Stop`, - '$dto = [DateTimeOffset]$item.LastWriteTimeUtc', - 'Write-Output $dto.ToUnixTimeSeconds()' - ].join('; ') - ) -} - export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string { if (!isWindowsRemoteHost(host)) { return `ls -1 ${shellEscape(baseDir)} 2>/dev/null || true` diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index c1de6809c15..74fd755e6fd 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -51,7 +51,14 @@ async function loadPty(): Promise { ptyModule = await import('node-pty') return ptyModule } catch { - return null + try { + // Why: Node caches rejected dynamic imports, while a failed CommonJS + // load remains retryable after deploy repairs node_modules in place. + ptyModule = require('node-pty') as typeof NodePty + return ptyModule + } catch { + return null + } } } diff --git a/src/relay/subprocess.test.ts b/src/relay/subprocess.test.ts index ee20461dd68..ea017bb8316 100644 --- a/src/relay/subprocess.test.ts +++ b/src/relay/subprocess.test.ts @@ -1,5 +1,13 @@ import { afterAll, beforeAll, describe, expect, it, afterEach } from 'vitest' -import { existsSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs' +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + unlinkSync, + writeFileSync +} from 'node:fs' import { rm } from 'node:fs/promises' import * as path from 'node:path' import { tmpdir } from 'node:os' @@ -103,6 +111,35 @@ describe('Subprocess: Relay entry point', () => { expect(readFileSync(relayEntry, 'utf8')).not.toContain('.toReversed(') }) + it('loads node-pty after an in-place dependency repair without restarting', async () => { + tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-native-repair-')) + const repairedRelayEntry = path.join(tmpDir, 'relay.js') + copyFileSync(relayEntry, repairedRelayEntry) + + relay = spawnRelay(repairedRelayEntry) + await relay.sentinelReceived + + const failedId = relay.send('pty.spawn', { cols: 80, rows: 24 }) + const failed = await relay.waitForResponse(failedId) + expect(failed.error?.message).toContain('node-pty is not available') + + const nodePtyDir = path.join(tmpDir, 'node_modules', 'node-pty') + mkdirSync(nodePtyDir, { recursive: true }) + writeFileSync( + path.join(nodePtyDir, 'index.js'), + `module.exports = { spawn() { return { + pid: process.pid, + process: 'mock-shell', + onData() {}, onExit() {}, write() {}, resize() {}, kill() {}, clear() {} + } } }\n` + ) + + const repairedId = relay.send('pty.spawn', { cols: 80, rows: 24 }) + const repaired = await relay.waitForResponse(repairedId) + expect(repaired.error).toBeUndefined() + expect(repaired.result).toMatchObject({ id: 'pty-1' }) + }, 10_000) + it('responds to fs.stat over stdin/stdout', async () => { tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-sub-')) writeFileSync(path.join(tmpDir, 'test.txt'), 'hello')