From 448b7e2f55cfd87a6c586a973dbdbaae36a79e2d Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 29 Aug 2026 03:36:32 -0700 Subject: [PATCH] fix(ssh): recover install locks after host reboot --- config/reliability-gates.jsonc | 46 +++++-- .../ssh/ssh-relay-install-lock-commands.ts | 116 ++++++++++++++++-- src/main/ssh/ssh-relay-install-lock.ts | 8 +- src/main/ssh/ssh-remote-commands.test.ts | 96 +++++++++++++++ 4 files changed, 243 insertions(+), 23 deletions(-) diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 95349190bdc..38c64abe945 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -998,7 +998,7 @@ }, { "id": "ssh-relay.staged-upload-recovery", - "title": "SSH relay uploads remain retryable before the shared install lock", + "title": "SSH relay uploads and install locks remain retryable", "maturity": "experimental", "protection": "partial", "owner": "ssh-relay-install", @@ -1007,19 +1007,21 @@ "SSH relay first install", "split shell and SFTP namespaces", "system SSH transfer fallback", - "relay install retry after cancellation" + "relay install retry after cancellation", + "post-promotion retry after execution-host restart" ], "platforms": ["macos", "linux", "windows"], "providers": ["ssh2", "system-ssh"], "coveredPlatforms": ["macos", "linux"], "coveredProviders": ["ssh2", "system-ssh"], - "coverageNotes": "Deterministic unit, exact POSIX shell, native ARM macOS PowerShell 7.6.4, and real ssh2 SFTP-wire tests cover lock ordering, concurrent-install loss, fixed-slot ownership identity, payload-only promotion, bounded stale-stage reclamation, installed-fast-path draining, joined cancellation teardown, cross-version isolation, split-SFTP redirection, and system-SSH bypass. A throwaway linux-arm64 Docker sshd reached through a non-loopback LAN address covers live bytes-in-flight SFTP cancellation, injected unconfirmed cancellation, immediate retry against a real Git repository, fixed-slot recovery behind unclaimable entries, and real version-GC filtering with 15,197 unrelated names.", + "coverageNotes": "Deterministic unit, exact POSIX shell, native ARM macOS PowerShell 7.6.4, and real ssh2 SFTP-wire tests cover lock ordering, concurrent-install loss, fixed-slot ownership identity, payload-only promotion, bounded stale-stage reclamation, boot-identity takeover, installed-fast-path draining, joined cancellation teardown, cross-version isolation, split-SFTP redirection, and system-SSH bypass. A throwaway linux-arm64 Docker sshd reached through a non-loopback LAN address covers live bytes-in-flight SFTP cancellation, injected unconfirmed cancellation, immediate retry against a real Git repository, fixed-slot recovery behind unclaimable entries, and real version-GC filtering with 15,197 unrelated names.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/9828", - "https://github.com/stablyai/orca/pull/10207" + "https://github.com/stablyai/orca/pull/10207", + "https://github.com/stablyai/orca/issues/17144" ], - "invariant": "A first-install relay transfer must complete in an attempt-owned fixed staging slot before acquiring the shared version install lock. Reservation, promotion, confirmed cleanup, and stale recovery must reject path replacement, persisted-identity mismatch, POSIX symlinks, and Windows reparse points. Recovery examines only eight fixed slot/claim/delete names and removes at most one stale valid stage per call; eight unclaimable states fail with an explicit manual-recovery message. Split-SFTP hosts must prove the stage identity on the exact transfer session, only payload contents may be promoted under the shared lock, and cancellation must boundedly join SFTP, stream, local file-handle, and transfer settlement.", - "oracle": "Pause a real ssh2 SFTP relay.js write after one remotely acknowledged chunk, prove the remote file is partial, abort the live transfer, and require no shared .install-lock, leaked local descriptor, or foreign-process termination. Separately inject two unconfirmed cancellations, require an independent deployment to install, launch, answer relay RPC, and read a real repository HEAD. Replace one retained fixed slot with an old-mtime same-owner directory while preserving the original, add a fixed-slot POSIX symlink, and require installed-path recovery to skip both while reclaiming a valid stale slot behind them. Add 15,197 unrelated relay-shaped names and run the real version GC, requiring bounded stdout and no removal. Unit and wire contracts cover exact POSIX and native PowerShell 0/1/7/8/9+ quota behavior, no-follow identity fencing, payload symlink/reparse rejection, one-item repeated draining, zero lock acquisition before upload settlement, joined transfer/channel teardown including never-settling failures, SFTP redirection, package.json namespace ownership, promotion only after the lock, cross-version isolation, and system-SSH behavior.", + "invariant": "A first-install relay transfer must complete in an attempt-owned fixed staging slot before acquiring the shared version install lock. Reservation, promotion, confirmed cleanup, and stale recovery must reject path replacement, persisted-identity mismatch, POSIX symlinks, and Windows reparse points. A newly acquired install lock atomically records the execution host's boot identity; only a verified identity change or the existing stale-age proof may replace it, while legacy, missing, malformed, and unreadable identity state must retain the conservative stale fallback. Recovery examines only eight fixed slot/claim/delete names and removes at most one stale valid stage per call; eight unclaimable states fail with an explicit manual-recovery message. Split-SFTP hosts must prove the stage identity on the exact transfer session, only payload contents may be promoted under the shared lock, and cancellation must boundedly join SFTP, stream, local file-handle, and transfer settlement.", + "oracle": "Pause a real ssh2 SFTP relay.js write after one remotely acknowledged chunk, prove the remote file is partial, abort the live transfer, and require no shared .install-lock, leaked local descriptor, or foreign-process termination. Separately inject two unconfirmed cancellations, require an independent deployment to install, launch, answer relay RPC, and read a real repository HEAD. Keep a fresh install lock on the current POSIX or Windows boot and require takeover to fail; replace its bounded identity with a prior-boot value and race concurrent recoverers, requiring exactly one atomic winner, a current successor identity, and no tombstone residue; omit the marker and require the legacy lock to remain fenced. Replace one retained fixed slot with an old-mtime same-owner directory while preserving the original, add a fixed-slot POSIX symlink, and require installed-path recovery to skip both while reclaiming a valid stale slot behind them. Add 15,197 unrelated relay-shaped names and run the real version GC, requiring bounded stdout and no removal. Unit and wire contracts cover exact POSIX and native PowerShell 0/1/7/8/9+ quota behavior, no-follow identity fencing, payload symlink/reparse rejection, one-item repeated draining, zero lock acquisition before upload settlement, joined transfer/channel teardown including never-settling failures, SFTP redirection, package.json namespace ownership, promotion only after the lock, cross-version isolation, and system-SSH behavior.", "commands": [ "node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/ssh-relay-deploy-staged-upload.test.ts src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts src/main/ssh/ssh-relay-sftp-namespace-install.test.ts src/main/ssh/ssh-relay-install-namespace.test.ts src/main/ssh/ssh-relay-upload-stage-commands.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts src/main/ssh/ssh-relay-cross-version-isolation.test.ts", "ORCA_REVIEW_SSH_UPLOAD_CANCEL=1 ORCA_REVIEW_SSH_TARGET_HOST= ORCA_REVIEW_SSH_IMAGE= ORCA_REVIEW_EXPECT_RECOVERY=1 pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-upload-cancel.docker.test.ts --maxWorkers=1 --reporter=verbose" @@ -1074,7 +1076,8 @@ "assertions": [ "uses encoded PowerShell for Windows deploy commands", "enumerates Windows staging children before copying", - "lets only one PowerShell caller acquire a legacy-visible lock" + "lets only one PowerShell caller acquire a legacy-visible lock", + "keeps current and legacy fresh locks fenced while one concurrent caller replaces a previous-boot lock" ] }, { @@ -1130,7 +1133,7 @@ }, "performanceBudget": { "required": true, - "evidence": "Stage recovery examines only eight fixed slot/claim/delete paths and reclaims at most one stale valid stage per invocation; installed reconnects launch before asynchronous recovery. Full quota produces an explicit error instead of unbounded cleanup. Version GC still scans the relay base directory, but remote filtering caps stdout and local candidate work at 64. Cancellation adds one bounded five-second join of channel and transfer settlement." + "evidence": "Stage recovery examines only eight fixed slot/claim/delete paths and reclaims at most one stale valid stage per invocation; installed reconnects launch before asynchronous recovery. Install-lock identity is recorded once per acquisition and checked only during the existing at-most-once-per-minute recovery probe; marker reads are capped at 128 bytes. Full quota produces an explicit error instead of unbounded cleanup. Version GC still scans the relay base directory, but remote filtering caps stdout and local candidate work at 64. Cancellation adds one bounded five-second join of channel and transfer settlement." }, "promotionCriteria": [ "Collect 100 consecutive CI passes or 14 days of soak history.", @@ -1140,6 +1143,7 @@ "knownGaps": [ "The live Docker target is Linux ARM64 with a unified namespace; split-SFTP behavior is covered by real ssh2 wire and deterministic deploy fixtures.", "Native PowerShell coverage runs on ARM macOS with POSIX filesystem paths; Windows OpenSSH, Windows PowerShell 5.1, and system-SSH behavior remain command and transfer-contract coverage rather than a live target.", + "No live VM or WSL reboot is injected during native-dependency installation; deterministic host-native command tests provide the previous-boot, current-boot, legacy-marker, and concurrent-takeover oracle.", "The fixed pool retains up to eight relay bundles; eight foreign or otherwise unclaimable fixed states require manual inspection instead of automatic deletion.", "Version GC remotely filters and caps output but still scans the base .orca-remote directory; it does not promise constant remote enumeration time.", "The Docker oracle is opt-in because it requires a local image and a reachable non-loopback host address." @@ -8405,7 +8409,7 @@ "providers": ["local", "daemon", "wsl"], "coveredPlatforms": ["windows"], "coveredProviders": ["daemon"], - "coverageNotes": "Issue #8048 now has deterministic wrapper and cold-restore re-anchor tests plus a Windows PR-CI harness that drives the built daemon through 25 real ConPTY workspace-close races while an unrelated witness PTY stays alive. Keyboard reset, CJK repaint, WSL, and full visible Electron coverage remain gaps.", + "coverageNotes": "Issue #8048 now has deterministic wrapper and cold-restore re-anchor tests plus a Windows PR-CI harness that drives the built daemon through 25 real ConPTY workspace-close races while an unrelated witness PTY stays alive. A Windows-only patched-node-pty test injects EAGAIN on one ConPTY input pipe and requires only that PTY to close while an unrelated PTY remains writable; a daemon-level classifier test keeps the native exception backstop narrow. Keyboard reset, CJK repaint, WSL, and full visible Electron coverage remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6541", "https://github.com/stablyai/orca/pull/6858", @@ -8413,18 +8417,21 @@ "https://github.com/stablyai/orca/pull/6968", "https://github.com/stablyai/orca/pull/6970", "https://github.com/stablyai/orca/pull/6999", - "https://github.com/stablyai/orca/issues/8048" + "https://github.com/stablyai/orca/issues/8048", + "https://github.com/stablyai/orca/issues/17027" ], - "invariant": "Windows local and daemon terminals must spawn with the intended shell, survive overlapping graceful/forced workspace teardown without affecting unrelated PTYs, retain recovered scrollback across the fresh daemon's first checkpoint, accept normal Enter/Backspace/Arrow input after agent or TUI exit, render cursor/CJK/wide-glyph redraws without stale cells, and converge to nonzero applied size.", - "oracle": "The issue #8048 slice asserts one node-pty ConPTY close for a graceful-then-force sequence, atomically seeds recovered history before fresh shell output and re-anchoring, preserves recovery after seed failure plus adapter restart, and runs 25 built-daemon close races while checking victim session/PID reaping, a stable daemon PID, and a live witness PTY. A broader Windows live gate still needs shell input, resize, cursor, and CJK/wide-glyph pixel evidence.", + "invariant": "Windows local and daemon terminals must spawn with the intended shell, survive overlapping graceful/forced workspace teardown without affecting unrelated PTYs, contain an asynchronous ConPTY input-pipe failure to the affected terminal without killing the daemon, retain recovered scrollback across the fresh daemon's first checkpoint, accept normal Enter/Backspace/Arrow input after agent or TUI exit, render cursor/CJK/wide-glyph redraws without stale cells, and converge to nonzero applied size.", + "oracle": "The issue #8048 slice asserts one node-pty ConPTY close for a graceful-then-force sequence, atomically seeds recovered history before fresh shell output and re-anchoring, preserves recovery after seed failure plus adapter restart, and runs 25 built-daemon close races while checking victim session/PID reaping, a stable daemon PID, and a live witness PTY. The EAGAIN slice emits an error from one real patched node-pty Windows input socket, requires its terminal to become unwritable and run the normal per-PTY kill path, then writes through an unrelated PTY without an uncaught exception. A broader Windows live gate still needs shell input, resize, cursor, and CJK/wide-glyph pixel evidence.", "commands": [ - "pnpm vitest run src/main/daemon/pty-subprocess.test.ts src/main/daemon/daemon-pty-adapter.test.ts", + "pnpm vitest run src/main/daemon/pty-subprocess.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/daemon/node-pty-windows-input-error.win32.test.ts src/main/daemon/daemon-native-pty-exception.test.ts", "pnpm build:electron-vite && node config/scripts/windows-daemon-workspace-close-repro.mjs", "node config/scripts/windows-daemon-workspace-close-repro.mjs" ], "testFiles": [ "src/main/daemon/pty-subprocess.test.ts", "src/main/daemon/daemon-pty-adapter.test.ts", + "src/main/daemon/node-pty-windows-input-error.win32.test.ts", + "src/main/daemon/daemon-native-pty-exception.test.ts", "config/scripts/windows-daemon-workspace-close-repro.mjs" ], "assertionRefs": [ @@ -8441,6 +8448,18 @@ "a failed atomic history seed remains non-authoritative across adapter restart and cannot overwrite the recovery files" ] }, + { + "file": "src/main/daemon/node-pty-windows-input-error.win32.test.ts", + "assertions": [ + "an EAGAIN event retires only the affected patched node-pty terminal while a witness remains writable" + ] + }, + { + "file": "src/main/daemon/daemon-native-pty-exception.test.ts", + "assertions": [ + "the daemon suppresses native PTY errno failures while rejecting non-Error and unrelated logic failures" + ] + }, { "file": "config/scripts/windows-daemon-workspace-close-repro.mjs", "assertions": [ @@ -8482,6 +8501,7 @@ ], "knownGaps": [ "Real IME composition may require a separate lower-layer/native-text-forwarding gate.", + "The EAGAIN oracle injects the real input socket event rather than inducing kernel resource exhaustion on a packaged Windows host.", "The built-daemon harness proves process/session liveness but not renderer pixels; visible shell input, resize, cursor, and CJK repaint remain uncovered." ], "demotionRule": "Cannot promote while Windows E2E is flaky, silently skipped, or screenshot-only." diff --git a/src/main/ssh/ssh-relay-install-lock-commands.ts b/src/main/ssh/ssh-relay-install-lock-commands.ts index d6eab561a2d..0021400f897 100644 --- a/src/main/ssh/ssh-relay-install-lock-commands.ts +++ b/src/main/ssh/ssh-relay-install-lock-commands.ts @@ -2,6 +2,8 @@ import { shellEscape } from './ssh-connection-utils' import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' +const INSTALL_LOCK_BOOT_ID_NAME = '.boot-id' + export function acquireInstallLockParentCommand( host: RemoteHostPlatform, remoteRelayDir: string @@ -16,7 +18,13 @@ export function acquireInstallLockParentCommand( export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: string): string { if (!isWindowsRemoteHost(host)) { - return `mkdir ${shellEscape(lockDir)} 2>&1 && echo OK || echo BUSY` + return [ + `if mkdir ${shellEscape(lockDir)} 2>/dev/null; then`, + posixCurrentBootIdentityAssignment(host, 'current_boot_id'), + posixWriteBootIdentity(lockDir, 'current_boot_id'), + 'echo OK;', + 'else echo BUSY; fi' + ].join(' ') } // Why: old Orca clients recognize only a directory at `.install-lock`, while // concurrent New-Item calls can both report success in PowerShell 5.1. Keep @@ -30,6 +38,8 @@ export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: s '$null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop', "$owner = Join-Path $lock '.owner'", '$stream = [System.IO.File]::Open($owner, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)', + ...windowsCurrentBootIdentityStatements('$currentBootId'), + windowsWriteBootIdentityStatement('$lock', '$currentBootId'), "'OK'", '}', `} catch { 'BUSY' } finally { if ($null -ne $stream) { $stream.Dispose() } }` @@ -68,17 +78,25 @@ export function tryStealInstallLockCommand( staleAfterSeconds: number ): string { if (!isWindowsRemoteHost(host)) { - return posixStealInstallLockCommand(lockDir, staleAfterSeconds) + return posixStealInstallLockCommand(host, lockDir, staleAfterSeconds) } return windowsStealInstallLockCommand(lockDir, staleAfterSeconds) } -function posixStealInstallLockCommand(lockDir: string, staleAfterSeconds: number): string { +function posixStealInstallLockCommand( + host: RemoteHostPlatform, + lockDir: string, + staleAfterSeconds: number +): string { const escapedLockDir = shellEscape(lockDir) const escapedStealLockPrefix = shellEscape(`${lockDir}.steal`) return [ + posixCurrentBootIdentityAssignment(host, 'current_boot_id'), + `${posixReadBootIdentity(lockDir, 'recorded_boot_id')}`, + 'rebooted=0;', + 'if [ -n "$recorded_boot_id" ] && [ -n "$current_boot_id" ] && [ "$recorded_boot_id" != "$current_boot_id" ]; then rebooted=1; fi;', `${posixLockIdentityAssignment(lockDir, 'lock_key')} && mtime=\${lock_key%%:*} && now=$(date +%s) && age=$((now - mtime)) || age=0;`, - `if [ "\${age:-0}" -le ${staleAfterSeconds} ] 2>/dev/null; then echo BUSY; else`, + `if [ "\${age:-0}" -le ${staleAfterSeconds} ] 2>/dev/null && [ "$rebooted" != 1 ]; then echo BUSY; else`, `steal_root=${escapedStealLockPrefix};`, 'steal_generation=0;', 'steal="$steal_root.$steal_generation";', @@ -94,9 +112,16 @@ function posixStealInstallLockCommand(lockDir: string, staleAfterSeconds: number 'if [ "$owns_steal" = 1 ]; then', `trap 'rm -rf "$steal_root".* 2>/dev/null || true; rm -rf "$lock_tombstone" 2>/dev/null || true' EXIT;`, `${posixLockIdentityAssignment(lockDir, 'current_key')} && current_mtime=\${current_key%%:*} && current_now=$(date +%s) && current_age=$((current_now - current_mtime)) || current_age=0;`, - `if [ "$current_key" = "$lock_key" ] && [ "\${current_age:-0}" -gt ${staleAfterSeconds} ] 2>/dev/null; then`, + `${posixReadBootIdentity(lockDir, 'current_recorded_boot_id')}`, + 'current_rebooted=0;', + 'if [ -n "$current_recorded_boot_id" ] && [ -n "$current_boot_id" ] && [ "$current_recorded_boot_id" != "$current_boot_id" ]; then current_rebooted=1; fi;', + `if [ "$current_key" = "$lock_key" ] && { [ "\${current_age:-0}" -gt ${staleAfterSeconds} ] 2>/dev/null || [ "$current_rebooted" = 1 ]; }; then`, `lock_tombstone=${escapedLockDir}.tombstone.$$.$(date +%s);`, - `if [ ! -e "$lock_tombstone" ] && mv ${escapedLockDir} "$lock_tombstone" 2>/dev/null; then mkdir ${escapedLockDir} 2>&1 && echo OK || echo BUSY; else echo BUSY; fi;`, + `if [ ! -e "$lock_tombstone" ] && mv ${escapedLockDir} "$lock_tombstone" 2>/dev/null; then`, + `if mkdir ${escapedLockDir} 2>/dev/null; then`, + posixWriteBootIdentity(lockDir, 'current_boot_id'), + 'if [ "$current_rebooted" = 1 ]; then echo REBOOT_OK; else echo OK; fi;', + 'else echo BUSY; fi; else echo BUSY; fi;', 'else echo BUSY; fi;', 'else echo BUSY; fi; fi' ].join(' ') @@ -107,11 +132,14 @@ function windowsStealInstallLockCommand(lockDir: string, staleAfterSeconds: numb [ `$lock = ${powerShellLiteral(lockDir)}`, 'try {', + ...windowsCurrentBootIdentityStatements('$currentBootId'), + ...windowsReadBootIdentityStatements('$lock', '$recordedBootId'), + '$rebooted = (-not [string]::IsNullOrWhiteSpace($recordedBootId)) -and (-not [string]::IsNullOrWhiteSpace($currentBootId)) -and ($recordedBootId -cne $currentBootId)', '$item = Get-Item -LiteralPath $lock -ErrorAction Stop', '$mtime = ([DateTimeOffset]$item.LastWriteTimeUtc).ToUnixTimeSeconds()', '$lockIdentity = "${mtime}:$($item.CreationTimeUtc.Ticks)"', '$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()', - `if (($now - $mtime) -le ${staleAfterSeconds}) { 'BUSY' } else {`, + `if ((($now - $mtime) -le ${staleAfterSeconds}) -and (-not $rebooted)) { 'BUSY' } else {`, '$stealRoot = "$lock.steal"', '$stealGeneration = 0', '$steal = "$stealRoot.$stealGeneration"', @@ -140,11 +168,13 @@ function windowsStealInstallLockCommand(lockDir: string, staleAfterSeconds: numb '$currentMtime = ([DateTimeOffset]$current.LastWriteTimeUtc).ToUnixTimeSeconds()', '$currentIdentity = "${currentMtime}:$($current.CreationTimeUtc.Ticks)"', '$currentNow = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()', - `if (($currentIdentity -eq $lockIdentity) -and (($currentNow - $currentMtime) -gt ${staleAfterSeconds})) {`, + ...windowsReadBootIdentityStatements('$lock', '$currentRecordedBootId'), + '$currentRebooted = (-not [string]::IsNullOrWhiteSpace($currentRecordedBootId)) -and (-not [string]::IsNullOrWhiteSpace($currentBootId)) -and ($currentRecordedBootId -cne $currentBootId)', + `if (($currentIdentity -eq $lockIdentity) -and ((($currentNow - $currentMtime) -gt ${staleAfterSeconds}) -or $currentRebooted)) {`, '$lockTombstone = "$lock.tombstone.$PID.$([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds())"', 'Move-Item -LiteralPath $lock -Destination $lockTombstone -ErrorAction Stop', '$successorStream = $null', - "try { $null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop; $successorOwner = Join-Path $lock '.owner'; $successorStream = [System.IO.File]::Open($successorOwner, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None); 'OK' } catch { 'BUSY' } finally { if ($null -ne $successorStream) { $successorStream.Dispose() } }", + `try { $null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop; $successorOwner = Join-Path $lock '.owner'; $successorStream = [System.IO.File]::Open($successorOwner, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None); ${windowsWriteBootIdentityStatement('$lock', '$currentBootId')}; if ($currentRebooted) { 'REBOOT_OK' } else { 'OK' } } catch { 'BUSY' } finally { if ($null -ne $successorStream) { $successorStream.Dispose() } }`, "} else { 'BUSY' }", '}', "} catch { 'BUSY' } finally {", @@ -161,6 +191,74 @@ function windowsStealInstallLockCommand(lockDir: string, staleAfterSeconds: numb ) } +function posixCurrentBootIdentityAssignment( + host: RemoteHostPlatform, + variableName: string +): string { + if (host.os === 'darwin') { + return `${variableName}=$(sysctl -n kern.boottime 2>/dev/null | sed -n 's/^.*{ sec = \\([0-9][0-9]*\\),.*$/darwin:\\1/p');` + } + return [ + `${variableName}=;`, + 'kernel_boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) || kernel_boot_id=;', + `pid1_start_ticks=$(sed 's/^.*) //' /proc/1/stat 2>/dev/null | awk '{ print $20 }') || pid1_start_ticks=;`, + `if [ -n "$kernel_boot_id" ] && [ -n "$pid1_start_ticks" ]; then ${variableName}="linux:$kernel_boot_id:$pid1_start_ticks"; fi;` + ].join(' ') +} + +function posixReadBootIdentity(lockDir: string, variableName: string): string { + const markerPath = shellEscape(`${lockDir}/${INSTALL_LOCK_BOOT_ID_NAME}`) + return [ + `${variableName}=$(head -c 128 ${markerPath} 2>/dev/null | tr -d '\\r\\n') || ${variableName}=;`, + `if ! printf '%s\\n' "$${variableName}" | grep -Eq '^(linux:[0-9a-fA-F-]{1,64}:[0-9]{1,32}|darwin:[0-9]{1,32})$'; then ${variableName}=; fi;` + ].join(' ') +} + +function posixWriteBootIdentity(lockDir: string, variableName: string): string { + const markerPath = shellEscape(`${lockDir}/${INSTALL_LOCK_BOOT_ID_NAME}`) + const markerTempPrefix = shellEscape(`${lockDir}/${INSTALL_LOCK_BOOT_ID_NAME}.tmp`) + return [ + `if [ -n "$${variableName}" ]; then`, + `boot_marker_tmp=${markerTempPrefix}.$$;`, + `if printf '%s\\n' "$${variableName}" > "$boot_marker_tmp" 2>/dev/null; then mv "$boot_marker_tmp" ${markerPath} 2>/dev/null || rm -f "$boot_marker_tmp"; else rm -f "$boot_marker_tmp"; fi;`, + 'fi;' + ].join(' ') +} + +function windowsCurrentBootIdentityStatements(variableName: string): string[] { + return [ + `${variableName} = $null`, + 'try {', + '$operatingSystem = Get-CimInstance -ClassName Win32_OperatingSystem -ErrorAction Stop', + '$bootTicks = ([DateTime]$operatingSystem.LastBootUpTime).ToUniversalTime().Ticks', + `${variableName} = "win32:$bootTicks"`, + '} catch {}' + ] +} + +function windowsReadBootIdentityStatements(lockVariable: string, valueVariable: string): string[] { + return [ + `${valueVariable} = $null`, + 'try {', + `$bootMarker = Join-Path ${lockVariable} '${INSTALL_LOCK_BOOT_ID_NAME}'`, + '$bootMarkerItem = Get-Item -LiteralPath $bootMarker -ErrorAction Stop', + `if ($bootMarkerItem.Length -le 128) { ${valueVariable} = [System.IO.File]::ReadAllText($bootMarker).Trim() }`, + `if (${valueVariable} -notmatch '^win32:[0-9]{1,32}$') { ${valueVariable} = $null }`, + '} catch {}' + ] +} + +function windowsWriteBootIdentityStatement(lockVariable: string, valueVariable: string): string { + return [ + `if (-not [string]::IsNullOrWhiteSpace(${valueVariable})) {`, + `$bootMarkerPath = Join-Path ${lockVariable} '${INSTALL_LOCK_BOOT_ID_NAME}'`, + '$bootMarkerTemp = "$bootMarkerPath.tmp.$PID.$([Guid]::NewGuid().ToString(\'N\'))"', + 'try { [System.IO.File]::WriteAllText($bootMarkerTemp, ' + + `${valueVariable}); [System.IO.File]::Move($bootMarkerTemp, $bootMarkerPath) } catch {} finally { Remove-Item -LiteralPath $bootMarkerTemp -Force -ErrorAction SilentlyContinue }`, + '}' + ].join('; ') +} + function posixLockAgeSecondsAssignment(lockDir: string): string { return `${posixLockMtimeSecondsAssignment(lockDir, 'mtime')} && now=$(date +%s) && age=$((now - mtime))` } diff --git a/src/main/ssh/ssh-relay-install-lock.ts b/src/main/ssh/ssh-relay-install-lock.ts index 0245875e141..7d5b26fdfae 100644 --- a/src/main/ssh/ssh-relay-install-lock.ts +++ b/src/main/ssh/ssh-relay-install-lock.ts @@ -74,6 +74,7 @@ export async function acquireInstallLock( const start = Date.now() let lastStaleCheckAt = Number.NEGATIVE_INFINITY + let lastWaitLogAt = Number.NEGATIVE_INFINITY while (true) { // Why: a crashed GC can leave the stable sibling claim behind. The shared // waiter recovers stale claims instead of polling that orphan forever. @@ -121,7 +122,8 @@ export async function acquireInstallLock( ).catch(() => 'BUSY') options?.signal?.throwIfAborted() if (steal.trim().endsWith('OK')) { - console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`) + const reason = steal.trim().endsWith('REBOOT_OK') ? 'previous-boot' : 'stale' + console.warn(`[ssh-relay] Stealing ${reason} install lock at ${lockDir}`) const claimedAfterSteal = await isRelayGcClaimed( conn, remoteRelayDir, @@ -135,6 +137,10 @@ export async function acquireInstallLock( options?.signal?.throwIfAborted() } } + if (Date.now() - lastWaitLogAt >= INSTALL_LOCK_STALE_RECHECK_MS) { + lastWaitLogAt = Date.now() + console.info(`[ssh-relay] Waiting for install lock at ${lockDir}`) + } if (Date.now() - start >= INSTALL_LOCK_TIMEOUT_MS) { throw new Error( `Could not acquire relay install lock at ${lockDir} after ${ diff --git a/src/main/ssh/ssh-remote-commands.test.ts b/src/main/ssh/ssh-remote-commands.test.ts index 4e75720121b..f005496f128 100644 --- a/src/main/ssh/ssh-remote-commands.test.ts +++ b/src/main/ssh/ssh-remote-commands.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, + readFileSync, readdirSync, writeFileSync, rmSync, @@ -37,6 +38,9 @@ import { } from '../../shared/relay-artifacts' const posix = getRemoteHostPlatform('linux-x64') +const nativePosix = getRemoteHostPlatform( + process.platform === 'darwin' ? 'darwin-x64' : 'linux-x64' +) const windows = getRemoteHostPlatform('win32-x64') const powerShellExecutable = [ process.env.ORCA_POWERSHELL_EXECUTABLE, @@ -415,6 +419,43 @@ describe('ssh remote command builders', () => { expect(outputs.filter((output) => output.trim().endsWith('OK'))).toHaveLength(1) expect(statSync(lockPath).isDirectory()).toBe(true) expect(statSync(join(lockPath, '.owner')).isFile()).toBe(true) + expect(readFileSync(join(lockPath, '.boot-id'), 'utf8')).toMatch(/^win32:\d+$/u) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }, + 30_000 + ) + + it.runIf(powerShell51Executable)( + 'atomically replaces a fresh Windows lock only after the remote boot changes', + async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-windows-reboot-')) + try { + const lockPath = join(root, '.install-lock') + const acquire = decodePowerShellCommand(tryCreateInstallLockCommand(windows, lockPath)) + const recover = decodePowerShellCommand( + tryStealInstallLockCommand(windows, lockPath, 20 * 60) + ) + + await expect(runPowerShellCommand(powerShell51Executable!, acquire)).resolves.toMatch(/OK/u) + await expect(runPowerShellCommand(powerShell51Executable!, recover)).resolves.toMatch( + /^BUSY\s*$/u + ) + + writeFileSync(join(lockPath, '.boot-id'), 'partial') + await expect(runPowerShellCommand(powerShell51Executable!, recover)).resolves.toMatch( + /^BUSY\s*$/u + ) + + writeFileSync(join(lockPath, '.boot-id'), 'win32:0') + const outputs = await Promise.all( + Array.from({ length: 4 }, () => runPowerShellCommand(powerShell51Executable!, recover)) + ) + + expect(outputs.filter((output) => output.trim() === 'REBOOT_OK')).toHaveLength(1) + expect(readFileSync(join(lockPath, '.boot-id'), 'utf8')).toMatch(/^win32:\d+$/u) + expect(readdirSync(root).filter((name) => name.includes('.tombstone.'))).toHaveLength(0) } finally { rmSync(root, { recursive: true, force: true }) } @@ -531,6 +572,61 @@ describe('ssh remote command builders', () => { } ) + it.runIf(process.platform !== 'win32')( + 'atomically replaces a fresh POSIX lock only after the execution host changes', + async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-reboot-')) + try { + const lockDir = join(root, '.install-lock') + const acquire = tryCreateInstallLockCommand(nativePosix, lockDir) + const recover = tryStealInstallLockCommand(nativePosix, lockDir, 20 * 60) + + expect((await runShellCommand(acquire)).trim()).toBe('OK') + const currentBootId = readFileSync(join(lockDir, '.boot-id'), 'utf8').trim() + expect(currentBootId).toMatch(/^(?:darwin|linux):/u) + expect((await runShellCommand(recover)).trim()).toBe('BUSY') + + const previousBootId = + nativePosix.os === 'darwin' ? 'darwin:0' : 'linux:00000000-0000-0000-0000-000000000000:0' + writeFileSync(join(lockDir, '.boot-id'), previousBootId) + const outputs = await Promise.all( + Array.from({ length: 32 }, () => runShellCommand(recover)) + ) + + expect(outputs.filter((output) => output.trim() === 'REBOOT_OK')).toHaveLength(1) + expect(readFileSync(join(lockDir, '.boot-id'), 'utf8').trim()).toBe(currentBootId) + expect(readdirSync(root).some((name) => name.includes('.tombstone'))).toBe(false) + } finally { + rmSync(root, { recursive: true, force: true }) + } + } + ) + + it.runIf(process.platform !== 'win32')( + 'keeps a fresh legacy POSIX lock when no boot identity is available', + async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-legacy-')) + try { + const lockDir = join(root, '.install-lock') + mkdirSync(lockDir) + + const output = await runShellCommand( + tryStealInstallLockCommand(nativePosix, lockDir, 20 * 60) + ) + + expect(output.trim()).toBe('BUSY') + expect(existsSync(lockDir)).toBe(true) + + writeFileSync(join(lockDir, '.boot-id'), 'partial') + expect( + (await runShellCommand(tryStealInstallLockCommand(nativePosix, lockDir, 20 * 60))).trim() + ).toBe('BUSY') + } finally { + rmSync(root, { recursive: true, force: true }) + } + } + ) + it.runIf(process.platform !== 'win32')( 'lets only one POSIX caller move and recreate a stale install lock', async () => {