fix(ssh): recover orphaned relay install locks (#9828) (#10207)

* fix(ssh): recover orphaned relay install locks (#9828)

* test(ssh): split staged upload relay specs (#9828)

* fix(ssh): verify staged relay upload namespace

* fix(ssh): bound stale relay stage cleanup

* fix(ssh): complete bounded stage recovery

* fix(ssh): generate valid PowerShell stage scripts

* fix(ssh): make staged upload cancellation safe

* fix(ssh): fence staged relay recovery

* test(ssh): align deploy timeout oracle

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
Rod Boev
2026-07-31 16:17:37 -07:00
committed by GitHub
co-authored by OrcaWin
parent 139f756064
commit f56e6ade80
35 changed files with 3342 additions and 421 deletions
+151 -1
View File
@@ -1,6 +1,6 @@
{
"schemaVersion": 1,
"updatedAt": "2026-07-30",
"updatedAt": "2026-07-31",
"policy": {
"maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"],
"blockingPromotion": {
@@ -10,6 +10,156 @@
}
},
"gates": [
{
"id": "ssh-relay.staged-upload-recovery",
"title": "SSH relay uploads remain retryable before the shared install lock",
"maturity": "experimental",
"protection": "partial",
"owner": "ssh-relay-install",
"layer": "ssh-transfer-install-contract",
"surfaces": [
"SSH relay first install",
"split shell and SFTP namespaces",
"system SSH transfer fallback",
"relay install retry after cancellation"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["ssh2", "system-ssh"],
"coveredPlatforms": ["macos", "linux"],
"coveredProviders": ["ssh2", "system-ssh"],
"coverageNotes": "Deterministic unit, exact POSIX shell, native ARM macOS PowerShell 7.6.4, and real ssh2 SFTP-wire tests cover lock ordering, concurrent-install loss, fixed-slot ownership identity, payload-only promotion, bounded stale-stage reclamation, installed-fast-path draining, joined cancellation teardown, cross-version isolation, split-SFTP redirection, and system-SSH bypass. A throwaway linux-arm64 Docker sshd reached through a non-loopback LAN address covers live bytes-in-flight SFTP cancellation, injected unconfirmed cancellation, immediate retry against a real Git repository, fixed-slot recovery behind unclaimable entries, and real version-GC filtering with 15,197 unrelated names.",
"motivatingLinks": [
"https://github.com/stablyai/orca/issues/9828",
"https://github.com/stablyai/orca/pull/10207"
],
"invariant": "A first-install relay transfer must complete in an attempt-owned fixed staging slot before acquiring the shared version install lock. Reservation, promotion, confirmed cleanup, and stale recovery must reject path replacement, persisted-identity mismatch, POSIX symlinks, and Windows reparse points. Recovery examines only eight fixed slot/claim/delete names and removes at most one stale valid stage per call; eight unclaimable states fail with an explicit manual-recovery message. Split-SFTP hosts must prove the stage identity on the exact transfer session, only payload contents may be promoted under the shared lock, and cancellation must boundedly join SFTP, stream, local file-handle, and transfer settlement.",
"oracle": "Pause a real ssh2 SFTP relay.js write after one remotely acknowledged chunk, prove the remote file is partial, abort the live transfer, and require no shared .install-lock, leaked local descriptor, or foreign-process termination. Separately inject two unconfirmed cancellations, require an independent deployment to install, launch, answer relay RPC, and read a real repository HEAD. Replace one retained fixed slot with an old-mtime same-owner directory while preserving the original, add a fixed-slot POSIX symlink, and require installed-path recovery to skip both while reclaiming a valid stale slot behind them. Add 15,197 unrelated relay-shaped names and run the real version GC, requiring bounded stdout and no removal. Unit and wire contracts cover exact POSIX and native PowerShell 0/1/7/8/9+ quota behavior, no-follow identity fencing, payload symlink/reparse rejection, one-item repeated draining, zero lock acquisition before upload settlement, joined transfer/channel teardown including never-settling failures, SFTP redirection, package.json namespace ownership, promotion only after the lock, cross-version isolation, and system-SSH behavior.",
"commands": [
"node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell <PowerShell-7.6.4-executable> src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/ssh-relay-deploy-staged-upload.test.ts src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts src/main/ssh/ssh-relay-sftp-namespace-install.test.ts src/main/ssh/ssh-relay-install-namespace.test.ts src/main/ssh/ssh-relay-upload-stage-commands.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts src/main/ssh/ssh-relay-cross-version-isolation.test.ts",
"ORCA_REVIEW_SSH_UPLOAD_CANCEL=1 ORCA_REVIEW_SSH_TARGET_HOST=<non-loopback-host> ORCA_REVIEW_SSH_IMAGE=<throwaway-sshd-image> ORCA_REVIEW_EXPECT_RECOVERY=1 pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-upload-cancel.docker.test.ts --maxWorkers=1 --reporter=verbose"
],
"testFiles": [
"src/main/ssh/sftp-upload.test.ts",
"src/main/ssh/ssh-file-transfer-abort.test.ts",
"src/main/ssh/ssh-relay-deploy-staged-upload.test.ts",
"src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts",
"src/main/ssh/ssh-relay-sftp-namespace-install.test.ts",
"src/main/ssh/ssh-relay-install-namespace.test.ts",
"src/main/ssh/ssh-relay-upload-stage-commands.test.ts",
"src/main/ssh/sftp-namespace-resolution.test.ts",
"src/main/ssh/ssh-connection-sftp-wire.test.ts",
"src/main/ssh/ssh-remote-commands.test.ts",
"src/main/ssh/ssh-relay-cross-version-isolation.test.ts",
"src/main/ssh/ssh-relay-upload-cancel.docker.test.ts"
],
"assertionRefs": [
{
"file": "src/main/ssh/sftp-upload.test.ts",
"assertions": ["joins local file-descriptor teardown when a live upload is aborted"]
},
{
"file": "src/main/ssh/ssh-file-transfer-abort.test.ts",
"assertions": [
"joins confirmed SFTP close and transfer teardown before rejecting an abort",
"marks transfer teardown unconfirmed when close wins but the transfer never settles"
]
},
{
"file": "src/main/ssh/ssh-relay-upload-stage-commands.test.ts",
"assertions": [
"bounds reservation at 0, 1, 7, 8, and 9+ entries on POSIX and native PowerShell",
"rejects same-path replacement, symlink, reparse, and identity substitution before promotion or deletion",
"reclaims at most one valid stale fixed slot and progresses across repeated deployments"
]
},
{
"file": "src/main/ssh/ssh-relay-deploy-staged-upload.test.ts",
"assertions": [
"waits for a deferred SFTP upload before acquiring the install lock",
"drops only its stage when a sibling finishes before the locked re-probe",
"recovers one fixed stale stage before a fresh upload",
"launches before bounded installed-path recovery",
"never enumerates arbitrary stage paths during installation",
"retries immediately after an unconfirmed upload termination instead of waiting on a fresh install lock"
]
},
{
"file": "src/main/ssh/ssh-remote-commands.test.ts",
"assertions": [
"uses encoded PowerShell for Windows deploy commands",
"enumerates Windows staging children before copying",
"lets only one PowerShell caller acquire a legacy-visible lock"
]
},
{
"file": "src/main/ssh/ssh-relay-cross-version-isolation.test.ts",
"assertions": ["a v2 deploy never references the v1 install dir or v1 socket path"]
},
{
"file": "src/main/ssh/ssh-relay-sftp-namespace-install.test.ts",
"assertions": [
"redirects every first-install artifact transfer while shell commands stay canonical",
"leaves system-SSH connections unmapped and unprobed"
]
},
{
"file": "src/main/ssh/ssh-relay-upload-cancel.docker.test.ts",
"assertions": [
"aborts a live SFTP upload after remote bytes arrive without creating the shared lock",
"recovers cancellation with bounded safe reclamation and bounded real version GC"
]
}
],
"evidenceRuns": [
{
"date": "2026-07-31",
"runner": "local",
"platform": "macos",
"command": "node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell <PowerShell-7.6.4-executable> src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/ssh-relay-deploy-staged-upload.test.ts src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts src/main/ssh/ssh-relay-sftp-namespace-install.test.ts src/main/ssh/ssh-relay-install-namespace.test.ts src/main/ssh/ssh-relay-upload-stage-commands.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts src/main/ssh/ssh-relay-cross-version-isolation.test.ts",
"result": "passed",
"durationSeconds": 57.9,
"summary": "Eleven focused files passed 152 tests with 3 platform skips using exact POSIX sh, native ARM macOS PowerShell 7.6.4, real ssh2 split-SFTP wire sessions, fixed-slot identity races, bounded recovery, joined cancellation teardown, and cross-version isolation."
},
{
"date": "2026-07-31",
"runner": "local",
"platform": "linux",
"command": "ORCA_REVIEW_SSH_UPLOAD_CANCEL=1 ORCA_REVIEW_SSH_TARGET_HOST=<non-loopback-host> ORCA_REVIEW_SSH_IMAGE=<throwaway-sshd-image> ORCA_REVIEW_EXPECT_RECOVERY=1 pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-upload-cancel.docker.test.ts --maxWorkers=1 --reporter=verbose",
"result": "passed",
"durationSeconds": 12.58,
"summary": "A throwaway linux-arm64 Docker sshd acknowledged 65,536 of 837,401 relay.js bytes before live abort with no shared lock and preserved the foreign sleep sentinel. Two injected unconfirmed stages remained pre-lock; retry launched the relay, answered RPC, and read a real Git HEAD. Installed recovery preserved a same-owner identity-mismatched replacement, its original, a POSIX symlink, and its foreign target while draining one valid stale slot behind them. Real version GC retained all 15,197 unrelated names, emitted 25 bytes, and completed in 25 ms."
}
],
"runtimeBudget": {
"p95Seconds": 35,
"scope": "focused unit, SFTP-wire, and local Docker SSH contracts"
},
"flakeHistory": {
"status": "unknown",
"evidence": "Focused deterministic and Docker runs pass locally; CI and soak history are not yet available."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "A byte-identical local Docker oracle run on latest main left a fresh shared .install-lock and blocked retry; the staged candidate left no shared lock and recovered immediately, and disabling staged ordering restored the blocked result. No committed baseline artifact or baseline installed-fast-path cleanup oracle is retained, so this evidence is intentionally not marked complete."
},
"performanceBudget": {
"required": true,
"evidence": "Stage recovery examines only eight fixed slot/claim/delete paths and reclaims at most one stale valid stage per invocation; installed reconnects launch before asynchronous recovery. Full quota produces an explicit error instead of unbounded cleanup. Version GC still scans the relay base directory, but remote filtering caps stdout and local candidate work at 64. Cancellation adds one bounded five-second join of channel and transfer settlement."
},
"promotionCriteria": [
"Collect 100 consecutive CI passes or 14 days of soak history.",
"Run live first-install cancellation and retry on Windows OpenSSH and a split-SFTP Synology-class host.",
"Keep exact lock-order, no-follow identity, promotion, bounded reclamation, and teardown assertions in the gate command."
],
"knownGaps": [
"The live Docker target is Linux ARM64 with a unified namespace; split-SFTP behavior is covered by real ssh2 wire and deterministic deploy fixtures.",
"Native PowerShell coverage runs on ARM macOS with POSIX filesystem paths; Windows OpenSSH, Windows PowerShell 5.1, and system-SSH behavior remain command and transfer-contract coverage rather than a live target.",
"The fixed pool retains up to eight relay bundles; eight foreign or otherwise unclaimable fixed states require manual inspection instead of automatic deletion.",
"Version GC remotely filters and caps output but still scans the base .orca-remote directory; it does not promise constant remote enumeration time.",
"The Docker oracle is opt-in because it requires a local image and a reachable non-loopback host address."
],
"demotionRule": "Demote or quarantine if cancellation creates the shared install lock before transfer settlement, a split-SFTP transfer loses identity proof, recovery deletes a replacement/symlink/reparse/foreign stage, fixed-path work exceeds its eight-slot bound, cancellation leaks a local descriptor, or the focused gate flakes without a product or harness bug."
},
{
"id": "mobile-ui.drawer-close-continuity",
"title": "Mobile drawers finish closing despite parent rerenders",
@@ -0,0 +1,81 @@
import { spawnSync } from 'node:child_process'
const defaultFiles = [
'src/main/ssh/sftp-upload.test.ts',
'src/main/ssh/ssh-file-transfer-abort.test.ts',
'src/main/ssh/ssh-relay-deploy-staged-upload.test.ts',
'src/main/ssh/ssh-relay-native-deps-install-staged-upload.test.ts',
'src/main/ssh/ssh-relay-sftp-namespace-install.test.ts',
'src/main/ssh/ssh-relay-install-namespace.test.ts',
'src/main/ssh/ssh-relay-upload-stage-commands.test.ts',
'src/main/ssh/sftp-namespace-resolution.test.ts',
'src/main/ssh/ssh-connection-sftp-wire.test.ts',
'src/main/ssh/ssh-remote-commands.test.ts',
'src/main/ssh/ssh-relay-cross-version-isolation.test.ts'
]
const cliArguments = process.argv.slice(2)
const powerShellFlag = cliArguments.indexOf('--powershell')
const configuredPowerShell =
powerShellFlag >= 0 ? cliArguments[powerShellFlag + 1] : process.env.ORCA_POWERSHELL_EXECUTABLE
if (powerShellFlag >= 0 && !configuredPowerShell) {
console.error('--powershell requires an executable path')
process.exit(2)
}
const powerShellExecutable = [
configuredPowerShell,
...(process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh'])
].find((candidate) => {
if (!candidate) {
return false
}
return (
spawnSync(
candidate,
['-NoProfile', '-NonInteractive', '-Command', '$PSVersionTable.PSVersion.ToString()'],
{ encoding: 'utf8' }
).status === 0
)
})
if (!powerShellExecutable) {
console.error('A native PowerShell executable is required')
process.exit(2)
}
const powerShellVersion = spawnSync(
powerShellExecutable,
['-NoProfile', '-NonInteractive', '-Command', '$PSVersionTable.PSVersion.ToString()'],
{ encoding: 'utf8' }
).stdout.trim()
console.log(`SSH staged-upload reliability: PowerShell ${powerShellVersion}`)
const requestedFiles = cliArguments.filter(
(_argument, index) => index !== powerShellFlag && index !== powerShellFlag + 1
)
const files = requestedFiles.length > 0 ? requestedFiles : defaultFiles
const result = spawnSync(
process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm',
[
'exec',
'vitest',
'run',
'--config',
'config/vitest.config.ts',
...files,
'--maxWorkers=1',
'--reporter=dot'
],
{
cwd: process.cwd(),
env: {
...process.env,
ORCA_POWERSHELL_EXECUTABLE: powerShellExecutable
},
stdio: 'inherit'
}
)
if (result.error) {
console.error(result.error.message)
process.exit(1)
}
process.exit(result.status ?? 1)
@@ -15,6 +15,7 @@ const RELAY_DIR = '.orca-remote/relay-0.1.0+hash'
const MARKER = '.install-lock/.sftp-namespace-deadbeef'
const mapping: SftpNamespacePathMapping = {
homeRelativeNamespaceRoot: RELAY_DIR,
homeRelativePath: RELAY_DIR,
shellProbePath: `${SHELL_HOME}/${RELAY_DIR}/${MARKER}`,
homeRelativeProbePath: `${RELAY_DIR}/${MARKER}`
@@ -249,12 +250,12 @@ describe('resolveSftpTransferPath', () => {
'marker paths must share one marker basename'
],
[
'a marker outside the install lock',
'a marker at the namespace root',
{
shellProbePath: `${SHELL_HOME}/${RELAY_DIR}/other-lock/.sftp-namespace-deadbeef`,
homeRelativeProbePath: `${RELAY_DIR}/other-lock/.sftp-namespace-deadbeef`
shellProbePath: `${SHELL_HOME}/${RELAY_DIR}`,
homeRelativeProbePath: RELAY_DIR
},
'inside the transfer relay install lock'
'inside one namespace root'
],
[
'a marker under another relay tree',
@@ -262,7 +263,7 @@ describe('resolveSftpTransferPath', () => {
shellProbePath: `${SHELL_HOME}/other/.install-lock/.sftp-namespace-deadbeef`,
homeRelativeProbePath: 'other/.install-lock/.sftp-namespace-deadbeef'
},
'inside the transfer relay install lock'
'inside one namespace root'
]
])('rejects %s before issuing any SFTP request', async (_label, overrides, expected) => {
const { shell, ...mappingOverrides } = overrides as Record<string, string>
@@ -282,6 +283,7 @@ describe('resolveSftpTransferPath', () => {
const token = 'a'.repeat(32)
const secretMarker = `.install-lock/.sftp-namespace-${token}`
const secretMapping: SftpNamespacePathMapping = {
homeRelativeNamespaceRoot: RELAY_DIR,
homeRelativePath: RELAY_DIR,
shellProbePath: `${SHELL_HOME}/${RELAY_DIR}/${secretMarker}`,
homeRelativeProbePath: `${RELAY_DIR}/${secretMarker}`
+7 -6
View File
@@ -10,6 +10,7 @@ import type { RemoteHostPlatform } from './ssh-remote-platform'
import { redactRelayInstallMarkerTokens } from './ssh-relay-install-marker'
export type SftpNamespacePathMapping = {
homeRelativeNamespaceRoot: string
homeRelativePath: string
shellProbePath: string
homeRelativeProbePath: string
@@ -72,14 +73,13 @@ function assertMappingIdentity(shellAbsolutePath: string, mapping: SftpNamespace
throw new Error('SFTP namespace transfer and marker must share one shell namespace prefix')
}
const lockSegmentIndex = probeSegments.length - 2
const relayDir = probeSegments.slice(0, lockSegmentIndex).join('/')
const namespaceRoot = mapping.homeRelativeNamespaceRoot
if (
lockSegmentIndex < 1 ||
probeSegments[lockSegmentIndex] !== '.install-lock' ||
(mapping.homeRelativePath !== relayDir && !mapping.homeRelativePath.startsWith(`${relayDir}/`))
(mapping.homeRelativePath !== namespaceRoot &&
!mapping.homeRelativePath.startsWith(`${namespaceRoot}/`)) ||
!mapping.homeRelativeProbePath.startsWith(`${namespaceRoot}/`)
) {
throw new Error('SFTP namespace marker must be inside the transfer relay install lock')
throw new Error('SFTP namespace transfer and marker must be inside one namespace root')
}
}
@@ -159,6 +159,7 @@ export async function resolveSftpTransferPath(
): Promise<string> {
assertAbsolutePosixPath('transfer path', shellAbsolutePath)
assertAbsolutePosixPath('marker path', mapping.shellProbePath)
assertHomeRelativePosixPath('relative namespace root', mapping.homeRelativeNamespaceRoot)
assertHomeRelativePosixPath('relative transfer path', mapping.homeRelativePath)
assertHomeRelativePosixPath('relative marker path', mapping.homeRelativeProbePath)
assertMappingIdentity(shellAbsolutePath, mapping)
+58 -1
View File
@@ -1,4 +1,5 @@
import { mkdtemp, mkdir, realpath, symlink, writeFile } from 'node:fs/promises'
import { spawnSync } from 'node:child_process'
import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Writable } from 'node:stream'
@@ -114,6 +115,62 @@ describe('sftp-upload', () => {
expect(sftp.createWriteStream).not.toHaveBeenCalled()
})
it('joins local file-descriptor teardown when a live upload is aborted', async () => {
const localDir = await mkdtemp(join(tmpdir(), 'orca-sftp-upload-abort-'))
const localPath = join(localDir, 'relay.js')
const controller = new AbortController()
const blockedWrite = new Writable({
write() {}
})
const sftp = createSftpMock()
vi.mocked(sftp.createWriteStream).mockReturnValue(blockedWrite as never)
try {
await writeFile(localPath, Buffer.alloc(1024 * 1024, 7))
const upload = uploadFile(sftp, localPath, '/remote/relay.js', {
signal: controller.signal
})
await vi.waitFor(() => expect(sftp.createWriteStream).toHaveBeenCalledTimes(1))
controller.abort()
await expect(upload).rejects.toMatchObject({ name: 'AbortError' })
if (process.platform !== 'win32') {
const descriptorProbe = spawnSync(
'lsof',
['-a', '-p', String(process.pid), '--', localPath],
{ encoding: 'utf8' }
)
if (!descriptorProbe.error) {
expect(descriptorProbe.stdout).not.toContain(localPath)
}
}
} finally {
await rm(localDir, { recursive: true, force: true })
}
})
it('joins the local read when the remote write fails', async () => {
const localDir = await mkdtemp(join(tmpdir(), 'orca-sftp-upload-failure-'))
const localPath = join(localDir, 'relay.js')
const sftp = createSftpMock()
vi.mocked(sftp.createWriteStream).mockReturnValue(
new Writable({
write(_chunk, _encoding, callback) {
callback(new Error('remote write failed'))
}
}) as never
)
try {
await writeFile(localPath, Buffer.alloc(1024 * 1024, 7))
await expect(uploadFile(sftp, localPath, '/remote/relay.js')).rejects.toThrow(
'remote write failed'
)
} finally {
await rm(localDir, { recursive: true, force: true })
}
})
it('removes remote directory contents before removing the directory', async () => {
const sftp = createSftpMock()
vi.mocked(sftp.readdir).mockImplementation((remotePath, cb) => {
+74 -57
View File
@@ -2,6 +2,7 @@ import { constants } from 'node:fs'
import type { ReadStream } from 'node:fs'
import { lstat, open, readdir, realpath } from 'node:fs/promises'
import { isAbsolute, join as pathJoin, relative, sep } from 'node:path'
import { finished } from 'node:stream/promises'
import type { SFTPWrapper } from 'ssh2'
export function mkdirSftp(
@@ -28,67 +29,81 @@ export function uploadFile(
sftp: SFTPWrapper,
localPath: string,
remotePath: string,
options?: { exclusive?: boolean }
options?: { exclusive?: boolean; signal?: AbortSignal }
): Promise<void> {
return new Promise((resolve, reject) => {
let settled = false
let readStream: ReadStream | null = null
let fileHandle: Awaited<ReturnType<typeof open>> | null = null
let writeStream: ReturnType<SFTPWrapper['createWriteStream']> | null = null
return uploadFileAndJoinTeardown(sftp, localPath, remotePath, options)
}
const cleanupListeners = (): void => {
writeStream?.off('close', onWriteClose)
writeStream?.off('error', onWriteError)
readStream?.off('error', onReadError)
async function uploadFileAndJoinTeardown(
sftp: SFTPWrapper,
localPath: string,
remotePath: string,
options?: { exclusive?: boolean; signal?: AbortSignal }
): Promise<void> {
const handle = await open(localPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0))
let handleClose: Promise<void> | undefined
let readStream: ReadStream | undefined
let writeStream: ReturnType<SFTPWrapper['createWriteStream']> | undefined
const closeHandle = (): Promise<void> => {
handleClose ??= handle.close()
return handleClose
}
try {
options?.signal?.throwIfAborted()
const statResult = await lstat(localPath)
if (statResult.isSymbolicLink() || !statResult.isFile()) {
throw new Error(`Unsupported upload source: ${localPath}`)
}
const settle = (fn: typeof resolve | typeof reject, val?: unknown): void => {
if (settled) {
return
}
settled = true
cleanupListeners()
readStream?.destroy()
const openedStat = await handle.stat()
if (
!openedStat.isFile() ||
openedStat.size !== statResult.size ||
(statResult.ino !== 0 && openedStat.ino !== 0 && openedStat.ino !== statResult.ino) ||
(statResult.dev !== 0 && openedStat.dev !== 0 && openedStat.dev !== statResult.dev)
) {
throw new Error(`File changed during upload: ${localPath}`)
}
// Why: rejected local sources must not leave an empty remote file.
writeStream = sftp.createWriteStream(remotePath, {
flags: options?.exclusive ? 'wx' : 'w'
})
readStream = handle.createReadStream({ autoClose: false })
const abortTransfer = (): void => {
const reason =
options?.signal?.reason instanceof Error
? options.signal.reason
: Object.assign(new Error('Upload aborted'), { name: 'AbortError' })
readStream?.destroy(reason)
writeStream?.destroy()
void fileHandle?.close().catch(() => {})
fn(val as never)
void closeHandle().catch(() => {})
}
const onWriteClose = (): void => settle(resolve)
const onWriteError = (err: Error): void => settle(reject, err)
const onReadError = (err: Error): void => settle(reject, err)
void open(localPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0))
.then(async (handle) => {
if (settled) {
void handle.close().catch(() => {})
return
}
fileHandle = handle
const statResult = await lstat(localPath)
if (statResult.isSymbolicLink() || !statResult.isFile()) {
throw new Error(`Unsupported upload source: ${localPath}`)
}
const openedStat = await handle.stat()
if (
!openedStat.isFile() ||
openedStat.size !== statResult.size ||
(statResult.ino !== 0 && openedStat.ino !== 0 && openedStat.ino !== statResult.ino) ||
(statResult.dev !== 0 && openedStat.dev !== 0 && openedStat.dev !== statResult.dev)
) {
throw new Error(`File changed during upload: ${localPath}`)
}
// Why: validate the local source before creating the remote write
// target, so rejected sources do not leave empty files behind.
writeStream = sftp.createWriteStream(remotePath, {
flags: options?.exclusive ? 'wx' : 'w'
})
writeStream.on('close', onWriteClose)
writeStream.on('error', onWriteError)
readStream = handle.createReadStream()
readStream.on('error', onReadError)
readStream.pipe(writeStream)
options?.signal?.addEventListener('abort', abortTransfer, { once: true })
if (options?.signal?.aborted) {
abortTransfer()
}
try {
const readDone = finished(readStream, { cleanup: true }).catch((error: unknown) => {
writeStream?.destroy()
throw error
})
.catch((err: unknown) => settle(reject, err))
})
const writeDone = finished(writeStream, { cleanup: true }).catch((error: unknown) => {
readStream?.destroy(error instanceof Error ? error : undefined)
throw error
})
readStream.pipe(writeStream)
const results = await Promise.allSettled([readDone, writeDone])
const failure = results.find((result) => result.status === 'rejected')
if (failure?.status === 'rejected') {
throw failure.reason
}
} finally {
options?.signal?.removeEventListener('abort', abortTransfer)
}
} finally {
readStream?.destroy()
writeStream?.destroy()
await closeHandle()
}
}
export function uploadBuffer(
@@ -167,11 +182,13 @@ export async function uploadDirectory(
localDir: string,
remoteDir: string,
rootRealPath = localDir,
options?: { exclusive?: boolean }
options?: { exclusive?: boolean; signal?: AbortSignal }
): Promise<void> {
options?.signal?.throwIfAborted()
await assertLocalUploadPathInsideRoot(rootRealPath, localDir)
const entries = await readdir(localDir, { withFileTypes: true })
for (const entry of entries) {
options?.signal?.throwIfAborted()
const localPath = pathJoin(localDir, entry.name)
const remotePath = `${remoteDir}/${entry.name}`
await assertLocalUploadPathInsideRoot(rootRealPath, localPath)
@@ -189,7 +206,7 @@ export async function uploadDirectory(
await mkdirSftp(sftp, remotePath, { allowExisting: !options?.exclusive })
await uploadDirectory(sftp, localPath, remotePath, rootRealPath, options)
} else {
await uploadFile(sftp, localPath, remotePath, { exclusive: options?.exclusive })
await uploadFile(sftp, localPath, remotePath, options)
}
}
}
@@ -43,6 +43,7 @@ const MARKER = '.install-lock/.sftp-namespace-cafebabe'
const SHELL_RELAY_DIR = `${SHELL_HOME}/${RELAY_DIR}`
const namespace: SftpNamespacePathMapping = {
homeRelativeNamespaceRoot: RELAY_DIR,
homeRelativePath: RELAY_DIR,
shellProbePath: `${SHELL_RELAY_DIR}/${MARKER}`,
homeRelativeProbePath: `${RELAY_DIR}/${MARKER}`
@@ -267,6 +267,7 @@ async function boundedTransfer(
function splitNamespaceMapping(homeRelativePath: string): SftpNamespacePathMapping {
return {
homeRelativeNamespaceRoot: RELAY_DIR,
homeRelativePath,
shellProbePath: `${SHELL_RELAY_DIR}/${MARKER_PATH}`,
homeRelativeProbePath: `${RELAY_DIR}/${MARKER_PATH}`
+5 -1
View File
@@ -440,11 +440,14 @@ export class SshConnection {
const targetDir = await resolveSftpTransferPathIfMapped(sftp, remoteDir, options)
linkedSignal.signal.throwIfAborted()
const { uploadDirectory } = await import('./ssh-relay-deploy-helpers')
await uploadDirectory(sftp, localDir, targetDir)
await uploadDirectory(sftp, localDir, targetDir, localDir, {
signal: linkedSignal.signal
})
})()
await raceSftpFileTransferWithAbort(transfer, linkedSignal.signal, (onClose) => {
sftp.once('close', onClose)
endSftp()
return () => sftp.removeListener('close', onClose)
})
} finally {
endSftp()
@@ -543,6 +546,7 @@ export class SshConnection {
await raceSftpFileTransferWithAbort(write, linkedSignal.signal, (onClose) => {
sftp.once('close', onClose)
endSftp()
return () => sftp.removeListener('close', onClose)
})
} finally {
endSftp()
+66 -4
View File
@@ -2,11 +2,14 @@ import { describe, expect, it, vi } from 'vitest'
import { raceSftpFileTransferWithAbort } from './ssh-file-transfer-abort'
describe('raceSftpFileTransferWithAbort', () => {
it('waits for confirmed SFTP close before rejecting an abort', async () => {
it('joins confirmed SFTP close and transfer teardown before rejecting an abort', async () => {
const controller = new AbortController()
let confirmClose: () => void = () => {}
let settleTransfer: () => void = () => {}
const promise = raceSftpFileTransferWithAbort(
new Promise<void>(() => {}),
new Promise<void>((resolve) => {
settleTransfer = resolve
}),
controller.signal,
(onClose) => {
confirmClose = onClose
@@ -24,9 +27,20 @@ describe('raceSftpFileTransferWithAbort', () => {
expect(pending).toBe('pending')
confirmClose()
const stillPending = await Promise.race([
promise.then(
() => 'settled',
() => 'settled'
),
Promise.resolve('pending')
])
expect(stillPending).toBe('pending')
settleTransfer()
await expect(promise).rejects.toMatchObject({
name: 'AbortError',
sshChannelCloseConfirmed: true
sshChannelCloseConfirmed: true,
sshTransferTeardownConfirmed: true
})
})
@@ -46,10 +60,58 @@ describe('raceSftpFileTransferWithAbort', () => {
await expect(outcome).resolves.toMatchObject({
name: 'AbortError',
sshChannelCloseConfirmed: false
sshChannelCloseConfirmed: false,
sshTransferTeardownConfirmed: false
})
} finally {
vi.useRealTimers()
}
})
it('marks transfer teardown unconfirmed when close wins but the transfer never settles', async () => {
vi.useFakeTimers()
try {
const controller = new AbortController()
const promise = raceSftpFileTransferWithAbort(
new Promise<void>(() => {}),
controller.signal,
(onClose) => onClose()
)
const outcome = promise.catch((error: Error) => error)
controller.abort()
await vi.advanceTimersByTimeAsync(5_000)
await expect(outcome).resolves.toMatchObject({
name: 'AbortError',
sshChannelCloseConfirmed: true,
sshTransferTeardownConfirmed: false
})
} finally {
vi.useRealTimers()
}
})
it('removes the close waiter when the teardown deadline expires', async () => {
vi.useFakeTimers()
try {
const controller = new AbortController()
const removeCloseListener = vi.fn()
const outcome = raceSftpFileTransferWithAbort(
new Promise<void>(() => {}),
controller.signal,
() => removeCloseListener
).catch((error: Error) => error)
controller.abort()
await vi.advanceTimersByTimeAsync(5_000)
await expect(outcome).resolves.toMatchObject({
sshTransferTeardownConfirmed: false
})
expect(removeCloseListener).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
})
})
+29 -5
View File
@@ -27,12 +27,20 @@ export function createLinkedSshFileTransferSignal(signals: readonly AbortSignal[
export function raceSftpFileTransferWithAbort<T>(
operation: Promise<T>,
signal: AbortSignal,
closeSftp: (onClose: () => void) => void
closeSftp: (onClose: () => void) => (() => void) | void
): Promise<T> {
return new Promise((resolve, reject) => {
let settled = false
let abortError: (Error & { sshChannelCloseConfirmed: boolean }) | null = null
let operationSettled = false
let sftpClosed = false
let abortError:
| (Error & {
sshChannelCloseConfirmed: boolean
sshTransferTeardownConfirmed: boolean
})
| null = null
let closeGraceTimer: ReturnType<typeof setTimeout> | null = null
let removeCloseListener: (() => void) | undefined
const settle = (fn: typeof resolve | typeof reject, value: T | Error): void => {
if (settled) {
return
@@ -42,30 +50,46 @@ export function raceSftpFileTransferWithAbort<T>(
clearTimeout(closeGraceTimer)
}
signal.removeEventListener('abort', onAbort)
removeCloseListener?.()
removeCloseListener = undefined
fn(value as never)
}
const onAbort = (): void => {
// Why: rejecting the caller is insufficient; ending SFTP stops the
// abandoned transfer from mutating a successor relay install.
abortError = Object.assign(createSshOperationAbortError(), {
sshChannelCloseConfirmed: false
sshChannelCloseConfirmed: false,
sshTransferTeardownConfirmed: false
})
closeGraceTimer = setTimeout(() => settle(reject, abortError!), 5_000)
closeSftp(() => {
const unregisterClose = closeSftp(() => {
sftpClosed = true
abortError!.sshChannelCloseConfirmed = true
settle(reject, abortError!)
if (operationSettled) {
abortError!.sshTransferTeardownConfirmed = true
settle(reject, abortError!)
}
})
removeCloseListener = typeof unregisterClose === 'function' ? unregisterClose : undefined
}
signal.addEventListener('abort', onAbort, { once: true })
void operation.then(
(value) => {
operationSettled = true
if (!abortError) {
settle(resolve, value)
} else if (sftpClosed) {
abortError.sshTransferTeardownConfirmed = true
settle(reject, abortError)
}
},
(error: unknown) => {
operationSettled = true
if (!abortError) {
settle(reject, error instanceof Error ? error : new Error(String(error)))
} else if (sftpClosed) {
abortError.sshTransferTeardownConfirmed = true
settle(reject, abortError)
}
}
)
@@ -9,6 +9,7 @@
import { EventEmitter } from 'node:events'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as RelayInstallMarkerModule from './ssh-relay-install-marker'
vi.mock('electron', () => ({
app: { getAppPath: () => '/mock/app' }
@@ -16,7 +17,7 @@ vi.mock('electron', () => ({
vi.mock('fs', () => ({
existsSync: vi.fn().mockReturnValue(true),
readFileSync: vi.fn().mockReturnValue('0.1.0+v2hash')
readFileSync: vi.fn().mockReturnValue('0.1.0+222222222222')
}))
vi.mock('./relay-protocol', () => ({
@@ -44,6 +45,11 @@ vi.mock('./ssh-remote-node-resolution', () => ({
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
}))
vi.mock('./ssh-relay-install-marker', async (importOriginal) => ({
...(await importOriginal<typeof RelayInstallMarkerModule>()),
createRelayInstallMarkerFileName: () => '.sftp-namespace-00000000000000000000000000000000'
}))
vi.mock('./ssh-connection-utils', () => ({
shellEscape: (s: string) => `'${s}'`
}))
@@ -98,52 +104,75 @@ describe('cross-version isolation', () => {
const mockExec = vi.mocked(execCommand)
// Simulated remote where:
// v1 dir = ~/.orca-remote/relay-0.1.0+v1hash/ (live daemon, listening)
// v2 dir = ~/.orca-remote/relay-0.1.0+v2hash/ (does not yet exist)
// The v2 client has fullVersion='0.1.0+v2hash' (from the fs mock above).
// v1 dir = ~/.orca-remote/relay-0.1.0+111111111111/ (live daemon, listening)
// v2 dir = ~/.orca-remote/relay-0.1.0+222222222222/ (does not yet exist)
// The v2 client has fullVersion='0.1.0+222222222222' (from the fs mock above).
//
// We feed enough exec results to walk through the deploy: platform,
// $HOME, isRelayAlreadyInstalled probe, lock acquire, upload (no exec),
// npm install, finalize, socket probe, credential publication, socket poll, then GC scan.
const responses: string[] = [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // tagged POSIX platform probe
'/home/u', // echo $HOME
'MISSING', // isRelayAlreadyInstalled (v2 dir doesn't exist)
'OPEN', // no sibling GC claim
'', // mkdir -p remoteRelayDir (v2)
'OK', // mkdir lock OK
'OPEN', // GC did not claim while the install lock was acquired
'MISSING', // re-probe after lock → still missing → proceed with install
'', // mkdir remoteDir (uploadRelay)
'', // chmod +x node
'', // npm install
'', // chmod prebuilds
'ORCA-NPTY-PROBE-OK\n', // node -e require() load-test (post-install verify)
'', // rm -f probe-stderr (best-effort cleanup after probe resolved)
'', // touch .install-complete (finalizeInstall)
'DEAD', // launch socket probe
'', // publish the per-launch credential
'READY', // socket poll
'', // release .install-lock after relay liveness is observable
// GC scan begins here
'relay-0.1.0+v1hash\nrelay-0.1.0+v2hash\n', // ls listing
'OPEN', // v1 lock probe (siblings only — current dir is v2)
'COMPLETE', // v1 .install-complete probe
'ALIVE' // v1 socket probe → live → SKIP (don't GC v1)
]
for (const r of responses) {
mockExec.mockResolvedValueOnce(r)
}
mockExec.mockImplementation((_conn, command) => {
if (command.includes('__ORCA_UPLOAD_STAGE_SLOT__')) {
return Promise.resolve(
'__ORCA_UPLOAD_STAGE_SLOT__.sftp-namespace-00000000000000000000000000000000:slot-0'
)
}
if (command.includes('__ORCA_UPLOAD_STAGE_PROMOTION__')) {
return Promise.resolve(
'__ORCA_UPLOAD_STAGE_PROMOTION__.sftp-namespace-00000000000000000000000000000000:PROMOTED'
)
}
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/u')
}
if (command.includes("-name 'relay-0.1.0+222222222222.upload-*'")) {
return Promise.resolve('')
}
if (command.includes('relay-watcher.js') && command.includes('.install-complete')) {
return Promise.resolve('MISSING')
}
if (command.includes('.gc-claim') && command.includes('echo LOCKED || echo OPEN')) {
return Promise.resolve('OPEN')
}
if (command.includes('.install-lock') && command.includes('&& echo OK || echo BUSY')) {
return Promise.resolve('OK')
}
if (command.includes('ORCA-NPTY-PROBE-OK')) {
return Promise.resolve('ORCA-NPTY-PROBE-OK\n')
}
if (command.includes('process.stdout.write("READY")')) {
return Promise.resolve('READY')
}
if (command.includes('test -S') && command.includes('echo ALIVE || echo DEAD')) {
return Promise.resolve('DEAD')
}
if (command.includes('__ORCA_RELAY_GC_FIND_STATUS__')) {
return Promise.resolve('relay-0.1.0+111111111111\nrelay-0.1.0+222222222222\n')
}
if (command.includes('relay-0.1.0+111111111111/.install-lock')) {
return Promise.resolve('OPEN')
}
if (command.includes('relay-0.1.0+111111111111/.install-complete')) {
return Promise.resolve('COMPLETE')
}
if (command.includes('relay-0.1.0+111111111111') && command.includes('relay-*.sock')) {
return Promise.resolve('ALIVE')
}
return Promise.resolve('')
})
await deployAndLaunchRelay(conn)
await vi.waitFor(() =>
expect(mockExec.mock.calls.some(([, command]) => command.includes('relay-*.sock'))).toBe(true)
)
const allCmds = [
...mockExec.mock.calls.map(([, c]) => c),
...vi.mocked(conn.exec).mock.calls.map(([c]) => c as string)
]
// (a) the v2 deploy creates dirs/files under relay-0.1.0+v2hash
expect(allCmds.some((c) => c.includes('relay-0.1.0+v2hash'))).toBe(true)
// (a) the v2 deploy creates dirs/files under its content-hashed directory
expect(allCmds.some((c) => c.includes('relay-0.1.0+222222222222'))).toBe(true)
// (b) the v2 launch and connect socket paths are rooted in v2 dir, never v1
const launchAndConnectCmds = vi
@@ -152,13 +181,13 @@ describe('cross-version isolation', () => {
.filter((c) => c.includes('--sock-path'))
expect(launchAndConnectCmds.length).toBeGreaterThan(0)
for (const cmd of launchAndConnectCmds) {
expect(cmd).toContain('relay-0.1.0+v2hash')
expect(cmd).not.toContain('relay-0.1.0+v1hash')
expect(cmd).toContain('relay-0.1.0+222222222222')
expect(cmd).not.toContain('relay-0.1.0+111111111111')
}
// (c) GC observes v1 has a live socket and never issues an rm -rf for it
const v1RemoveCmds = allCmds.filter(
(c) => c.includes('rm -rf') && c.includes('relay-0.1.0+v1hash')
(c) => c.includes('rm -rf') && c.includes('relay-0.1.0+111111111111')
)
expect(v1RemoveCmds).toHaveLength(0)
@@ -167,11 +196,12 @@ describe('cross-version isolation', () => {
// — never a write, mkdir, chmod, touch, rm, node launch, or socket poll.
// This prevents a future refactor that accidentally writes to the v1 dir
// (e.g. shared install-complete, upload over symlink) from passing.
const v1Refs = allCmds.filter((c) => c.includes('relay-0.1.0+v1hash'))
const v1Refs = allCmds.filter((c) => c.includes('relay-0.1.0+111111111111'))
for (const cmd of v1Refs) {
const isReadOnlyProbe =
/^\s*ls\b/.test(cmd) ||
/\btest -d\b/.test(cmd) ||
/\btest -e\b/.test(cmd) ||
/\btest -f\b/.test(cmd) ||
/\btest -S\b/.test(cmd) ||
/\bfor f in .*\.sock\b/.test(cmd)
@@ -0,0 +1,566 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: { getAppPath: () => '/mock/app' }
}))
vi.mock('fs', () => ({
existsSync: vi.fn().mockReturnValue(true),
readFileSync: vi.fn().mockReturnValue('0.1.0+abcdef012345')
}))
vi.mock('./relay-protocol', () => ({
RELAY_VERSION: '0.1.0',
RELAY_REMOTE_DIR: '.orca-remote',
parseUnameToRelayPlatform: vi.fn((os: string, arch: string) => {
const normalizedOs = os.toLowerCase()
const normalizedArch = arch.toLowerCase()
const relayArch = normalizedArch === 'arm64' || normalizedArch === 'aarch64' ? 'arm64' : 'x64'
if (normalizedOs === 'windows' || normalizedOs === 'win32') {
return `win32-${relayArch}`
}
if (normalizedOs === 'darwin') {
return `darwin-${relayArch}`
}
if (normalizedOs === 'linux') {
return `linux-${relayArch}`
}
return null
}),
RELAY_SENTINEL: 'ORCA-RELAY v0.1.0 READY\n',
RELAY_SENTINEL_TIMEOUT_MS: 10_000
}))
vi.mock('./ssh-relay-deploy-helpers', () => ({
uploadDirectory: vi.fn().mockResolvedValue(undefined),
waitForSentinel: vi.fn().mockResolvedValue({
write: vi.fn(),
onData: vi.fn(),
onClose: vi.fn()
}),
isUnconfirmedSshCommandTermination: (error: unknown) =>
error instanceof Error &&
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false,
execCommand: vi.fn().mockResolvedValue('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}))
vi.mock('./ssh-remote-node-resolution', () => ({
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
}))
vi.mock('./ssh-relay-versioned-install', () => ({
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+abcdef012345'),
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
isRelayAlreadyInstalled: vi.fn().mockResolvedValue(true),
finalizeInstall: vi.fn().mockResolvedValue(undefined),
abandonInstall: vi.fn().mockResolvedValue(undefined),
gcOldRelayVersions: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./ssh-relay-install-lock', () => ({
acquireInstallLock: vi.fn().mockResolvedValue(undefined),
RELAY_INSTALL_LOCK_NAME: '.install-lock'
}))
vi.mock('./ssh-relay-repair-lock', () => ({
tryAcquireRelayRepairLock: vi.fn().mockResolvedValue('acquired')
}))
vi.mock('./ssh-connection-utils', () => ({
shellEscape: (s: string) => `'${s}'`,
createSshOperationAbortError: () =>
Object.assign(new Error('SSH operation was cancelled'), {
name: 'AbortError'
})
}))
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { execCommand, waitForSentinel } from './ssh-relay-deploy-helpers'
import { resolveRemoteNodePath } from './ssh-remote-node-resolution'
import { isRelayAlreadyInstalled } from './ssh-relay-versioned-install'
import { acquireInstallLock } from './ssh-relay-install-lock'
import {
RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS,
RELAY_DEPLOY_TIMEOUT_MS
} from './ssh-relay-deploy-timing'
import type { SshConnection } from './ssh-connection'
function makeMockConnection(): SshConnection {
return {
canRunConcurrentExecCommands: vi.fn().mockReturnValue(true),
exec: vi.fn().mockResolvedValue({
on: vi.fn(),
stderr: { on: vi.fn() },
stdin: {},
stdout: { on: vi.fn() },
close: vi.fn()
}),
sftp: vi.fn().mockResolvedValue({
mkdir: vi.fn((_p: string, cb: (err: Error | null) => void) => cb(null)),
createWriteStream: vi.fn().mockReturnValue({
on: vi.fn((_event: string, cb: () => void) => {
if (_event === 'close') {
setTimeout(cb, 0)
}
}),
end: vi.fn()
}),
end: vi.fn()
})
} as unknown as SshConnection
}
function stageCommandResponse(command: string): string | undefined {
const marker = command.match(/\.sftp-namespace-[0-9a-f]{32}/u)?.[0]
if (command.includes('__ORCA_UPLOAD_STAGE_SLOT__') && marker) {
return `__ORCA_UPLOAD_STAGE_SLOT__${marker}:slot-0`
}
if (command.includes('__ORCA_UPLOAD_STAGE_PROMOTION__') && marker) {
return `__ORCA_UPLOAD_STAGE_PROMOTION__${marker}:PROMOTED`
}
return command.includes('.upload-stages') ? '' : undefined
}
describe('deployAndLaunchRelay staged uploads', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(execCommand).mockReset().mockResolvedValue('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
vi.mocked(waitForSentinel).mockReset().mockResolvedValue({
write: vi.fn(),
onData: vi.fn(),
onClose: vi.fn()
})
vi.mocked(resolveRemoteNodePath).mockReset().mockResolvedValue('/usr/bin/node')
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(true)
vi.mocked(acquireInstallLock).mockReset().mockResolvedValue(undefined)
})
it('aborts an in-progress relay upload at the overall deploy timeout', async () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection()
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(true)
vi.mocked(execCommand).mockImplementation((_conn, command) => {
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
return Promise.resolve(stageCommandResponse(command) ?? '')
})
vi.mocked(isRelayAlreadyInstalled).mockResolvedValueOnce(false).mockResolvedValueOnce(false)
let uploadSignal: AbortSignal | undefined
conn.uploadDirectory = vi.fn((_localDir, _remoteDir, options) => {
uploadSignal = options?.signal
return new Promise<void>((_resolve, reject) => {
uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), {
once: true
})
})
})
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.advanceTimersByTimeAsync(0)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(900_000)
const result = await promise
expect(result).toBeInstanceOf(Error)
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
expect(uploadSignal?.aborted).toBe(true)
} finally {
vi.useRealTimers()
}
})
it('joins the bounded teardown window when an aborted transfer never settles', async () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection()
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(false)
vi.mocked(execCommand).mockImplementation((_conn, command) => {
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
return Promise.resolve(stageCommandResponse(command) ?? '')
})
let uploadSignal: AbortSignal | undefined
conn.uploadDirectory = vi.fn((_localDir, _remoteDir, options) => {
uploadSignal = options?.signal
return new Promise<void>(() => {})
})
const deployment = deployAndLaunchRelay(conn).catch((error: Error) => error)
await vi.advanceTimersByTimeAsync(0)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
expect(uploadSignal?.aborted).toBe(true)
expect(
await Promise.race([deployment.then(() => 'settled'), Promise.resolve('pending')])
).toBe('pending')
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS)
await expect(deployment).resolves.toMatchObject({
message: 'Relay deployment timed out after 900s',
sshChannelCloseConfirmed: false,
sshTransferTeardownConfirmed: false
})
} finally {
vi.useRealTimers()
}
})
it('waits for a deferred SFTP upload before acquiring the install lock', async () => {
const conn = makeMockConnection()
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockResolvedValueOnce(false)
.mockResolvedValue(true)
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
return Promise.resolve('')
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
let finishUpload: () => void = () => {}
conn.uploadDirectory = vi.fn(
() =>
new Promise<void>((resolve) => {
finishUpload = resolve
})
)
const deploy = deployAndLaunchRelay(conn).catch(() => undefined)
await vi.waitFor(() => expect(conn.uploadDirectory).toHaveBeenCalledTimes(1))
expect(acquireInstallLock).not.toHaveBeenCalled()
finishUpload()
await vi.waitFor(() => expect(acquireInstallLock).toHaveBeenCalledTimes(1))
await deploy
})
it('drops only its stage when a sibling finishes before the locked re-probe', async () => {
const conn = makeMockConnection()
const events: string[] = []
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockImplementationOnce(async () => {
events.push('initial-probe')
return false
})
.mockImplementationOnce(async () => {
events.push('locked-re-probe')
return true
})
vi.mocked(acquireInstallLock).mockImplementationOnce(async () => {
events.push('lock')
})
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
return Promise.resolve('')
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
conn.uploadDirectory = vi.fn().mockImplementation(async () => {
events.push('upload')
})
await deployAndLaunchRelay(conn)
expect(events).toEqual(['initial-probe', 'upload', 'lock', 'locked-re-probe'])
const commands = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
expect(commands.some((command) => command.includes('cp -a'))).toBe(false)
const uploadStageRemovals = commands.filter(
(command) => /\.sftp-namespace-[0-9a-f]{32}/u.test(command) && command.includes('rm -rf')
)
expect(uploadStageRemovals).toHaveLength(1)
expect(uploadStageRemovals[0]).toContain('/.orca-remote/.upload-stages/claim-0')
})
it('runs bounded fixed-path recovery before a fresh upload', async () => {
const conn = makeMockConnection()
const events: string[] = []
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockResolvedValueOnce(false)
.mockResolvedValue(true)
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
if (command.includes('deleting_old=')) {
events.push('recover')
}
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
return Promise.resolve('')
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
conn.uploadDirectory = vi.fn().mockImplementation(async () => {
events.push('upload')
})
await deployAndLaunchRelay(conn)
expect(events.indexOf('recover')).toBeLessThan(events.indexOf('upload'))
})
it('launches before one bounded installed-path recovery exec', async () => {
const conn = makeMockConnection()
const events: string[] = []
let socketProbe = 0
vi.mocked(waitForSentinel).mockImplementation(async () => {
events.push('launch-ready')
return {
write: vi.fn(),
onData: vi.fn(),
onClose: vi.fn()
}
})
vi.mocked(execCommand).mockImplementation((_conn, command) => {
if (command.includes('deleting_old=')) {
events.push('recover')
}
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ % 2 === 0 ? 'DEAD' : 'READY')
}
return Promise.resolve('')
})
for (let deployment = 0; deployment < 12; deployment += 1) {
await deployAndLaunchRelay(conn)
}
expect(events).toEqual(Array.from({ length: 12 }, () => ['launch-ready', 'recover']).flat())
const commands = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
const recoveryCommands = commands.filter((command) => command.includes('deleting_old='))
expect(recoveryCommands).toHaveLength(12)
expect(recoveryCommands.every((command) => command.includes('.upload-stages'))).toBe(true)
expect(recoveryCommands.every((command) => !command.includes('find "$pool"'))).toBe(true)
})
it('never enumerates arbitrary stage paths during installation', async () => {
const conn = makeMockConnection()
conn.uploadDirectory = vi.fn().mockResolvedValue(undefined)
conn.writeFile = vi.fn().mockResolvedValue(undefined)
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockResolvedValueOnce(false)
.mockResolvedValue(true)
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
return Promise.resolve('')
})
await deployAndLaunchRelay(conn)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(1)
const stageCommands = vi
.mocked(execCommand)
.mock.calls.map(([, command]) => command)
.filter((command) => command.includes('.upload-stages'))
expect(stageCommands.length).toBeGreaterThan(0)
expect(stageCommands.every((command) => !command.includes('-mindepth'))).toBe(true)
})
it('keeps the staging tree after an unconfirmed system SSH upload termination', async () => {
const conn = makeMockConnection()
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(false)
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
return Promise.resolve(
command.includes('uname') ? '__ORCA_REMOTE_PLATFORM__ Linux x86_64' : '/home/user'
)
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
const termination = Object.assign(new Error('upload teardown unconfirmed'), {
sshChannelCloseConfirmed: false
})
conn.uploadDirectory = vi.fn().mockRejectedValue(termination)
await expect(deployAndLaunchRelay(conn)).rejects.toBe(termination)
expect(acquireInstallLock).not.toHaveBeenCalled()
expect(
vi
.mocked(execCommand)
.mock.calls.some(
([, command]) =>
/\.sftp-namespace-[0-9a-f]{32}/u.test(command) && command.includes('rm -rf')
)
).toBe(false)
})
it('retries immediately after an unconfirmed upload termination instead of waiting on a fresh install lock', async () => {
const conn = makeMockConnection()
const termination = Object.assign(new Error('upload teardown unconfirmed'), {
sshChannelCloseConfirmed: false
})
let lockHeld = false
vi.mocked(acquireInstallLock).mockImplementation((_conn, _dir, _host, options) => {
if (!lockHeld) {
lockHeld = true
return Promise.resolve()
}
return new Promise<void>((_resolve, reject) => {
options?.signal?.addEventListener('abort', () => reject(options.signal?.reason), {
once: true
})
})
})
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockResolvedValueOnce(false)
.mockResolvedValueOnce(false)
.mockResolvedValue(true)
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
return Promise.resolve('')
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
conn.uploadDirectory = vi.fn().mockRejectedValueOnce(termination).mockResolvedValue(undefined)
await expect(deployAndLaunchRelay(conn)).rejects.toBe(termination)
await deployAndLaunchRelay(conn)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(2)
expect(acquireInstallLock).toHaveBeenCalledTimes(1)
})
it('cleans a confirmed-abort staging tree so the next deployment retries immediately', async () => {
const conn = makeMockConnection()
const termination = Object.assign(new Error('upload aborted'), {
sshChannelCloseConfirmed: true
})
vi.mocked(isRelayAlreadyInstalled)
.mockReset()
.mockResolvedValueOnce(false)
.mockResolvedValueOnce(false)
.mockResolvedValue(true)
let socketProbe = 0
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const stageResponse = stageCommandResponse(command)
if (stageResponse !== undefined) {
return Promise.resolve(stageResponse)
}
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('test -S')) {
return Promise.resolve(socketProbe++ === 0 ? 'DEAD' : 'READY')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
return Promise.resolve('')
})
conn.writeFile = vi.fn().mockResolvedValue(undefined)
conn.uploadDirectory = vi.fn().mockRejectedValueOnce(termination).mockResolvedValue(undefined)
await expect(deployAndLaunchRelay(conn)).rejects.toBe(termination)
expect(
vi
.mocked(execCommand)
.mock.calls.some(
([, command]) =>
/\.sftp-namespace-[0-9a-f]{32}/u.test(command) && command.includes('rm -rf')
)
).toBe(true)
await deployAndLaunchRelay(conn)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(2)
})
})
+3
View File
@@ -11,3 +11,6 @@ const NATIVE_DEPS_REPAIR_BUDGET_MS = 2 * NATIVE_DEPS_COMMAND_TIMEOUT_MS
// finalize commands around install + rebuild. Keep the outer bound above that
// valid worst path while remaining below the 20-minute stale-lock threshold.
export const RELAY_DEPLOY_TIMEOUT_MS = NATIVE_DEPS_REPAIR_BUDGET_MS + 7 * 60_000
// Lets an aborted SSH transfer report whether its channel and local resources actually settled.
export const RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS = 5_000
+50 -52
View File
@@ -89,6 +89,7 @@ import { execCommand, waitForSentinel } from './ssh-relay-deploy-helpers'
import { resolveRemoteNodePath } from './ssh-remote-node-resolution'
import { isRelayAlreadyInstalled } from './ssh-relay-versioned-install'
import { acquireInstallLock } from './ssh-relay-install-lock'
import * as DeployTiming from './ssh-relay-deploy-timing'
import type { SshConnection } from './ssh-connection'
import type * as SshRemoteNodeResolution from './ssh-remote-node-resolution'
import {
@@ -140,6 +141,15 @@ function queueLaunchNamespaceAndDeadSocketProbe(): void {
describe('deployAndLaunchRelay', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(execCommand).mockReset().mockResolvedValue('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
vi.mocked(waitForSentinel).mockReset().mockResolvedValue({
write: vi.fn(),
onData: vi.fn(),
onClose: vi.fn()
})
vi.mocked(resolveRemoteNodePath).mockReset().mockResolvedValue('/usr/bin/node')
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(true)
vi.mocked(acquireInstallLock).mockReset().mockResolvedValue(undefined)
})
it('calls exec to detect remote platform', async () => {
@@ -575,7 +585,10 @@ describe('deployAndLaunchRelay', () => {
await vi.advanceTimersByTimeAsync(301_000)
expect(await Promise.race([promise, Promise.resolve('pending')])).toBe('pending')
await vi.advanceTimersByTimeAsync(600_000)
await vi.advanceTimersByTimeAsync(DeployTiming.RELAY_DEPLOY_TIMEOUT_MS - 301_000)
expect(await Promise.race([promise, Promise.resolve('pending')])).toBe('pending')
await vi.advanceTimersByTimeAsync(DeployTiming.RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS)
const result = await promise
expect(result).toBeInstanceOf(Error)
@@ -588,9 +601,22 @@ describe('deployAndLaunchRelay', () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection()
vi.mocked(execCommand)
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
.mockResolvedValueOnce('/home/user')
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(false)
conn.uploadDirectory = vi.fn().mockResolvedValue(undefined)
conn.writeFile = vi.fn().mockResolvedValue(undefined)
vi.mocked(execCommand).mockImplementation((_conn, command) => {
if (command.includes('uname')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
const marker = command.match(/\.sftp-namespace-[0-9a-f]{32}/u)?.[0]
if (command.includes('__ORCA_UPLOAD_STAGE_SLOT__') && marker) {
return Promise.resolve(`__ORCA_UPLOAD_STAGE_SLOT__${marker}:slot-0`)
}
return Promise.resolve('')
})
vi.mocked(isRelayAlreadyInstalled).mockResolvedValueOnce(false)
let lockSignal: AbortSignal | undefined
vi.mocked(acquireInstallLock).mockImplementationOnce((_conn, _dir, _host, options) => {
@@ -604,7 +630,7 @@ describe('deployAndLaunchRelay', () => {
await vi.advanceTimersByTimeAsync(0)
expect(acquireInstallLock).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(900_000)
await vi.advanceTimersByTimeAsync(DeployTiming.RELAY_DEPLOY_TIMEOUT_MS)
const result = await promise
expect(result).toBeInstanceOf(Error)
@@ -615,40 +641,6 @@ describe('deployAndLaunchRelay', () => {
}
})
it('aborts an in-progress relay upload at the overall deploy timeout', async () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection()
vi.mocked(execCommand)
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
.mockResolvedValueOnce('/home/user')
.mockResolvedValueOnce('') // mkdir remote relay dir
vi.mocked(isRelayAlreadyInstalled).mockResolvedValueOnce(false).mockResolvedValueOnce(false)
let uploadSignal: AbortSignal | undefined
conn.uploadDirectory = vi.fn((_localDir, _remoteDir, options) => {
uploadSignal = options?.signal
return new Promise<void>((_resolve, reject) => {
uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), {
once: true
})
})
})
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.advanceTimersByTimeAsync(0)
expect(conn.uploadDirectory).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(900_000)
const result = await promise
expect(result).toBeInstanceOf(Error)
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
expect(uploadSignal?.aborted).toBe(true)
} finally {
vi.useRealTimers()
}
})
it('aborts a launch started near the deploy deadline and closes its channel once', async () => {
vi.useFakeTimers()
try {
@@ -660,6 +652,7 @@ describe('deployAndLaunchRelay', () => {
close: vi.fn()
}
const conn = makeMockConnection()
vi.mocked(isRelayAlreadyInstalled).mockReset().mockResolvedValue(true)
vi.mocked(conn.exec).mockResolvedValue(launchChannel as never)
const mockExecCommand = vi.mocked(execCommand)
mockExecCommand
@@ -710,19 +703,24 @@ describe('deployAndLaunchRelay', () => {
const connA = makeMockConnection()
const connB = makeMockConnection()
const mockExecCommand = vi.mocked(execCommand)
mockExecCommand
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') // tagged POSIX platform probe A
.mockResolvedValueOnce('/home/user') // $HOME A
.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe A
.mockResolvedValueOnce('') // launch namespace marker A
.mockResolvedValueOnce('DEAD') // probe A
.mockResolvedValueOnce('READY') // poll A
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') // tagged POSIX platform probe B
.mockResolvedValueOnce('/home/user') // $HOME B
.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe B
.mockResolvedValueOnce('') // launch namespace marker B
.mockResolvedValueOnce('DEAD') // probe B
.mockResolvedValueOnce('READY') // poll B
mockExecCommand.mockImplementation((_conn, command) => {
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
}
if (command === 'echo $HOME') {
return Promise.resolve('/home/user')
}
if (command.includes('ORCA-NATIVE')) {
return Promise.resolve('ORCA-NATIVE-DEPS-OK')
}
if (command.includes('process.stdout.write("READY")')) {
return Promise.resolve('READY')
}
if (command.includes('test -S')) {
return Promise.resolve('DEAD')
}
return Promise.resolve('')
})
await deployAndLaunchRelay(connA, undefined, 300, 'target-a')
await deployAndLaunchRelay(connB, undefined, 300, 'target-b')
+249 -78
View File
@@ -3,7 +3,7 @@ import { join } from 'node:path'
import { existsSync } from 'node:fs'
import { app } from 'electron'
import type { SshConnection } from './ssh-connection'
import type { RelayPlatform } from './relay-protocol'
import { RELAY_REMOTE_DIR, type RelayPlatform } from './relay-protocol'
import type { MultiplexerTransport } from './ssh-channel-multiplexer'
import {
waitForSentinel,
@@ -15,11 +15,15 @@ import { writeRelayEndpointCredential } from './ssh-relay-endpoint-credential'
import {
createRelayInstallMarkerCommand,
createRelayInstallNamespace,
createRelayUploadStageNamespace,
makeRelayInstallDirectoryCommand,
relayHomeRelativeDir,
relaySftpNamespaceMapping,
type RelayInstallNamespace
relayUploadStageSftpNamespaceMapping,
type RelayInstallNamespace,
type RelayUploadStageNamespace
} from './ssh-relay-install-namespace'
import { createRelayInstallMarkerFileName } from './ssh-relay-install-marker'
import { resolveRemoteNodePath } from './ssh-remote-node-resolution'
import {
readLocalFullVersion,
@@ -36,7 +40,11 @@ import {
tryAcquireRelayGcClaim,
waitForRelayGcClaimRelease
} from './ssh-relay-gc-claim'
import { NATIVE_DEPS_COMMAND_TIMEOUT_MS, RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
import {
NATIVE_DEPS_COMMAND_TIMEOUT_MS,
RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS,
RELAY_DEPLOY_TIMEOUT_MS
} from './ssh-relay-deploy-timing'
import { createSshOperationAbortError, shellEscape } from './ssh-connection-utils'
import {
probeBuildToolchain,
@@ -50,6 +58,15 @@ import {
readRemoteHomeCommand,
removeRemoteFileCommand
} from './ssh-remote-commands'
import {
cleanupOwnedRelayUploadStageCommand,
parseReservedRelayUploadStage,
promoteOwnedRelayUploadStageCommand,
recoverOneStaleRelayUploadStageCommand,
relayUploadStagePromotionConfirmed,
RELAY_UPLOAD_STAGE_POOL_NAME,
reserveRelayUploadStageCommand
} from './ssh-relay-upload-stage-commands'
import {
isWindowsRemoteHost,
joinRemotePath,
@@ -118,31 +135,77 @@ export async function deployAndLaunchRelay(
relayInstanceId?: string
): Promise<RelayDeployResult> {
let timeoutHandle: ReturnType<typeof setTimeout>
// Why: Promise.race doesn't cancel its loser; abort so a contended install-lock waiter can't mutate the relay after this call times out.
const deployAbortController = new AbortController()
const timeoutPromise = new Promise<never>((_resolve, reject) => {
const timedOut = Symbol('relay-deploy-timeout')
const deployment = deployAndLaunchRelayInner(
conn,
onProgress,
graceTimeSeconds,
relayInstanceId,
deployAbortController.signal
).then(
(result) => ({ status: 'fulfilled' as const, result }),
(error: unknown) => ({ status: 'rejected' as const, error })
)
const timeoutPromise = new Promise<typeof timedOut>((resolve) => {
timeoutHandle = setTimeout(() => {
deployAbortController.abort()
reject(new Error(`Relay deployment timed out after ${RELAY_DEPLOY_TIMEOUT_MS / 1000}s`))
resolve(timedOut)
}, RELAY_DEPLOY_TIMEOUT_MS)
})
try {
return await Promise.race([
deployAndLaunchRelayInner(
conn,
onProgress,
graceTimeSeconds,
relayInstanceId,
deployAbortController.signal
),
timeoutPromise
])
const outcome = await Promise.race([deployment, timeoutPromise])
if (outcome !== timedOut) {
if (outcome.status === 'fulfilled') {
return outcome.result
}
throw outcome.error
}
const teardownExpired = Symbol('relay-deploy-teardown-timeout')
let teardownTimeoutHandle: ReturnType<typeof setTimeout>
const teardown = await Promise.race([
deployment,
new Promise<typeof teardownExpired>((resolve) => {
teardownTimeoutHandle = setTimeout(
() => resolve(teardownExpired),
RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS
)
})
]).finally(() => clearTimeout(teardownTimeoutHandle!))
const timeoutError = Object.assign(
new Error(`Relay deployment timed out after ${RELAY_DEPLOY_TIMEOUT_MS / 1000}s`),
teardownConfirmation(teardown === teardownExpired ? undefined : teardown)
)
throw timeoutError
} finally {
clearTimeout(timeoutHandle!)
}
}
function teardownConfirmation(
outcome:
| { status: 'fulfilled'; result: RelayDeployResult }
| { status: 'rejected'; error: unknown }
| undefined
): { sshChannelCloseConfirmed: boolean; sshTransferTeardownConfirmed: boolean } {
if (!outcome) {
return { sshChannelCloseConfirmed: false, sshTransferTeardownConfirmed: false }
}
if (outcome.status === 'fulfilled') {
return { sshChannelCloseConfirmed: true, sshTransferTeardownConfirmed: true }
}
const error = outcome.error as {
sshChannelCloseConfirmed?: unknown
sshTransferTeardownConfirmed?: unknown
}
return {
sshChannelCloseConfirmed: error?.sshChannelCloseConfirmed === true,
sshTransferTeardownConfirmed: error?.sshTransferTeardownConfirmed === true
}
}
/**
* Resolve the remote home, derive the versioned relay dir, and check whether the relay is installed there.
*
@@ -324,6 +387,13 @@ async function deployAndLaunchRelayAttempt(
// Why: derive the home-relative suffix once — recomputing it by stripping the shell home breaks on a split namespace.
const homeRelativeRelayDir = relayHomeRelativeDir(fullVersion)
const uploadStagePoolDir = joinRemotePath(
hostPlatform,
remoteHome,
RELAY_REMOTE_DIR,
RELAY_UPLOAD_STAGE_POOL_NAME
)
const homeRelativeUploadStagePoolDir = `${RELAY_REMOTE_DIR}/${RELAY_UPLOAD_STAGE_POOL_NAME}`
let ownsInstallLock = false
let launchGcClaimToken: string | undefined
@@ -343,63 +413,137 @@ async function deployAndLaunchRelayAttempt(
launchNamespace = launchFence.sftpNamespace
deploySignal?.throwIfAborted()
} else {
// Why: serialize concurrent first-installs via a host-native exclusive lock; the loser polls to re-check installed or steal a stale lock.
await acquireInstallLock(conn, remoteRelayDir, hostPlatform, { signal: deploySignal })
ownsInstallLock = true
await execHostCommand(
conn,
hostPlatform,
recoverOneStaleRelayUploadStageCommand(hostPlatform, uploadStagePoolDir),
{ signal: deploySignal }
)
const uploadStageOwner = createRelayInstallMarkerFileName()
const reservation = await execHostCommand(
conn,
hostPlatform,
reserveRelayUploadStageCommand(hostPlatform, uploadStagePoolDir, uploadStageOwner),
{ signal: deploySignal }
)
const uploadStage = parseReservedRelayUploadStage(
hostPlatform,
uploadStagePoolDir,
uploadStageOwner,
reservation
)
const uploadStagePayloadDir = joinRemotePath(hostPlatform, uploadStage.slotDir, 'payload')
const uploadStageNamespace = createRelayUploadStageNamespace(
`${homeRelativeUploadStagePoolDir}/${uploadStage.slotName}`,
uploadStageOwner
)
const uploadStageSftpNamespace = uploadStageNamespaceIfSupported(
conn,
hostPlatform,
uploadStageNamespace
)
let uploadStageCleanupAllowed = true
onProgress?.('Uploading relay...')
console.log('[ssh-relay] Uploading relay...')
try {
// Re-probe after acquiring the lock — a sibling installer may have finished while we waited.
if (
!(await isRelayAlreadyInstalled(conn, remoteRelayDir, hostPlatform, {
signal: deploySignal
}))
) {
launchNamespace = createInstallNamespaceIfSupported(
conn,
hostPlatform,
homeRelativeRelayDir
)
onProgress?.('Uploading relay...')
console.log('[ssh-relay] Uploading relay...')
try {
await uploadRelay(
conn,
platform,
remoteRelayDir,
uploadStagePayloadDir,
fullVersion,
hostPlatform,
deploySignal,
launchNamespace
{ rootDir: uploadStage.slotDir, namespace: uploadStageSftpNamespace }
)
console.log('[ssh-relay] Upload complete')
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
uploadStageCleanupAllowed = false
}
throw err
}
onProgress?.('Installing native dependencies...')
console.log('[ssh-relay] Installing native dependencies...')
await installNativeDeps(
try {
await acquireInstallLock(conn, remoteRelayDir, hostPlatform, { signal: deploySignal })
ownsInstallLock = true
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
ownsInstallLock = true
}
throw err
}
try {
// Re-probe after acquiring the lock — a sibling installer may have finished while we waited.
if (
!(await isRelayAlreadyInstalled(conn, remoteRelayDir, hostPlatform, {
signal: deploySignal
}))
) {
launchNamespace = await createRelayLaunchNamespace(
conn,
hostPlatform,
remoteRelayDir,
homeRelativeRelayDir,
deploySignal
)
try {
const promotion = await execHostCommand(
conn,
hostPlatform,
promoteOwnedRelayUploadStageCommand(
hostPlatform,
uploadStage,
uploadStageOwner,
remoteRelayDir
),
{ signal: deploySignal }
)
if (!relayUploadStagePromotionConfirmed(uploadStageOwner, promotion)) {
throw new Error('Relay upload stage ownership was lost before promotion')
}
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
uploadStageCleanupAllowed = false
}
throw err
}
console.log('[ssh-relay] Upload complete')
onProgress?.('Installing native dependencies...')
console.log('[ssh-relay] Installing native dependencies...')
await installNativeDeps(
conn,
remoteRelayDir,
platform,
hostPlatform,
nodePath,
deploySignal,
[],
launchNamespace
)
console.log('[ssh-relay] Native deps installed')
// Why: mark complete but retain the lock until launch makes daemon liveness observable to cross-version GC.
await finalizeInstall(conn, remoteRelayDir, hostPlatform, {
signal: deploySignal,
releaseLock: false
})
}
} catch (err) {
if (!isUnconfirmedSshCommandTermination(err)) {
await abandonInstall(conn, remoteRelayDir, hostPlatform)
ownsInstallLock = false
}
throw err
}
} finally {
if (uploadStageCleanupAllowed) {
await execHostCommand(
conn,
remoteRelayDir,
platform,
hostPlatform,
nodePath,
deploySignal,
[],
launchNamespace
)
console.log('[ssh-relay] Native deps installed')
// Why: mark complete but retain the lock until launch makes daemon liveness observable to cross-version GC.
await finalizeInstall(conn, remoteRelayDir, hostPlatform, {
signal: deploySignal,
releaseLock: false
})
cleanupOwnedRelayUploadStageCommand(hostPlatform, uploadStage, uploadStageOwner)
).catch(() => {})
}
} catch (err) {
// Why: leave a partial install dir (no .install-complete) so the next deploy re-runs upload + install.
// Why: keep the lock if remote termination was unconfirmed — stale recovery beats overlapping a still-running npm.
if (!isUnconfirmedSshCommandTermination(err)) {
await abandonInstall(conn, remoteRelayDir, hostPlatform)
ownsInstallLock = false
}
throw err
}
}
@@ -431,11 +575,19 @@ async function deployAndLaunchRelayAttempt(
}
console.log('[ssh-relay] Relay started successfully')
// Why: best-effort GC of unreferenced sibling version dirs; errors are swallowed so a GC failure never blocks connecting.
void gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
windowsNodePath: launched.nodePath,
windowsSockNames: [relaySocketNameForInstanceId(relayInstanceId)]
}).catch(() => {})
void execHostCommand(
conn,
hostPlatform,
recoverOneStaleRelayUploadStageCommand(hostPlatform, uploadStagePoolDir)
)
.catch(() => {})
.then(() =>
gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
windowsNodePath: launched.nodePath,
windowsSockNames: [relaySocketNameForInstanceId(relayInstanceId)]
})
)
.catch(() => {})
return {
transport: launched.transport,
@@ -457,7 +609,7 @@ async function uploadRelay(
fullVersion: string,
hostPlatform: RemoteHostPlatform,
signal?: AbortSignal,
namespace?: RelayInstallNamespace
stage?: { rootDir: string; namespace?: RelayUploadStageNamespace }
): Promise<void> {
const localRelayDir = getLocalRelayPath(platform)
if (!localRelayDir || !existsSync(localRelayDir)) {
@@ -467,18 +619,19 @@ async function uploadRelay(
)
}
// Why: the install-owner marker rides along with mkdir, so a standard install spends no extra exec channel on it.
await execHostCommand(
conn,
hostPlatform,
makeRelayInstallDirectoryCommand(hostPlatform, remoteDir, namespace),
{ signal }
)
if (!stage) {
await execHostCommand(
conn,
hostPlatform,
makeRelayInstallDirectoryCommand(hostPlatform, remoteDir),
{ signal }
)
}
await uploadRelayDirectory(conn, localRelayDir, remoteDir, hostPlatform, {
signal,
sftpNamespace: namespace
? relaySftpNamespaceMapping(namespace, hostPlatform, remoteDir)
sftpNamespace: stage?.namespace
? relayUploadStageSftpNamespaceMapping(stage.namespace, hostPlatform, stage.rootDir)
: undefined
})
@@ -499,8 +652,13 @@ async function uploadRelay(
fullVersion,
{
signal,
sftpNamespace: namespace
? relaySftpNamespaceMapping(namespace, hostPlatform, remoteDir, '.version')
sftpNamespace: stage?.namespace
? relayUploadStageSftpNamespaceMapping(
stage.namespace,
hostPlatform,
stage.rootDir,
'.version'
)
: undefined
}
)
@@ -524,6 +682,19 @@ function createInstallNamespaceIfSupported(
return usesSystemSsh ? undefined : createRelayInstallNamespace(homeRelativeRelayDir)
}
function uploadStageNamespaceIfSupported(
conn: SshConnection,
hostPlatform: RemoteHostPlatform,
namespace: RelayUploadStageNamespace
): RelayUploadStageNamespace | undefined {
if (isWindowsRemoteHost(hostPlatform)) {
return undefined
}
const usesSystemSsh =
typeof conn.usesSystemSshTransport === 'function' ? conn.usesSystemSshTransport() : false
return usesSystemSsh ? undefined : namespace
}
const NODE_PTY_VERSION = '1.1.0'
const NODE_PTY_CONSOLE_LIST_PATCH_FILENAME = 'node-pty-1.1.0-console-list-agent-patch.cjs'
const RELAY_NATIVE_DEPS = {
@@ -0,0 +1,49 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { SshConnection } from './ssh-connection'
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: vi.fn(),
isUnconfirmedSshCommandTermination: (error: unknown) =>
error instanceof Error &&
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false
}))
vi.mock('./ssh-relay-gc-claim', () => ({
isRelayGcClaimed: vi.fn().mockResolvedValue(false),
waitForRelayGcClaimRelease: vi.fn().mockResolvedValue(undefined)
}))
import { execCommand } from './ssh-relay-deploy-helpers'
import { acquireInstallLock } from './ssh-relay-install-lock'
import { getRemoteHostPlatform } from './ssh-remote-platform'
describe('acquireInstallLock', () => {
afterEach(() => {
vi.resetAllMocks()
})
it('propagates an unconfirmed lock-create termination so deploy can retain the lock fence', async () => {
const controller = new AbortController()
const termination = Object.assign(new Error('lock creation termination was not confirmed'), {
sshChannelCloseConfirmed: false
})
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
if (command.includes('.install-lock')) {
controller.abort(
Object.assign(new Error('SSH operation was cancelled'), { name: 'AbortError' })
)
throw termination
}
return ''
})
await expect(
acquireInstallLock(
{} as SshConnection,
'/home/u/.orca-remote/relay-0.1.0',
getRemoteHostPlatform('linux-x64'),
{ signal: controller.signal }
)
).rejects.toBe(termination)
})
})
+5 -2
View File
@@ -1,5 +1,5 @@
import type { SshConnection } from './ssh-connection'
import { execCommand } from './ssh-relay-deploy-helpers'
import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
import { isRelayGcClaimed, waitForRelayGcClaimRelease } from './ssh-relay-gc-claim'
import {
@@ -101,7 +101,10 @@ export async function acquireInstallLock(
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {})
options?.signal?.throwIfAborted()
}
} catch {
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
options?.signal?.throwIfAborted()
// A failed mkdir is lock contention; keep the connection-specific error
// out of the user path until the bounded wait expires.
@@ -6,11 +6,14 @@ import { describe, expect, it } from 'vitest'
import {
createRelayInstallMarkerCommand,
createRelayInstallNamespace,
createRelayUploadStageNamespace,
makeRelayInstallDirectoryCommand,
makeRelayUploadStageDirectoryCommand,
relayHomeRelativeDir,
relayInstallMarkerShellPath,
relayRemoteDirSegments,
relaySftpNamespaceMapping
relaySftpNamespaceMapping,
relayUploadStageSftpNamespaceMapping
} from './ssh-relay-install-namespace'
import { getRemoteHostPlatform } from './ssh-remote-platform'
import { computeRemoteRelayDir } from './ssh-relay-versioned-install'
@@ -87,6 +90,7 @@ describe('relaySftpNamespaceMapping', () => {
const mapping = relaySftpNamespaceMapping(namespace, LINUX, SHELL_RELAY_DIR)
expect(mapping.homeRelativePath).toBe(`.orca-remote/relay-${VERSION}`)
expect(mapping.homeRelativeNamespaceRoot).toBe(`.orca-remote/relay-${VERSION}`)
expect(mapping.homeRelativeProbePath).toBe(
`.orca-remote/relay-${VERSION}/.install-lock/${namespace.markerFileName}`
)
@@ -119,6 +123,37 @@ describe('relaySftpNamespaceMapping', () => {
)
})
describe('relay upload stage namespace', () => {
const stageSuffix = `.orca-remote/relay-${VERSION}.upload-123e4567-e89b-12d3-a456-426614174000`
const shellStageDir = `/var/services/homes/alice/${stageSuffix}`
const namespace = createRelayUploadStageNamespace(stageSuffix)
it('maps only the attempt-owned payload subtree', () => {
const payload = relayUploadStageSftpNamespaceMapping(namespace, LINUX, shellStageDir)
const version = relayUploadStageSftpNamespaceMapping(
namespace,
LINUX,
shellStageDir,
'.version'
)
expect(payload.homeRelativeNamespaceRoot).toBe(stageSuffix)
expect(payload.homeRelativePath).toBe(`${stageSuffix}/payload`)
expect(version.homeRelativePath).toBe(`${stageSuffix}/payload/.version`)
expect(payload.shellProbePath).toBe(`${shellStageDir}/${namespace.markerFileName}`)
expect(payload.shellProbePath).toBe(version.shellProbePath)
})
it('creates the payload and marker without touching the shared install lock', () => {
const command = makeRelayUploadStageDirectoryCommand(namespace, LINUX, shellStageDir)
expect(command).toContain(`${shellStageDir}/payload`)
expect(command).toContain(`${shellStageDir}/${namespace.markerFileName}`)
expect(command).toContain('umask 077')
expect(command).not.toContain('.install-lock')
})
})
describe('install directory command', () => {
const namespace = createRelayInstallNamespace(relayHomeRelativeDir(VERSION))
+50 -3
View File
@@ -1,8 +1,8 @@
// Install-owner identity for relay uploads that cross a split shell/SFTP namespace.
// Transfer-owner identity for relay installs that cross a split shell/SFTP namespace.
//
// The shell and SFTP paths share one validated home-relative suffix
// (`.orca-remote/relay-<fullVersion>`); a random marker inside the install lock
// lets each SFTP session prove it is looking at THIS install's directory.
// (`.orca-remote/relay-<fullVersion>`); a random marker inside the shared install
// lock or attempt stage proves which shell-owned directory the SFTP session sees.
//
// See: docs/ssh-relay-sftp-namespace.md
@@ -24,6 +24,11 @@ export type RelayInstallNamespace = {
markerFileName: string
}
export type RelayUploadStageNamespace = {
homeRelativeStageDir: string
markerFileName: string
}
/**
* The two validated segments every relay path is built from. Both the shell
* builder and the SFTP-relative builder go through here so they cannot drift.
@@ -56,6 +61,16 @@ export function createRelayInstallNamespace(homeRelativeRelayDir: string): Relay
}
}
export function createRelayUploadStageNamespace(
homeRelativeStageDir: string,
markerFileName = createRelayInstallMarkerFileName()
): RelayUploadStageNamespace {
return {
homeRelativeStageDir,
markerFileName
}
}
/**
* Describe one relay transfer to the SFTP namespace resolver. `relativeFileName`
* is omitted for the bundle directory upload itself.
@@ -74,6 +89,7 @@ export function relaySftpNamespaceMapping(
}
const homeRelativeLockDir = `${namespace.homeRelativeRelayDir}/${RELAY_INSTALL_LOCK_NAME}`
return {
homeRelativeNamespaceRoot: namespace.homeRelativeRelayDir,
homeRelativePath:
relativeFileName !== undefined
? `${namespace.homeRelativeRelayDir}/${relativeFileName}`
@@ -83,6 +99,37 @@ export function relaySftpNamespaceMapping(
}
}
export function relayUploadStageSftpNamespaceMapping(
namespace: RelayUploadStageNamespace,
host: RemoteHostPlatform,
shellStageDir: string,
relativeFileName?: string
): SftpNamespacePathMapping {
if (relativeFileName !== undefined) {
assertSafeRemotePathSegment(relativeFileName, 'posix')
}
const homeRelativePayloadDir = `${namespace.homeRelativeStageDir}/payload`
return {
homeRelativeNamespaceRoot: namespace.homeRelativeStageDir,
homeRelativePath:
relativeFileName === undefined
? homeRelativePayloadDir
: `${homeRelativePayloadDir}/${relativeFileName}`,
shellProbePath: joinRemotePath(host, shellStageDir, namespace.markerFileName),
homeRelativeProbePath: `${namespace.homeRelativeStageDir}/${namespace.markerFileName}`
}
}
export function makeRelayUploadStageDirectoryCommand(
namespace: RelayUploadStageNamespace,
host: RemoteHostPlatform,
shellStageDir: string
): string {
const payloadDir = joinRemotePath(host, shellStageDir, 'payload')
const markerPath = joinRemotePath(host, shellStageDir, namespace.markerFileName)
return `${makeRemoteDirectoryCommand(host, payloadDir)} && umask 077 && touch ${shellEscape(markerPath)}`
}
export function relayInstallMarkerShellPath(
namespace: RelayInstallNamespace,
host: RemoteHostPlatform,
+4 -1
View File
@@ -40,7 +40,9 @@ export async function uploadRelayDirectory(
sftpNamespace: options?.sftpNamespace
})
options?.signal?.throwIfAborted()
await uploadDirectory(sftp, localRelayDir, targetDir)
await uploadDirectory(sftp, localRelayDir, targetDir, localRelayDir, {
signal: options?.signal
})
})
}
@@ -93,6 +95,7 @@ async function runSftpFallbackTransfer(
(onClose) => {
sftp.once('close', onClose)
endSftp()
return () => sftp.removeListener('close', onClose)
}
)
} finally {
@@ -54,6 +54,20 @@ export function makeMockConnection(capture: SftpWriteCapture): SshConnection {
export type ExecResponse = string | { reject: string }
const STAGE_OWNER = '.sftp-namespace-00000000000000000000000000000000'
export function makeStagedFirstInstallExecPrefix(): ExecResponse[] {
return [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // bounded stale-stage recovery
`__ORCA_UPLOAD_STAGE_SLOT__${STAGE_OWNER}:slot-0`,
'', // chmod staged node
'', // final install namespace marker
`__ORCA_UPLOAD_STAGE_PROMOTION__${STAGE_OWNER}:PROMOTED`
]
}
// Repair reconnect (isRelayAlreadyInstalled → true) where BOTH native deps are broken and the host
// cannot compile node-pty, so the caller's resets must survive into the node-pty-less reinstall.
export function makeRepairToolchainSkipExecResponses(): ExecResponse[] {
@@ -108,23 +122,18 @@ export function makeExecResponses(opts: {
// reinstall succeeds; only then are the chmod/probe/launch slots reached.
if (opts.npmInstall !== 'ok' && opts.nodePtySkipRetry !== 'ok') {
return [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir (uploadRelay)
'', // chmod +x node
...makeStagedFirstInstallExecPrefix(),
opts.npmInstall, // npm install rejects
opts.toolchainProbe ?? 'HAVE make\nHAVE g++\nHAVE cc\nHAVE python3\nPKG apt-get',
...(opts.nodePtySkipRetry ? [opts.nodePtySkipRetry] : []) // reinstall also rejects
...(opts.nodePtySkipRetry ? [opts.nodePtySkipRetry] : []), // reinstall also rejects
'' // clean stage root
]
}
if (opts.npmInstall !== 'ok') {
// Skip path, exactly as production runs it: no chmod-prebuilds (node-pty is gone) and no rebuild
// (it provably can't compile here). The probe still runs to catch a dead @parcel/watcher.
return [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir (uploadRelay)
'', // chmod +x node
...makeStagedFirstInstallExecPrefix(),
opts.npmInstall, // npm install rejects on the missing compiler
opts.toolchainProbe ?? 'HAVE python3\nPKG dnf',
'', // rm -rf node-pty + reinstall without it
@@ -134,6 +143,7 @@ export function makeExecResponses(opts: {
: 'ORCA-NATIVE-DEPS-MISSING:node-pty\nMISSING\n',
'', // cat probe stderr
'', // rm -f probe stderr
'', // clean stage root
'DEAD',
'', // publish the per-launch credential
'READY'
@@ -151,10 +161,7 @@ export function makeExecResponses(opts: {
? { reject: 'cd: no such file or directory' }
: probe
const slots: ExecResponse[] = [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir (uploadRelay)
'', // chmod +x node
...makeStagedFirstInstallExecPrefix(),
'', // npm install native deps
'', // chmod prebuilds
probeSlot
@@ -178,6 +185,6 @@ export function makeExecResponses(opts: {
slots.push('') // rm -f stderr after rebuild probe
}
}
slots.push('DEAD', '', 'READY')
slots.push('', 'DEAD', '', 'READY') // clean stage root, launch, credential, readiness
return slots
}
@@ -0,0 +1,227 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as RelayInstallMarkerModule from './ssh-relay-install-marker'
vi.mock('electron', () => ({
app: { getAppPath: () => '/mock/app' }
}))
vi.mock('fs', () => ({
existsSync: vi.fn().mockReturnValue(true),
readFileSync: vi.fn().mockReturnValue('0.1.0+testhash')
}))
vi.mock('./relay-protocol', () => ({
RELAY_VERSION: '0.1.0',
RELAY_REMOTE_DIR: '.orca-remote',
parseUnameToRelayPlatform: vi.fn().mockReturnValue('linux-x64'),
RELAY_SENTINEL: 'ORCA-RELAY v0.1.0 READY\n',
RELAY_SENTINEL_TIMEOUT_MS: 10_000
}))
vi.mock('./ssh-relay-deploy-helpers', () => ({
uploadDirectory: vi.fn().mockResolvedValue(undefined),
waitForSentinel: vi.fn().mockResolvedValue({
write: vi.fn(),
onData: vi.fn(),
onClose: vi.fn()
}),
isUnconfirmedSshCommandTermination: (error: unknown) =>
error instanceof Error &&
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false,
execCommand: vi.fn()
}))
vi.mock('./ssh-remote-node-resolution', () => ({
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
}))
vi.mock('./ssh-relay-install-marker', async (importOriginal) => ({
...(await importOriginal<typeof RelayInstallMarkerModule>()),
createRelayInstallMarkerFileName: () => '.sftp-namespace-00000000000000000000000000000000'
}))
vi.mock('./ssh-relay-versioned-install', () => ({
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+testhash'),
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
isRelayAlreadyInstalled: vi.fn().mockResolvedValue(false),
finalizeInstall: vi.fn().mockResolvedValue(undefined),
abandonInstall: vi.fn().mockResolvedValue(undefined),
gcOldRelayVersions: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./ssh-relay-install-lock', () => ({
acquireInstallLock: vi.fn().mockResolvedValue(undefined),
RELAY_INSTALL_LOCK_NAME: '.install-lock'
}))
vi.mock('./ssh-relay-repair-lock', () => ({
tryAcquireRelayRepairLock: vi.fn().mockResolvedValue('acquired')
}))
vi.mock('./ssh-relay-gc-claim', () => ({
releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'),
tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('launch-token'),
waitForRelayGcClaimRelease: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./ssh-connection-utils', () => ({
shellEscape: (s: string) => `'${s}'`
}))
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { execCommand, uploadDirectory } from './ssh-relay-deploy-helpers'
import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
import { parseUnameToRelayPlatform } from './relay-protocol'
import {
abandonInstall,
finalizeInstall,
isRelayAlreadyInstalled
} from './ssh-relay-versioned-install'
import { acquireInstallLock } from './ssh-relay-install-lock'
import {
makeExecResponses,
makeStagedFirstInstallExecPrefix,
makeMockConnection,
type ExecResponse,
type SftpWriteCapture
} from './ssh-relay-native-deps-install-fixture'
describe('installNativeDeps staged uploads', () => {
const sftpCapture: SftpWriteCapture = {
paths: [],
contents: {},
execCallCountAtWrite: {}
}
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(execCommand).mockReset().mockResolvedValue('')
vi.mocked(uploadDirectory).mockResolvedValue(undefined)
sftpCapture.paths.length = 0
for (const key of Object.keys(sftpCapture.contents)) {
delete sftpCapture.contents[key]
}
for (const key of Object.keys(sftpCapture.execCallCountAtWrite)) {
delete sftpCapture.execCallCountAtWrite[key]
}
vi.mocked(parseUnameToRelayPlatform).mockReturnValue('linux-x64')
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(false)
})
function feed(execResponses: ExecResponse[]): void {
const mockExec = vi.mocked(execCommand)
for (const response of execResponses) {
if (typeof response === 'string') {
mockExec.mockResolvedValueOnce(response)
} else {
mockExec.mockRejectedValueOnce(new Error(response.reject))
}
}
}
it('writes a hardcoded package.json BEFORE running npm install', async () => {
const conn = makeMockConnection(sftpCapture)
feed(makeExecResponses({ npmInstall: 'ok', probe: 'ok' }))
await deployAndLaunchRelay(conn)
const pkgPath = sftpCapture.paths.find((path) => path.endsWith('/package.json'))
expect(pkgPath, 'package.json must be written via SFTP').toBeTruthy()
const written = sftpCapture.contents[pkgPath as string]
expect(written).toBeTruthy()
const parsed = JSON.parse(written) as Record<string, unknown>
expect(parsed.name).toBe('orca-relay')
expect(parsed.version).toBe('1.0.0')
expect(parsed.private).toBe(true)
expect(parsed.type).toBe('commonjs')
expect(parsed.dependencies).toEqual({ '@parcel/watcher': '2.5.6', 'node-pty': '1.1.0' })
expect(parsed.allowScripts).toEqual({
'@parcel/watcher@2.5.6': true,
'node-pty@1.1.0': true
})
const execCalls = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
const npmInstallIdx = execCalls.findIndex(
(command) =>
command.includes('npm install') &&
command.includes('node-pty') &&
command.includes('@parcel/watcher')
)
expect(npmInstallIdx).toBeGreaterThanOrEqual(0)
expect(execCalls[npmInstallIdx]).toContain('--ignore-scripts=false')
const writeObservedAt = sftpCapture.execCallCountAtWrite[pkgPath as string]
expect(writeObservedAt).toBeLessThanOrEqual(npmInstallIdx)
})
it('promotes only after the first-install lock is acquired', async () => {
const conn = makeMockConnection(sftpCapture)
feed(makeExecResponses({ npmInstall: 'ok', probe: 'ok' }))
vi.mocked(acquireInstallLock).mockImplementationOnce(async () => {
const commands = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
expect(commands.some((command) => command.includes('cp -a'))).toBe(false)
})
await deployAndLaunchRelay(conn)
const commands = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
const promotionIndex = commands.findIndex((command) => command.includes('cp -a'))
const npmIndex = commands.findIndex((command) => command.includes('npm install'))
expect(promotionIndex).toBeGreaterThanOrEqual(0)
expect(npmIndex).toBeGreaterThan(promotionIndex)
})
it('cleans a staged upload when cancellation wins before lock acquisition', async () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection(sftpCapture)
feed(makeStagedFirstInstallExecPrefix())
vi.mocked(acquireInstallLock).mockImplementationOnce(
(_conn, _remoteDir, _host, options) =>
new Promise<void>((_resolve, reject) => {
options?.signal?.addEventListener('abort', () => reject(options.signal?.reason), {
once: true
})
})
)
vi.mocked(execCommand).mockResolvedValueOnce('')
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
const result = await deploy
expect(result).toBeInstanceOf(Error)
expect(vi.mocked(acquireInstallLock)).toHaveBeenCalledTimes(1)
const commands = vi.mocked(execCommand).mock.calls.map(([, command]) => command)
expect(commands.some((command) => command.includes('cp -a'))).toBe(false)
expect(commands.some((command) => command.includes('rm -rf'))).toBe(true)
} finally {
vi.useRealTimers()
}
})
it('retains the lock after an unconfirmed promotion termination', async () => {
const conn = makeMockConnection(sftpCapture)
vi.mocked(execCommand)
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
.mockResolvedValueOnce('/home/u')
.mockResolvedValueOnce('')
.mockResolvedValueOnce(
'__ORCA_UPLOAD_STAGE_SLOT__.sftp-namespace-00000000000000000000000000000000:slot-0'
)
.mockResolvedValueOnce('')
.mockResolvedValueOnce('')
.mockRejectedValueOnce(
Object.assign(new Error('promotion termination was not confirmed'), {
sshChannelCloseConfirmed: false
})
)
.mockResolvedValueOnce('')
await expect(deployAndLaunchRelay(conn)).rejects.toThrow(
'promotion termination was not confirmed'
)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
})
})
@@ -1,6 +1,7 @@
// Why: regression coverage for the install-probe contract — the "node-pty is not available" bug shipped because every guard layer was silent.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as RelayInstallMarkerModule from './ssh-relay-install-marker'
vi.mock('electron', () => ({
app: { getAppPath: () => '/mock/app' }
@@ -36,6 +37,11 @@ vi.mock('./ssh-remote-node-resolution', () => ({
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
}))
vi.mock('./ssh-relay-install-marker', async (importOriginal) => ({
...(await importOriginal<typeof RelayInstallMarkerModule>()),
createRelayInstallMarkerFileName: () => '.sftp-namespace-00000000000000000000000000000000'
}))
vi.mock('./ssh-relay-versioned-install', () => ({
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+testhash'),
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
@@ -81,6 +87,7 @@ import {
makeExecResponses,
makeMockConnection,
makeRepairToolchainSkipExecResponses,
makeStagedFirstInstallExecPrefix,
type ExecResponse,
type SftpWriteCapture
} from './ssh-relay-native-deps-install-fixture'
@@ -96,7 +103,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
beforeEach(() => {
vi.clearAllMocks()
// mockReset because clearAllMocks keeps queued mockResolvedValueOnce entries, so a leaked response would bleed into the next test.
vi.mocked(execCommand).mockReset()
vi.mocked(execCommand).mockReset().mockResolvedValue('')
vi.mocked(uploadDirectory).mockResolvedValue(undefined)
sftpCapture.paths.length = 0
for (const k of Object.keys(sftpCapture.contents)) {
@@ -126,40 +133,6 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
}
}
it('writes a hardcoded package.json BEFORE running npm install', async () => {
const conn = makeMockConnection(sftpCapture)
feed(makeExecResponses({ npmInstall: 'ok', probe: 'ok' }))
await deployAndLaunchRelay(conn)
const pkgPath = sftpCapture.paths.find((p) => p.endsWith('/package.json'))
expect(pkgPath, 'package.json must be written via SFTP').toBeTruthy()
const written = sftpCapture.contents[pkgPath as string]
expect(written).toBeTruthy()
const parsed = JSON.parse(written) as Record<string, unknown>
expect(parsed.name).toBe('orca-relay')
expect(parsed.version).toBe('1.0.0')
expect(parsed.private).toBe(true)
// Why: pin commonjs so a future Node default flip can't break require('node-pty').
expect(parsed.type).toBe('commonjs')
expect(parsed.dependencies).toEqual({ '@parcel/watcher': '2.5.6', 'node-pty': '1.1.0' })
expect(parsed.allowScripts).toEqual({
'@parcel/watcher@2.5.6': true,
'node-pty@1.1.0': true
})
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
const npmInstallIdx = execCalls.findIndex(
(c) => c.includes('npm install') && c.includes('node-pty') && c.includes('@parcel/watcher')
)
expect(npmInstallIdx).toBeGreaterThanOrEqual(0)
expect(execCalls[npmInstallIdx]).toContain('--ignore-scripts=false')
// Pin write-before-install ordering to catch a Promise.all refactor where the final-state assertions above still pass.
const writeObservedAt = sftpCapture.execCallCountAtWrite[pkgPath as string]
expect(writeObservedAt).toBeLessThanOrEqual(npmInstallIdx)
})
it('propagates a hard `npm install` failure so the deploy aborts before finalizeInstall', async () => {
const conn = makeMockConnection(sftpCapture)
feed(
@@ -366,10 +339,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
// Why: rebuild degrades gracefully, but the post-rebuild probe must surface transport death, else a dead channel finalizes a half-repaired install.
const conn = makeMockConnection(sftpCapture)
feed([
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // uname
'/home/u', // $HOME
'', // mkdir remoteDir (uploadRelay)
'', // chmod +x node
...makeStagedFirstInstallExecPrefix(),
'', // npm install native deps
'', // chmod prebuilds
'MISSING\n', // first probe: require() fails
@@ -393,12 +363,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection(sftpCapture)
feed([
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir
'' // chmod +x node
])
feed(makeStagedFirstInstallExecPrefix())
let installSignal: AbortSignal | undefined
vi.mocked(execCommand).mockImplementationOnce((_conn, command, options) => {
expect(command).toContain('npm install')
@@ -560,16 +525,19 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
feed([
'__ORCA_REMOTE_PLATFORM__ Windows AMD64',
'C:\\Users\\u',
'', // mkdir remoteDir
'', // bounded stale-stage recovery
'__ORCA_UPLOAD_STAGE_SLOT__.sftp-namespace-00000000000000000000000000000000:slot-0',
'__ORCA_UPLOAD_STAGE_PROMOTION__.sftp-namespace-00000000000000000000000000000000:PROMOTED',
'', // npm install native deps
'MISSING\n', // native process exit normalized by PowerShell command
'', // npm rebuild native deps
'MISSING\n', // rebuilt native process still cannot load
'', // clean stage root
'', // no persisted active pipe marker
'WAITING',
'WAITING', // initial pipe probe
'', // publish the per-launch credential
'', // WMI relay launch
'READY',
'READY', // readiness poll
'' // persist active pipe marker
])
@@ -637,7 +605,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
for (const k of Object.keys(sftpCapture.execCallCountAtWrite)) {
delete sftpCapture.execCallCountAtWrite[k]
}
vi.mocked(execCommand).mockReset()
vi.mocked(execCommand).mockReset().mockResolvedValue('')
const conn2 = makeMockConnection(sftpCapture)
feed(makeExecResponses({ npmInstall: 'ok', probe: 'ok' }))
@@ -756,12 +724,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
vi.useFakeTimers()
try {
const conn = makeMockConnection(sftpCapture)
feed([
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir
'' // chmod +x node
])
feed(makeStagedFirstInstallExecPrefix())
let installSignal: AbortSignal | undefined
vi.mocked(execCommand).mockImplementationOnce((_conn, command, options) => {
expect(command).toContain('npm install')
@@ -789,11 +752,10 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.waitFor(() => expect(installSignal).toBeDefined())
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
await vi.advanceTimersByTimeAsync(5_000)
const result = await deploy
expect(result).toBeInstanceOf(Error)
expect((result as Error).message).toContain('Relay deployment timed out')
await vi.advanceTimersByTimeAsync(5_000)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
} finally {
@@ -803,21 +765,19 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
it('does not finalize or release a first-install lock after unconfirmed rebuild teardown', async () => {
const conn = makeMockConnection(sftpCapture)
vi.mocked(execCommand)
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
.mockResolvedValueOnce('/home/u')
.mockResolvedValueOnce('')
.mockResolvedValueOnce('')
.mockResolvedValueOnce('')
.mockResolvedValueOnce('')
.mockResolvedValueOnce('MISSING')
.mockResolvedValueOnce('rebuild diagnostics')
.mockResolvedValueOnce('')
.mockRejectedValueOnce(
Object.assign(new Error('rebuild termination was not confirmed'), {
sshChannelCloseConfirmed: false
})
)
feed([
...makeStagedFirstInstallExecPrefix(),
'', // npm install
'', // chmod prebuilds
'MISSING',
'rebuild diagnostics',
'' // remove probe diagnostics
])
vi.mocked(execCommand).mockRejectedValueOnce(
Object.assign(new Error('rebuild termination was not confirmed'), {
sshChannelCloseConfirmed: false
})
)
await expect(deployAndLaunchRelay(conn)).rejects.toThrow(
'rebuild termination was not confirmed'
@@ -831,10 +791,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
try {
const conn = makeMockConnection(sftpCapture)
feed([
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
'/home/u',
'', // mkdir remoteDir
'', // chmod +x node
...makeStagedFirstInstallExecPrefix(),
'', // npm install
'', // chmod prebuilds
'MISSING',
@@ -867,11 +824,10 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.waitFor(() => expect(rebuildSignal).toBeDefined())
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
await vi.advanceTimersByTimeAsync(5_000)
const result = await deploy
expect(result).toBeInstanceOf(Error)
expect((result as Error).message).toContain('Relay deployment timed out')
await vi.advanceTimersByTimeAsync(5_000)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
} finally {
@@ -5,6 +5,7 @@
import { EventEmitter } from 'node:events'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as RelayInstallMarkerModule from './ssh-relay-install-marker'
vi.mock('electron', () => ({
app: { getAppPath: () => '/mock/app' }
@@ -40,6 +41,11 @@ vi.mock('./ssh-remote-node-resolution', () => ({
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
}))
vi.mock('./ssh-relay-install-marker', async (importOriginal) => ({
...(await importOriginal<typeof RelayInstallMarkerModule>()),
createRelayInstallMarkerFileName: () => '.sftp-namespace-00000000000000000000000000000000'
}))
vi.mock('./ssh-relay-versioned-install', () => ({
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+testhash'),
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
@@ -74,7 +80,10 @@ vi.mock('./ssh-connection-utils', () => ({
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { execCommand, uploadDirectory } from './ssh-relay-deploy-helpers'
import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
import {
RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS,
RELAY_DEPLOY_TIMEOUT_MS
} from './ssh-relay-deploy-timing'
import { parseUnameToRelayPlatform } from './relay-protocol'
import {
abandonInstall,
@@ -94,6 +103,9 @@ const RELAY_SUFFIX = '.orca-remote/relay-0.1.0+testhash'
const SHELL_RELAY_DIR = `${SHELL_HOME}/${RELAY_SUFFIX}`
const SFTP_RELAY_DIR = `${SFTP_HOME}/${RELAY_SUFFIX}`
const MARKER_PATTERN = /\.sftp-namespace-[0-9a-f]{32}/
const STAGE_OWNER = '.sftp-namespace-00000000000000000000000000000000'
const STAGE_RESERVED = `__ORCA_UPLOAD_STAGE_SLOT__${STAGE_OWNER}:slot-0`
const STAGE_PROMOTED = `__ORCA_UPLOAD_STAGE_PROMOTION__${STAGE_OWNER}:PROMOTED`
type ConnectionOptions = {
// '/homes/u' models a DSM host whose SFTP subsystem starts outside the shell home.
@@ -130,14 +142,15 @@ function newCapture(): Capture {
}
// The marker this run created, read back from the shell command that made it.
function issuedMarkerName(): string | undefined {
function issuedMarkerNames(): string[] {
const markers: string[] = []
for (const [, command] of vi.mocked(execCommand).mock.calls) {
const match = decodeCommand(command).match(MARKER_PATTERN)
if (match) {
return match[0]
markers.push(match[0])
}
}
return undefined
return markers
}
function decodeCommand(command: string): string {
@@ -155,7 +168,8 @@ function makeConnection(capture: Capture, options: ConnectionOptions = {}): SshC
const lstatPresent =
options.lstatPresent ??
((path: string) =>
path.startsWith(`${SFTP_HOME}/`) && path.includes(issuedMarkerName() ?? '\0'))
path.startsWith(`${SFTP_HOME}/`) &&
issuedMarkerNames().some((marker) => path.includes(marker)))
const makeSftp = (): unknown => {
const sftp = new EventEmitter()
@@ -231,12 +245,33 @@ function feed(responses: string[]): void {
const POSIX_FIRST_INSTALL = [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
SHELL_HOME,
'', // mkdir remoteDir (+ install-owner marker)
'', // chmod +x node
'', // bounded stale-stage recovery
STAGE_RESERVED,
'', // chmod staged node
'', // final install namespace marker
STAGE_PROMOTED,
'', // npm install native deps
'', // chmod prebuilds
'ORCA-NPTY-PROBE-OK\n',
'', // rm probe stderr
'', // clean stage root
'DEAD',
'', // publish the per-launch credential
'READY'
]
const POSIX_SYSTEM_SSH_FIRST_INSTALL = [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
SHELL_HOME,
'', // bounded stale-stage recovery
STAGE_RESERVED,
'', // chmod staged node
STAGE_PROMOTED,
'', // npm install native deps
'', // chmod prebuilds
'ORCA-NPTY-PROBE-OK\n',
'', // rm probe stderr
'', // clean stage root
'DEAD',
'', // publish the per-launch credential
'READY'
@@ -274,7 +309,7 @@ describe('relay install writes on a split SFTP namespace', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(execCommand).mockReset()
vi.mocked(execCommand).mockReset().mockResolvedValue('')
vi.mocked(uploadDirectory).mockImplementation((_sftp, _local, remote: string) => {
capture.uploadTargets.push(remote)
return Promise.resolve()
@@ -298,9 +333,10 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
expect(capture.uploadTargets).toEqual([SFTP_RELAY_DIR])
expect(capture.uploadTargets).toHaveLength(1)
expect(capture.uploadTargets[0]).toBe('/homes/u/.orca-remote/.upload-stages/slot-0/payload')
expect(capture.writePaths).toEqual([
`${SFTP_RELAY_DIR}/.version`,
`${capture.uploadTargets[0]}/.version`,
`${SFTP_RELAY_DIR}/package.json`
])
// Every shell command — mkdir, chmod, npm, launch — still names the shell path.
@@ -317,9 +353,14 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
const markerCommands = execCommands().filter((command) => MARKER_PATTERN.test(command))
expect(markerCommands).toHaveLength(1)
expect(markerCommands[0]).toContain('mkdir')
expect(markerCommands[0]).toContain(`${SHELL_RELAY_DIR}/.install-lock`)
const reservation = markerCommands.find((command) =>
command.includes('__ORCA_UPLOAD_STAGE_SLOT__')
)
const installMarker = markerCommands.find((command) => command.includes('.install-lock'))
expect(reservation).toContain('mkdir')
expect(reservation).toContain('/.orca-remote/.upload-stages')
expect(reservation).not.toContain('.install-lock')
expect(installMarker).toContain(`${SHELL_RELAY_DIR}/.install-lock`)
})
it('probes one shared marker for every first-install artifact transfer', async () => {
@@ -328,15 +369,17 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
const marker = issuedMarkerName()
expect(marker).toMatch(MARKER_PATTERN)
expect(capture.lstatCalls).toEqual([
`${SHELL_RELAY_DIR}/.install-lock/${marker}`,
`${SFTP_RELAY_DIR}/.install-lock/${marker}`,
`${SHELL_RELAY_DIR}/.install-lock/${marker}`,
`${SFTP_RELAY_DIR}/.install-lock/${marker}`,
`${SHELL_RELAY_DIR}/.install-lock/${marker}`,
`${SFTP_RELAY_DIR}/.install-lock/${marker}`
const [stageMarker, installMarker] = issuedMarkerNames()
expect(stageMarker).toMatch(MARKER_PATTERN)
expect(installMarker).toMatch(MARKER_PATTERN)
expect(capture.lstatCalls).toHaveLength(6)
for (const path of capture.lstatCalls.slice(0, 4)) {
expect(path).toContain('.upload-stages')
expect(path).toContain(stageMarker)
}
expect(capture.lstatCalls.slice(4)).toEqual([
`${SHELL_RELAY_DIR}/.install-lock/${installMarker}`,
`${SFTP_RELAY_DIR}/.install-lock/${installMarker}`
])
})
@@ -349,9 +392,10 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
expect(capture.uploadTargets).toEqual([SHELL_RELAY_DIR])
expect(capture.uploadTargets[0]).toContain('/.orca-remote/.upload-stages/slot-0/payload')
expect(capture.uploadTargets[0]).toMatch(/\/payload$/)
expect(capture.writePaths).toEqual([
`${SHELL_RELAY_DIR}/.version`,
`${capture.uploadTargets[0]}/.version`,
`${SHELL_RELAY_DIR}/package.json`
])
})
@@ -362,7 +406,8 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
expect(capture.uploadTargets).toEqual([SHELL_RELAY_DIR])
expect(capture.uploadTargets[0]).toContain('/.orca-remote/.upload-stages/slot-0/payload')
expect(capture.uploadTargets[0]).toMatch(/\/payload$/)
expect(capture.lstatCalls).toEqual([])
})
@@ -375,7 +420,10 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
expect(capture.uploadTargets).toEqual([`/volume1/shared/${RELAY_SUFFIX}`])
expect(capture.uploadTargets[0]).toContain(
'/volume1/shared/.orca-remote/.upload-stages/slot-0/payload'
)
expect(capture.uploadTargets[0]).toMatch(/\/payload$/)
})
it('passes the same mapping to a connection that owns its own transfer methods', async () => {
@@ -385,43 +433,43 @@ describe('relay install writes on a split SFTP namespace', () => {
await deployAndLaunchRelay(conn)
// The shipping path hands over shell paths plus a mapping; resolution happens on the write session.
expect(capture.uploadTargets).toEqual([SHELL_RELAY_DIR])
expect(capture.uploadTargets[0]).toContain('/.orca-remote/.upload-stages/slot-0/payload')
expect(capture.uploadTargets[0]).toMatch(/\/payload$/)
expect(capture.writePaths).toEqual([
`${SHELL_RELAY_DIR}/.version`,
`${capture.uploadTargets[0]}/.version`,
`${SHELL_RELAY_DIR}/package.json`
])
const marker = issuedMarkerName()
const [stageMarker, installMarker] = issuedMarkerNames()
const mappings = [...capture.uploadOptions, ...capture.writeOptions].map(
(options) => options?.sftpNamespace
)
expect(mappings).toHaveLength(3)
for (const mapping of mappings) {
expect(mapping?.shellProbePath).toBe(`${SHELL_RELAY_DIR}/.install-lock/${marker}`)
expect(mapping?.homeRelativeProbePath).toBe(`${RELAY_SUFFIX}/.install-lock/${marker}`)
}
expect(mappings.map((mapping) => mapping?.homeRelativePath)).toEqual([
RELAY_SUFFIX,
`${RELAY_SUFFIX}/.version`,
`${RELAY_SUFFIX}/package.json`
])
expect(mappings[0]?.shellProbePath).toContain('/.orca-remote/.upload-stages/slot-0/')
expect(mappings[0]?.shellProbePath).toContain(stageMarker)
expect(mappings[1]?.shellProbePath).toBe(mappings[0]?.shellProbePath)
expect(mappings[2]?.shellProbePath).toBe(`${SHELL_RELAY_DIR}/.install-lock/${installMarker}`)
expect(mappings[0]?.homeRelativePath).toBe('.orca-remote/.upload-stages/slot-0/payload')
expect(mappings[1]?.homeRelativePath).toBe(`${mappings[0]?.homeRelativePath}/.version`)
expect(mappings[2]?.homeRelativePath).toBe(`${RELAY_SUFFIX}/package.json`)
expect(capture.realpathCalls).toEqual([])
})
it('leaves system-SSH connections unmapped and unprobed', async () => {
// transferMethods models the shipping SshConnection path (uploadDirectory/writeFile → system SSH helpers).
const conn = makeConnection(capture, { systemSsh: true, transferMethods: true })
feed(POSIX_FIRST_INSTALL)
feed(POSIX_SYSTEM_SSH_FIRST_INSTALL)
await deployAndLaunchRelay(conn)
expect(execCommands().some((command) => MARKER_PATTERN.test(command))).toBe(false)
expect(execCommands().some((command) => MARKER_PATTERN.test(command))).toBe(true)
expect(capture.realpathCalls).toEqual([])
expect(capture.lstatCalls).toEqual([])
expect(conn.sftp).not.toHaveBeenCalled()
// System SSH never retargets: shell absolute paths, no mapping, no SFTP session.
expect(capture.uploadTargets).toEqual([SHELL_RELAY_DIR])
expect(capture.uploadTargets[0]).toContain('/.orca-remote/.upload-stages/slot-0/payload')
expect(capture.uploadTargets[0]).toMatch(/\/payload$/)
expect(capture.writePaths).toEqual([
`${SHELL_RELAY_DIR}/.version`,
`${capture.uploadTargets[0]}/.version`,
`${SHELL_RELAY_DIR}/package.json`
])
const transferOptions = [...capture.uploadOptions, ...capture.writeOptions]
@@ -434,22 +482,34 @@ describe('relay install writes on a split SFTP namespace', () => {
it('leaves Windows hosts unmapped and unprobed', async () => {
vi.mocked(parseUnameToRelayPlatform).mockReturnValue('win32-x64')
const conn = makeConnection(capture)
feed([
'__ORCA_REMOTE_PLATFORM__ Windows AMD64',
'C:\\Users\\u',
'' // mkdir remoteDir
])
// Fail the install right after the package.json write; the launch path is not what this asserts.
vi.mocked(execCommand).mockRejectedValueOnce(new Error('npm install failed'))
vi.mocked(execCommand).mockImplementation((_conn, command) => {
const decoded = decodeCommand(command)
if (decoded.includes('__ORCA_REMOTE_PLATFORM__')) {
return Promise.resolve('__ORCA_REMOTE_PLATFORM__ Windows AMD64')
}
if (decoded.includes('[Environment]::GetFolderPath')) {
return Promise.resolve('C:\\Users\\u')
}
if (decoded.includes('__ORCA_UPLOAD_STAGE_SLOT__')) {
return Promise.resolve(STAGE_RESERVED)
}
if (decoded.includes('__ORCA_UPLOAD_STAGE_PROMOTION__')) {
return Promise.resolve(STAGE_PROMOTED)
}
if (decoded.includes('npm install')) {
return Promise.reject(new Error('npm install failed'))
}
return Promise.resolve('')
})
await expect(deployAndLaunchRelay(conn)).rejects.toThrow('npm install failed')
expect(execCommands().some((command) => MARKER_PATTERN.test(command))).toBe(false)
expect(execCommands().some((command) => MARKER_PATTERN.test(command))).toBe(true)
expect(capture.realpathCalls).toEqual([])
expect(capture.writePaths).toEqual([
'C:/Users/u/.orca-remote/relay-0.1.0+testhash/.version',
'C:/Users/u/.orca-remote/relay-0.1.0+testhash/package.json'
])
expect(capture.writePaths[0]).toBe(
'C:/Users/u/.orca-remote/.upload-stages/slot-0/payload/.version'
)
expect(capture.writePaths[1]).toBe('C:/Users/u/.orca-remote/relay-0.1.0+testhash/package.json')
})
it('releases the first-install lock when a redirected upload fails', async () => {
@@ -459,7 +519,7 @@ describe('relay install writes on a split SFTP namespace', () => {
await expect(deployAndLaunchRelay(conn)).rejects.toThrow('sftp write failed')
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
})
@@ -472,13 +532,13 @@ describe('relay install writes on a split SFTP namespace', () => {
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.waitFor(() => expect(capture.realpathCalls).toHaveLength(1))
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS)
const result = await deploy
expect((result as Error).message).toContain('Relay deployment timed out')
await vi.advanceTimersByTimeAsync(5_000)
expect(capture.sftpEndCalls).toBe(1)
// A confirmed close releases the first-install lock and leaves the dir incomplete.
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
} finally {
vi.useRealTimers()
@@ -494,8 +554,8 @@ describe('relay install writes on a split SFTP namespace', () => {
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
await vi.waitFor(() => expect(capture.realpathCalls).toHaveLength(1))
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TEARDOWN_TIMEOUT_MS)
await deploy
await vi.advanceTimersByTimeAsync(5_000)
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
@@ -511,7 +571,7 @@ describe('relay repair writes on a split SFTP namespace', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(execCommand).mockReset()
vi.mocked(execCommand).mockReset().mockResolvedValue('')
vi.mocked(uploadDirectory).mockResolvedValue(undefined)
vi.mocked(parseUnameToRelayPlatform).mockReturnValue('linux-x64')
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true)
@@ -0,0 +1,422 @@
import { execFileSync, spawnSync } from 'node:child_process'
import { randomUUID } from 'node:crypto'
import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } }))
import { SshChannelMultiplexer } from './ssh-channel-multiplexer'
import { SshConnection } from './ssh-connection'
import { execCommand } from './ssh-relay-deploy-helpers'
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { acquireInstallLock } from './ssh-relay-install-lock'
import { getRemoteHostPlatform } from './ssh-remote-platform'
import { listRelayBaseDirsCommand } from './ssh-remote-commands'
import { gcOldRelayVersions } from './ssh-relay-versioned-install'
import type { SshTarget } from '../../shared/ssh-types'
const RUN_REVIEW_ORACLE = process.env.ORCA_REVIEW_SSH_UPLOAD_CANCEL === '1'
const REMOTE_REPO = '/tmp/orca-pr-10207-real-repo'
type TargetFixture = {
containerName: string
identityFile: string
port: number
tempDir: string
}
type RemoteInventory = {
installLock: boolean
payloadFiles: number
uploadStages: string[]
}
function run(command: string, args: string[], timeout = 30_000): string {
return execFileSync(command, args, {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout
}).trim()
}
function shellQuote(value: string): string {
return `'${value.replaceAll("'", "'\\''")}'`
}
function dockerExec(fixture: TargetFixture, command: string): string {
return run('docker', ['exec', fixture.containerName, 'bash', '-lc', command], 60_000)
}
function startTarget(): TargetFixture {
const image = process.env.ORCA_REVIEW_SSH_IMAGE
if (!image) {
throw new Error('ORCA_REVIEW_SSH_IMAGE is required')
}
const tempDir = mkdtempSync(join(tmpdir(), 'orca-pr10207-ssh-'))
const identityFile = join(tempDir, 'id_ed25519')
run('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', identityFile, '-q'])
const publicKey = readFileSync(`${identityFile}.pub`, 'utf8').trim()
const containerName = `orca-pr10207-${randomUUID().slice(0, 12)}`
run(
'docker',
[
'run',
'-d',
'--name',
containerName,
'-p',
'0.0.0.0::22',
'-e',
`AUTHORIZED_KEY=${publicKey}`,
image,
'bash',
'-lc',
'printf "%s\\n" "$AUTHORIZED_KEY" > /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys && exec /usr/sbin/sshd -D -e'
],
120_000
)
const port = Number(run('docker', ['port', containerName, '22/tcp']).split(':').at(-1))
const fixture = { containerName, identityFile, port, tempDir }
dockerExec(
fixture,
[
`mkdir -p ${shellQuote(REMOTE_REPO)}`,
`cd ${shellQuote(REMOTE_REPO)}`,
'git init',
'git config user.email review@test.local',
'git config user.name "PR 10207 Review"',
"printf '%s\\n' real-docker-ssh-repository > README.md",
'git add README.md',
'git commit -m initial'
].join(' && ')
)
return fixture
}
function stopTarget(fixture: TargetFixture | null): void {
if (!fixture) {
return
}
spawnSync('docker', ['stop', fixture.containerName], { stdio: 'ignore', timeout: 20_000 })
spawnSync('docker', ['rm', fixture.containerName], { stdio: 'ignore', timeout: 20_000 })
rmSync(fixture.tempDir, { recursive: true, force: true })
}
function createConnection(fixture: TargetFixture): SshConnection {
const host = process.env.ORCA_REVIEW_SSH_TARGET_HOST ?? ''
if (!host || host === 'localhost' || host === '::1' || host.startsWith('127.')) {
throw new Error(`Review SSH target must be non-loopback, received ${JSON.stringify(host)}`)
}
const target: SshTarget = {
id: `pr-10207-${randomUUID()}`,
label: 'PR 10207 Docker SSH target',
source: 'manual',
host,
port: fixture.port,
username: 'root',
identityFile: fixture.identityFile,
identitiesOnly: true
}
return new SshConnection(target, { onStateChange: vi.fn() })
}
function readInventory(fixture: TargetFixture, remoteRelayDir: string): RemoteInventory {
const stagePool = '/root/.orca-remote/.upload-stages'
const raw = dockerExec(
fixture,
[
`lock=0; test -d ${shellQuote(`${remoteRelayDir}/.install-lock`)} && lock=1`,
`files=0; test -d ${shellQuote(remoteRelayDir)} && files=$(find ${shellQuote(remoteRelayDir)} -type f ! -path '*/.install-lock/*' | wc -l | tr -d ' ')`,
`printf 'LOCK=%s\\nFILES=%s\\n' "$lock" "$files"`,
`find ${shellQuote(stagePool)} -mindepth 1 -maxdepth 1 \\( -name 'slot-*' -o -name 'claim-*' -o -name 'delete-*' \\) -print 2>/dev/null | sort || true`
].join('; ')
)
const lines = raw.split(/\r?\n/)
return {
installLock: lines[0] === 'LOCK=1',
payloadFiles: Number(lines[1]?.slice('FILES='.length) ?? 0),
uploadStages: lines.slice(2).filter(Boolean)
}
}
describe.skipIf(!RUN_REVIEW_ORACLE)('SSH relay upload cancellation recovery', () => {
let fixture: TargetFixture | null = null
beforeAll(() => {
fixture = startTarget()
})
afterAll(() => {
stopTarget(fixture)
})
it('aborts a live SFTP upload after remote bytes arrive without creating the shared lock', async () => {
const activeFixture = fixture as TargetFixture
const localRelayDir = join(process.cwd(), 'out', 'relay', 'linux-arm64')
const relayVersion = readFileSync(join(localRelayDir, '.version'), 'utf8').trim()
const relayJsSize = statSync(join(localRelayDir, 'relay.js')).size
const remoteRelayDir = `/root/.orca-remote/relay-${relayVersion}`
const connection = createConnection(activeFixture)
await connection.connect()
const sentinelPid = dockerExec(activeFixture, 'sleep 300 </dev/null >/dev/null 2>&1 & echo $!')
let releaseFirstWrite: () => void = () => {}
const firstWrite = new Promise<void>((resolve) => {
releaseFirstWrite = resolve
})
let acknowledgedBytes = 0
const openSftp = connection.sftp.bind(connection)
connection.sftp = vi.fn(async (signal) => {
const sftp = await openSftp(signal)
const createWriteStream = sftp.createWriteStream.bind(sftp)
sftp.createWriteStream = ((...args: Parameters<typeof createWriteStream>) => {
const [remotePath] = args
const stream = createWriteStream(...args)
if (!remotePath.endsWith('/payload/relay.js')) {
return stream
}
const writable = stream as typeof stream & {
_write: (
chunk: Buffer,
encoding: BufferEncoding,
callback: (error?: Error | null) => void
) => void
}
const write = writable._write.bind(writable)
writable._write = (chunk, encoding, callback): void => {
write(chunk, encoding, (error) => {
if (error) {
callback(error)
return
}
acknowledgedBytes += chunk.length
releaseFirstWrite()
})
}
return stream
}) as typeof sftp.createWriteStream
return sftp
})
const deployment = deployAndLaunchRelay(connection, undefined, 60)
let barrierTimeout: ReturnType<typeof setTimeout>
try {
await Promise.race([
firstWrite,
new Promise<never>((_resolve, reject) => {
barrierTimeout = setTimeout(
() => reject(new Error('Timed out waiting for first remote SFTP chunk')),
30_000
)
})
])
} finally {
clearTimeout(barrierTimeout!)
}
const partialRemoteBytes = Number(
dockerExec(
activeFixture,
"find /root/.orca-remote/.upload-stages -type f -path '*/slot-*/payload/relay.js' -printf '%s\\n'"
)
)
const operationController = (
connection as unknown as { systemOperationAbortController: AbortController }
).systemOperationAbortController
operationController.abort()
await expect(deployment).rejects.toMatchObject({ name: 'AbortError' })
const inventory = readInventory(activeFixture, remoteRelayDir)
const sentinelCommand = dockerExec(
activeFixture,
`tr '\\0' ' ' < /proc/${shellQuote(sentinelPid)}/cmdline`
)
await connection.disconnect()
console.log(
`[pr-10207-live-sftp-abort] ${JSON.stringify({ acknowledgedBytes, partialRemoteBytes, relayJsSize, inventory, sentinelPid, sentinelCommand })}`
)
expect(acknowledgedBytes).toBeGreaterThan(0)
expect(partialRemoteBytes).toBe(acknowledgedBytes)
expect(partialRemoteBytes).toBeLessThan(relayJsSize)
expect(inventory.installLock).toBe(false)
expect(sentinelCommand).toContain('sleep 300')
}, 180_000)
it('recovers cancellation with bounded safe reclamation and bounded real version GC', async () => {
const activeFixture = fixture as TargetFixture
const relayVersion = readFileSync(
join(process.cwd(), 'out', 'relay', 'linux-arm64', '.version'),
'utf8'
).trim()
const remoteRelayDir = `/root/.orca-remote/relay-${relayVersion}`
const firstConnection = createConnection(activeFixture)
const progress: string[] = []
await firstConnection.connect()
const unconfirmedCancellation = Object.assign(new Error('injected upload cancellation'), {
sshChannelCloseConfirmed: false
})
firstConnection.uploadDirectory = vi.fn().mockRejectedValue(unconfirmedCancellation)
await expect(
deployAndLaunchRelay(firstConnection, (status) => progress.push(status), 60)
).rejects.toBe(unconfirmedCancellation)
await firstConnection.disconnect()
const firstInventory = readInventory(activeFixture, remoteRelayDir)
const expected = process.env.ORCA_REVIEW_EXPECT_RECOVERY === '1' ? 'recovered' : 'blocked'
if (expected === 'recovered') {
const secondAbandonedConnection = createConnection(activeFixture)
await secondAbandonedConnection.connect()
secondAbandonedConnection.uploadDirectory = vi.fn().mockRejectedValue(unconfirmedCancellation)
await expect(deployAndLaunchRelay(secondAbandonedConnection, undefined, 60)).rejects.toBe(
unconfirmedCancellation
)
await secondAbandonedConnection.disconnect()
}
const retryConnection = createConnection(activeFixture)
await retryConnection.connect()
let retryResult: 'blocked' | 'recovered'
let finalInventory: RemoteInventory | undefined
let scaleEvidence:
| { gcDurationMs: number; listingBytes: number; scaleEntries: number }
| undefined
if (firstInventory.installLock) {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), 1_500)
await expect(
acquireInstallLock(retryConnection, remoteRelayDir, getRemoteHostPlatform('linux-arm64'), {
signal: controller.signal
})
).rejects.toMatchObject({ name: 'AbortError' })
clearTimeout(timer)
retryResult = 'blocked'
} else {
const deployed = await deployAndLaunchRelay(retryConnection, undefined, 60)
const mux = new SshChannelMultiplexer(deployed.transport)
await expect(mux.request('session.resolveHome', { path: '~' })).resolves.toEqual({
resolvedPath: '/root'
})
mux.dispose()
retryResult = 'recovered'
}
const repoHead = await execCommand(
retryConnection,
`cd ${shellQuote(REMOTE_REPO)} && git rev-parse --verify HEAD`
)
await retryConnection.disconnect()
if (expected === 'recovered') {
const replacedStage = firstInventory.uploadStages[0]!
const originalStage = `${replacedStage}.original`
const foreignTarget = '/root/orca-pr10207-foreign-stage-target'
const stagePool = '/root/.orca-remote/.upload-stages'
const symlinkStage = `${stagePool}/slot-2`
const ownerMarker = '.orca-upload-owner'
const identityMarker = '.orca-upload-identity'
dockerExec(
activeFixture,
[
`mv ${shellQuote(replacedStage)} ${shellQuote(originalStage)}`,
`mkdir -p ${shellQuote(`${replacedStage}/payload`)} ${shellQuote(foreignTarget)}`,
`cp ${shellQuote(`${originalStage}/${ownerMarker}`)} ${shellQuote(`${replacedStage}/${ownerMarker}`)}`,
`cp ${shellQuote(`${originalStage}/${identityMarker}`)} ${shellQuote(`${replacedStage}/${identityMarker}`)}`,
`touch -d '3 hours ago' ${shellQuote(`${replacedStage}/${ownerMarker}`)} ${shellQuote(`${originalStage}/${ownerMarker}`)}`,
`printf foreign > ${shellQuote(`${replacedStage}/payload/foreign`)}`,
`printf alive > ${shellQuote(`${foreignTarget}/sentinel`)}`,
`ln -s ${shellQuote(foreignTarget)} ${shellQuote(symlinkStage)}`,
`i=0; while [ "$i" -lt 15197 ]; do mkdir ${shellQuote(`/root/.orca-remote/relay-${relayVersion}.upload-scale-`)}"$i"; i=$((i + 1)); done`
].join(' && ')
)
const adversarialInventory = readInventory(activeFixture, remoteRelayDir)
const reclaimableStage = `${stagePool}/slot-1`
dockerExec(
activeFixture,
`touch -d '3 hours ago' ${shellQuote(`${reclaimableStage}/${ownerMarker}`)}`
)
const cleanupConnection = createConnection(activeFixture)
await cleanupConnection.connect()
const cleanedDeployment = await deployAndLaunchRelay(cleanupConnection, undefined, 60)
const cleanupMux = new SshChannelMultiplexer(cleanedDeployment.transport)
await expect(cleanupMux.request('session.resolveHome', { path: '~' })).resolves.toEqual({
resolvedPath: '/root'
})
const reclaimDeadline = Date.now() + 10_000
while (
dockerExec(
activeFixture,
`test -e ${shellQuote(reclaimableStage)} && echo PRESENT || true`
) &&
Date.now() < reclaimDeadline
) {
await new Promise((resolve) => setTimeout(resolve, 50))
}
expect(
dockerExec(activeFixture, `test ! -e ${shellQuote(reclaimableStage)} && echo RECLAIMED`)
).toBe('RECLAIMED')
const listing = await execCommand(
cleanupConnection,
listRelayBaseDirsCommand(getRemoteHostPlatform('linux-arm64'), '/root/.orca-remote')
)
const gcStartedAt = Date.now()
await gcOldRelayVersions(
cleanupConnection,
'/root',
remoteRelayDir,
getRemoteHostPlatform('linux-arm64')
)
const gcDurationMs = Date.now() - gcStartedAt
finalInventory = readInventory(activeFixture, remoteRelayDir)
expect(
dockerExec(
activeFixture,
`test -L ${shellQuote(symlinkStage)} && test -d ${shellQuote(foreignTarget)} && test "$(cat ${shellQuote(`${foreignTarget}/sentinel`)})" = alive && echo PRESERVED`
).trim()
).toBe('PRESERVED')
expect(
dockerExec(
activeFixture,
`test -d ${shellQuote(replacedStage)} && test -d ${shellQuote(originalStage)} && echo PRESERVED`
).trim()
).toBe('PRESERVED')
expect(finalInventory.uploadStages.sort()).toEqual(
adversarialInventory.uploadStages.filter((stage) => stage !== reclaimableStage).sort()
)
const listingBytes = Buffer.byteLength(listing)
expect(listingBytes).toBeLessThan(1_024)
expect(gcDurationMs).toBeLessThan(10_000)
const scaleEntries = Number(
dockerExec(
activeFixture,
`find /root/.orca-remote -mindepth 1 -maxdepth 1 -type d -name ${shellQuote(`relay-${relayVersion}.upload-scale-*`)} | wc -l`
)
)
expect(scaleEntries).toBe(15_197)
scaleEvidence = { gcDurationMs, listingBytes, scaleEntries }
cleanupMux.dispose()
await cleanupConnection.disconnect()
}
console.log(
`[pr-10207-oracle] ${JSON.stringify({ progress, firstInventory, retryResult, finalInventory, scaleEvidence, repoHead: repoHead.trim() })}`
)
expect(progress).toContain('Uploading relay...')
expect(repoHead.trim()).toMatch(/^[0-9a-f]{40}$/)
expect(retryResult).toBe(expected)
if (expected === 'recovered') {
expect(firstInventory.installLock).toBe(false)
expect(firstInventory.uploadStages.length).toBeGreaterThan(0)
expect(finalInventory?.installLock).toBe(false)
expect(finalInventory!.uploadStages).toEqual([
'/root/.orca-remote/.upload-stages/slot-0',
'/root/.orca-remote/.upload-stages/slot-0.original',
'/root/.orca-remote/.upload-stages/slot-2'
])
} else {
expect(firstInventory.installLock).toBe(true)
expect(firstInventory.payloadFiles).toBe(0)
}
}, 180_000)
})
@@ -0,0 +1,336 @@
import { spawnSync } from 'node:child_process'
import {
existsSync,
lstatSync,
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
symlinkSync,
utimesSync,
writeFileSync
} from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { getRemoteHostPlatform, type RemoteHostPlatform } from './ssh-remote-platform'
import {
cleanupOwnedRelayUploadStageCommand,
parseReservedRelayUploadStage,
promoteOwnedRelayUploadStageCommand,
recoverOneStaleRelayUploadStageCommand,
relayUploadStagePromotionConfirmed,
reserveRelayUploadStageCommand,
RELAY_UPLOAD_STAGE_SLOT_COUNT
} from './ssh-relay-upload-stage-commands'
const posix = getRemoteHostPlatform('linux-x64')
const windows = getRemoteHostPlatform('win32-x64')
const owner = '.sftp-namespace-123e4567e89b12d3a456426614174000'
const roots: string[] = []
const configuredPowerShell = process.env.ORCA_POWERSHELL_EXECUTABLE
const powerShellExecutable = [
configuredPowerShell,
...(process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh'])
].find(
(candidate) =>
candidate &&
spawnSync(candidate, ['-NoProfile', '-NonInteractive', '-Command', 'exit 0'], {
stdio: 'ignore'
}).status === 0
)
function decodePowerShellCommand(command: string): string {
const encoded = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/u)?.[1] ?? ''
return Buffer.from(encoded, 'base64').toString('utf16le')
}
function createPool(): string {
const root = mkdtempSync(join(tmpdir(), 'orca-upload-stage-pool-'))
roots.push(root)
const pool = join(root, 'pool')
mkdirSync(pool)
return pool
}
function runCommand(host: RemoteHostPlatform, command: string, prefix = '') {
if (host.commandDialect === 'powershell') {
if (!powerShellExecutable) {
throw new Error('PowerShell is unavailable')
}
return spawnSync(
powerShellExecutable,
[
'-NoProfile',
'-NonInteractive',
'-Command',
`${prefix}\n${decodePowerShellCommand(command)}`
],
{ encoding: 'utf8' }
)
}
return spawnSync('/bin/sh', ['-c', `${prefix}\n${command}`], { encoding: 'utf8' })
}
function createStage(
host: RemoteHostPlatform,
pool: string,
index: number,
stageOwner = owner,
stale = false,
state: 'slot' | 'claim' | 'delete' = 'slot'
): string {
const reservedOwner = /^\.sftp-namespace-[0-9a-f]{32}$/u.test(stageOwner) ? stageOwner : owner
const reservation = runCommand(host, reserveRelayUploadStageCommand(host, pool, reservedOwner))
expect(reservation.status, reservation.stderr).toBe(0)
const slot = join(pool, `slot-${index}`)
const stage = join(pool, `${state}-${index}`)
if (stage !== slot) {
expect(existsSync(stage)).toBe(false)
renameSync(slot, stage)
}
const marker = join(stage, '.orca-upload-owner')
if (stageOwner !== reservedOwner) {
writeFileSync(marker, stageOwner)
}
writeFileSync(join(stage, 'payload', 'relay.js'), `relay-${index}`)
if (stale) {
const old = new Date(Date.now() - 60 * 60_000)
utimesSync(marker, old, old)
}
return stage
}
afterEach(() => {
for (const root of roots.splice(0)) {
rmSync(root, { recursive: true, force: true })
}
})
describe.each([
['POSIX', posix] as const,
...((powerShellExecutable ? [['PowerShell', windows] as const] : []) as (readonly [
string,
RemoteHostPlatform
])[])
])('%s relay upload stage commands', (_label, host) => {
it.each([0, 1, 7, 8, 9])('bounds reservation with %i occupied entries', (count) => {
const pool = createPool()
for (let index = 0; index < Math.min(count, RELAY_UPLOAD_STAGE_SLOT_COUNT); index += 1) {
createStage(host, pool, index)
}
if (count > RELAY_UPLOAD_STAGE_SLOT_COUNT) {
mkdirSync(join(pool, 'foreign-extra'))
}
const result = runCommand(host, reserveRelayUploadStageCommand(host, pool, owner))
if (count < RELAY_UPLOAD_STAGE_SLOT_COUNT) {
expect(result.status, result.stderr).toBe(0)
expect(parseReservedRelayUploadStage(host, pool, owner, result.stdout).slotName).toBe(
`slot-${count}`
)
} else {
expect(result.status).not.toBe(0)
expect(result.stderr).toContain('staging quota is full')
}
})
it('promotes only the post-claim owned payload and removes its fixed stage', () => {
const pool = createPool()
const destination = join(pool, 'destination')
mkdirSync(destination)
createStage(host, pool, 0)
const stage = parseReservedRelayUploadStage(
host,
pool,
owner,
`noise\n__ORCA_UPLOAD_STAGE_SLOT__${owner}:slot-0\n`
)
const result = runCommand(
host,
promoteOwnedRelayUploadStageCommand(host, stage, owner, destination)
)
expect(result.status, result.stderr).toBe(0)
expect(relayUploadStagePromotionConfirmed(owner, result.stdout)).toBe(true)
expect(readFileSync(join(destination, 'relay.js'), 'utf8')).toBe('relay-0')
expect(existsSync(join(pool, 'slot-0'))).toBe(false)
expect(existsSync(join(pool, 'claim-0'))).toBe(false)
expect(existsSync(join(pool, 'delete-0'))).toBe(false)
})
it('rejects a payload reparse point without copying or deleting it', () => {
const pool = createPool()
const destination = join(pool, 'destination')
const foreign = join(pool, 'foreign.js')
mkdirSync(destination)
const stagePath = createStage(host, pool, 0)
rmSync(join(stagePath, 'payload', 'relay.js'))
writeFileSync(foreign, 'foreign')
symlinkSync(foreign, join(stagePath, 'payload', 'relay.js'))
const stage = parseReservedRelayUploadStage(
host,
pool,
owner,
`__ORCA_UPLOAD_STAGE_SLOT__${owner}:slot-0`
)
const result = runCommand(
host,
promoteOwnedRelayUploadStageCommand(host, stage, owner, destination)
)
expect(result.status, result.stderr).toBe(0)
expect(relayUploadStagePromotionConfirmed(owner, result.stdout)).toBe(false)
expect(existsSync(join(destination, 'relay.js'))).toBe(false)
expect(lstatSync(join(pool, 'slot-0', 'payload', 'relay.js')).isSymbolicLink()).toBe(true)
expect(readFileSync(foreign, 'utf8')).toBe('foreign')
})
it('reclaims one stale owned stage but preserves fresh and foreign stages', () => {
const pool = createPool()
createStage(host, pool, 0, owner, false)
createStage(host, pool, 1, '.foreign-owner', true)
createStage(host, pool, 2, owner, true)
const result = runCommand(host, recoverOneStaleRelayUploadStageCommand(host, pool, 60))
expect(result.status, result.stderr).toBe(0)
expect(existsSync(join(pool, 'slot-0'))).toBe(true)
expect(existsSync(join(pool, 'slot-1'))).toBe(true)
expect(existsSync(join(pool, 'slot-2'))).toBe(false)
})
it('drains fixed stale claim and delete states across repeated deployments', () => {
const pool = createPool()
createStage(host, pool, 0, owner, true, 'claim')
createStage(host, pool, 1, owner, true, 'delete')
for (let attempt = 0; attempt < 2; attempt += 1) {
const result = runCommand(host, recoverOneStaleRelayUploadStageCommand(host, pool, 60))
expect(result.status, result.stderr).toBe(0)
}
expect(existsSync(join(pool, 'claim-0'))).toBe(false)
expect(existsSync(join(pool, 'delete-1'))).toBe(false)
})
})
describe('POSIX ownership race fencing', () => {
it('restores a replacement directory and preserves the original moved aside before claim', () => {
const pool = createPool()
const destination = join(pool, 'destination')
mkdirSync(destination)
createStage(posix, pool, 0)
const stage = parseReservedRelayUploadStage(
posix,
pool,
owner,
`__ORCA_UPLOAD_STAGE_SLOT__${owner}:slot-0`
)
const prefix = [
'raced=0',
'mv() {',
'if [ "$raced" -eq 0 ]; then',
'raced=1; command mv "$1" "$1.original"; mkdir "$1"; mkdir "$1/payload"; cp "$1.original/.orca-upload-owner" "$1/.orca-upload-owner"; cp "$1.original/.orca-upload-identity" "$1/.orca-upload-identity"; touch -t 202001010000 "$1/.orca-upload-owner"; printf foreign > "$1/foreign";',
'fi;',
'command mv "$@";',
'}'
].join('\n')
const result = runCommand(
posix,
promoteOwnedRelayUploadStageCommand(posix, stage, owner, destination),
prefix
)
expect(result.status, result.stderr).toBe(0)
expect(relayUploadStagePromotionConfirmed(owner, result.stdout)).toBe(false)
expect(existsSync(join(pool, 'slot-0', 'foreign'))).toBe(true)
expect(existsSync(join(pool, 'slot-0.original', '.orca-upload-owner'))).toBe(true)
expect(existsSync(join(destination, 'relay.js'))).toBe(false)
})
it('restores a replacement symlink without touching its target', () => {
const pool = createPool()
const destination = join(pool, 'destination')
const foreign = join(pool, 'foreign')
mkdirSync(destination)
mkdirSync(foreign)
writeFileSync(join(foreign, 'sentinel'), 'alive')
createStage(posix, pool, 0)
const stage = parseReservedRelayUploadStage(
posix,
pool,
owner,
`__ORCA_UPLOAD_STAGE_SLOT__${owner}:slot-0`
)
const prefix = [
'raced=0',
'mv() {',
'if [ "$raced" -eq 0 ]; then raced=1; command mv "$1" "$1.original"; ln -s ' +
`'${foreign}' "$1"; fi;`,
'command mv "$@";',
'}'
].join('\n')
const result = runCommand(
posix,
cleanupOwnedRelayUploadStageCommand(posix, stage, owner),
prefix
)
expect(result.status, result.stderr).toBe(0)
expect(lstatSync(join(pool, 'slot-0')).isSymbolicLink()).toBe(true)
expect(readFileSync(join(foreign, 'sentinel'), 'utf8')).toBe('alive')
expect(existsSync(join(pool, 'slot-0.original', '.orca-upload-owner'))).toBe(true)
})
})
describe.runIf(powerShellExecutable)('PowerShell ownership race fencing', () => {
it('restores an old same-owner replacement whose persisted file ID does not match', () => {
const pool = createPool()
const destination = join(pool, 'destination')
mkdirSync(destination)
createStage(windows, pool, 0)
const stage = parseReservedRelayUploadStage(
windows,
pool,
owner,
`__ORCA_UPLOAD_STAGE_SLOT__${owner}:slot-0`
)
const prefix = [
'$script:raced = $false',
'function Move-Item {',
'param($LiteralPath, $Destination, $ErrorAction)',
'if (-not $script:raced) {',
'$script:raced = $true',
'Microsoft.PowerShell.Management\\Move-Item -LiteralPath $LiteralPath -Destination ($LiteralPath + ".original") -ErrorAction Stop',
'$null = New-Item -ItemType Directory -Path $LiteralPath',
'$null = New-Item -ItemType Directory -Path (Join-Path $LiteralPath "payload")',
'$newPath = (Get-Item -LiteralPath $LiteralPath).FullName',
'$originalPath = (Get-Item -LiteralPath ($LiteralPath + ".original")).FullName',
'[System.IO.File]::WriteAllText((Join-Path $newPath ".orca-upload-owner"), [System.IO.File]::ReadAllText((Join-Path $originalPath ".orca-upload-owner")))',
'[System.IO.File]::WriteAllText((Join-Path $newPath ".orca-upload-identity"), [System.IO.File]::ReadAllText((Join-Path $originalPath ".orca-upload-identity")))',
'(Get-Item -LiteralPath (Join-Path $newPath ".orca-upload-owner") -Force).LastWriteTimeUtc = [DateTime]::UtcNow.AddHours(-2)',
'[System.IO.File]::WriteAllText((Join-Path $newPath "foreign"), "foreign")',
'}',
'Microsoft.PowerShell.Management\\Move-Item -LiteralPath $LiteralPath -Destination $Destination -ErrorAction $ErrorAction',
'}'
].join('\n')
const result = runCommand(
windows,
promoteOwnedRelayUploadStageCommand(windows, stage, owner, destination),
prefix
)
expect(result.status, result.stderr).toBe(0)
expect(relayUploadStagePromotionConfirmed(owner, result.stdout)).toBe(false)
expect(existsSync(join(pool, 'slot-0', 'foreign'))).toBe(true)
expect(existsSync(join(pool, 'slot-0.original', '.orca-upload-owner'))).toBe(true)
expect(existsSync(join(destination, 'relay.js'))).toBe(false)
})
})
@@ -0,0 +1,254 @@
import { shellEscape } from './ssh-connection-utils'
import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
import {
RELAY_UPLOAD_IDENTITY_FILE_NAME as IDENTITY_FILE_NAME,
RELAY_UPLOAD_OWNER_FILE_NAME as OWNER_FILE_NAME,
RELAY_UPLOAD_PROMOTION_RESULT_PREFIX as PROMOTION_RESULT_PREFIX,
RELAY_UPLOAD_SLOT_RESULT_PREFIX as SLOT_RESULT_PREFIX,
RELAY_UPLOAD_STAGE_POOL_NAME,
RELAY_UPLOAD_STAGE_SLOT_COUNT,
RELAY_UPLOAD_STAGE_STALE_SECONDS,
type RelayUploadStageSlot
} from './ssh-relay-upload-stage-contract'
import {
cleanupWindowsRelayUploadStageCommand,
promoteWindowsRelayUploadStageCommand,
recoverWindowsRelayUploadStageCommand,
reserveWindowsRelayUploadStageCommand
} from './ssh-relay-upload-stage-windows-commands'
export {
RELAY_UPLOAD_STAGE_POOL_NAME,
RELAY_UPLOAD_STAGE_SLOT_COUNT,
RELAY_UPLOAD_STAGE_STALE_SECONDS,
type RelayUploadStageSlot
} from './ssh-relay-upload-stage-contract'
const OWNER_PATTERN = /^\.sftp-namespace-[0-9a-f]{32}$/u
function assertOwner(owner: string): void {
if (!OWNER_PATTERN.test(owner)) {
throw new Error('Invalid relay upload stage owner')
}
}
function slotPaths(
host: RemoteHostPlatform,
poolDir: string,
slotName: string
): RelayUploadStageSlot {
if (!/^slot-[0-7]$/u.test(slotName)) {
throw new Error('Invalid relay upload stage slot')
}
return {
poolDir,
slotName,
slotDir: joinRemotePath(host, poolDir, slotName),
claimDir: joinRemotePath(host, poolDir, `claim-${slotName.slice(5)}`),
deleteDir: joinRemotePath(host, poolDir, `delete-${slotName.slice(5)}`)
}
}
export function reserveRelayUploadStageCommand(
host: RemoteHostPlatform,
poolDir: string,
owner: string
): string {
assertOwner(owner)
return isWindowsRemoteHost(host)
? reserveWindowsRelayUploadStageCommand(poolDir, owner)
: reservePosixStageCommand(poolDir, owner)
}
export function parseReservedRelayUploadStage(
host: RemoteHostPlatform,
poolDir: string,
owner: string,
output: string
): RelayUploadStageSlot {
assertOwner(owner)
const authenticatedPrefix = `${SLOT_RESULT_PREFIX}${owner}:`
const line = output
.split(/\r?\n/u)
.map((entry) => entry.trim())
.find((entry) => entry.startsWith(authenticatedPrefix))
if (!line) {
throw new Error('Remote relay upload stage reservation returned no authenticated slot')
}
return slotPaths(host, poolDir, line.slice(authenticatedPrefix.length))
}
export function promoteOwnedRelayUploadStageCommand(
host: RemoteHostPlatform,
stage: RelayUploadStageSlot,
owner: string,
destinationDir: string
): string {
assertOwner(owner)
return isWindowsRemoteHost(host)
? promoteWindowsRelayUploadStageCommand(stage, owner, destinationDir)
: promotePosixStageCommand(stage, owner, destinationDir)
}
export function relayUploadStagePromotionConfirmed(owner: string, output: string): boolean {
assertOwner(owner)
return output
.split(/\r?\n/u)
.some((line) => line.trim() === `${PROMOTION_RESULT_PREFIX}${owner}:PROMOTED`)
}
export function cleanupOwnedRelayUploadStageCommand(
host: RemoteHostPlatform,
stage: RelayUploadStageSlot,
owner: string
): string {
assertOwner(owner)
return isWindowsRemoteHost(host)
? cleanupWindowsRelayUploadStageCommand(stage, owner)
: cleanupPosixStageCommand(stage, owner)
}
export function recoverOneStaleRelayUploadStageCommand(
host: RemoteHostPlatform,
poolDir: string,
staleSeconds = RELAY_UPLOAD_STAGE_STALE_SECONDS
): string {
const cutoffSeconds = Math.max(1, Math.ceil(staleSeconds))
return isWindowsRemoteHost(host)
? recoverWindowsRelayUploadStageCommand(poolDir, cutoffSeconds)
: recoverPosixStageCommand(poolDir, Math.ceil(cutoffSeconds / 60))
}
function reservePosixStageCommand(poolDir: string, owner: string): string {
const pool = shellEscape(poolDir)
const slots = Array.from({ length: RELAY_UPLOAD_STAGE_SLOT_COUNT }, (_, index) => index).join(' ')
return [
'umask 077;',
`pool=${pool};`,
'mkdir -p "$pool" || exit 1;',
'[ -d "$pool" ] && [ ! -L "$pool" ] || exit 1;',
`for n in ${slots}; do`,
'slot="$pool/slot-$n"; claim="$pool/claim-$n"; deleting="$pool/delete-$n";',
'if [ ! -e "$claim" ] && [ ! -L "$claim" ] && [ ! -e "$deleting" ] && [ ! -L "$deleting" ] && mkdir "$slot" 2>/dev/null; then',
`identity=$(ls -id "$slot" 2>/dev/null | awk '{print $1}') || identity=;`,
`if [ -n "$identity" ] && mkdir "$slot/payload" && printf '%s' ${shellEscape(owner)} > "$slot/${OWNER_FILE_NAME}" && printf '%s' "$identity" > "$slot/${IDENTITY_FILE_NAME}" && : > "$slot/${owner}"; then`,
`printf '%s%s:%s\\n' ${shellEscape(SLOT_RESULT_PREFIX)} ${shellEscape(owner)} "slot-$n"; exit 0;`,
'fi;',
'fi;',
'done;',
`printf '%s\\n' 'Orca relay upload staging quota is full; reconnect after 40 minutes or inspect .orca-remote/${RELAY_UPLOAD_STAGE_POOL_NAME}' >&2;`,
'exit 75'
].join(' ')
}
function posixClaimPrelude(stage: RelayUploadStageSlot): string {
return [
`pool=${shellEscape(stage.poolDir)}; slot=${shellEscape(stage.slotDir)}; claim=${shellEscape(stage.claimDir)}; deleting=${shellEscape(stage.deleteDir)};`,
'[ -d "$pool" ] && [ ! -L "$pool" ] || exit 0;',
'[ ! -e "$claim" ] && [ ! -L "$claim" ] && [ ! -e "$deleting" ] && [ ! -L "$deleting" ] || exit 0;',
'mv "$slot" "$claim" 2>/dev/null || exit 0;'
].join(' ')
}
function posixRestoreClaim(): string {
return 'if [ ! -e "$slot" ] && [ ! -L "$slot" ]; then mv "$claim" "$slot" 2>/dev/null || true; fi;'
}
function posixOwnedDirectoryCheck(
pathVariable: string,
owner: string,
requirePayload: boolean
): string {
const path = `$${pathVariable}`
const checks = [
`owner_file="${path}/${OWNER_FILE_NAME}";`,
`identity_file="${path}/${IDENTITY_FILE_NAME}";`,
`actual=$(cat "$owner_file" 2>/dev/null) || actual=;`,
`expected_identity=$(cat "$identity_file" 2>/dev/null) || expected_identity=;`,
`identity=$(ls -id "${path}" 2>/dev/null | awk '{print $1}') || identity=;`,
`[ -n "$identity" ] && [ "$identity" = "$expected_identity" ] && [ -d "${path}" ] && [ ! -L "${path}" ] && [ -f "$owner_file" ] && [ ! -L "$owner_file" ] && [ -f "$identity_file" ] && [ ! -L "$identity_file" ] && [ "$actual" = ${shellEscape(owner)} ]`
]
if (requirePayload) {
checks.push(
`&& payload="${path}/payload" && [ -d "$payload" ] && [ ! -L "$payload" ]`,
`&& payload_identity=$(ls -id "$payload" 2>/dev/null | awk '{print $1}') && [ -n "$payload_identity" ]`,
'&& links=$(find "$payload" -type l -print -quit 2>/dev/null) && [ -z "$links" ]'
)
}
return checks.join(' ')
}
function posixMoveOwnedClaimToDelete(owner: string): string {
return [
'[ ! -e "$deleting" ] && [ ! -L "$deleting" ] || exit 0;',
'mv "$claim" "$deleting" 2>/dev/null || exit 0;',
`if ${posixOwnedDirectoryCheck('deleting', owner, true)} && [ "$identity" = "$claim_identity" ]; then`,
'rm -rf "$deleting";',
'else',
'if [ ! -e "$claim" ] && [ ! -L "$claim" ]; then mv "$deleting" "$claim" 2>/dev/null || true; fi;',
'fi;'
].join(' ')
}
function promotePosixStageCommand(
stage: RelayUploadStageSlot,
owner: string,
destinationDir: string
): string {
const expected = shellEscape(owner)
return [
posixClaimPrelude(stage),
`if ! { ${posixOwnedDirectoryCheck('claim', owner, true)}; }; then`,
posixRestoreClaim(),
`printf '%s%s:OWNERSHIP_LOST\\n' ${shellEscape(PROMOTION_RESULT_PREFIX)} ${expected}; exit 0;`,
'fi;',
'claim_identity="$identity";',
`if (cd "$payload" && [ "$(ls -id . 2>/dev/null | awk '{print $1}')" = "$payload_identity" ] && cp -a . ${shellEscape(destinationDir)}/); then`,
posixMoveOwnedClaimToDelete(owner),
`printf '%s%s:PROMOTED\\n' ${shellEscape(PROMOTION_RESULT_PREFIX)} ${expected}; exit 0;`,
'fi;',
'exit 1'
].join(' ')
}
function cleanupPosixStageCommand(stage: RelayUploadStageSlot, owner: string): string {
return [
posixClaimPrelude(stage),
`if ${posixOwnedDirectoryCheck('claim', owner, true)}; then`,
'claim_identity="$identity";',
posixMoveOwnedClaimToDelete(owner),
'else',
posixRestoreClaim(),
'fi;',
'exit 0'
].join(' ')
}
function recoverPosixStageCommand(poolDir: string, staleMinutes: number): string {
const pool = shellEscape(poolDir)
const slots = Array.from({ length: RELAY_UPLOAD_STAGE_SLOT_COUNT }, (_, index) => index).join(' ')
return [
`pool=${pool}; [ -d "$pool" ] && [ ! -L "$pool" ] || exit 0;`,
`for n in ${slots}; do`,
'slot="$pool/slot-$n"; claim="$pool/claim-$n"; deleting="$pool/delete-$n";',
`if [ -d "$deleting" ] && [ ! -L "$deleting" ]; then owner_file="$deleting/${OWNER_FILE_NAME}"; identity_file="$deleting/${IDENTITY_FILE_NAME}"; deleting_actual=$(cat "$owner_file" 2>/dev/null) || deleting_actual=; expected_identity=$(cat "$identity_file" 2>/dev/null) || expected_identity=; deleting_identity=$(ls -id "$deleting" 2>/dev/null | awk '{print $1}') || deleting_identity=; deleting_old=$(find "$owner_file" -prune -type f -mmin +${staleMinutes} -print 2>/dev/null) || deleting_old=; if [ -n "$deleting_identity" ] && [ "$deleting_identity" = "$expected_identity" ] && [ -f "$owner_file" ] && [ ! -L "$owner_file" ] && [ -f "$identity_file" ] && [ ! -L "$identity_file" ] && printf '%s' "$deleting_actual" | grep -Eq '^\\.sftp-namespace-[0-9a-f]{32}$' && [ -n "$deleting_old" ] && links=$(find "$deleting" -type l -print -quit 2>/dev/null) && [ -z "$links" ]; then rm -rf "$deleting"; exit 0; fi; fi;`,
'if [ ! -e "$claim" ] && [ ! -L "$claim" ] && [ ! -e "$deleting" ] && [ ! -L "$deleting" ] && [ -d "$slot" ] && [ ! -L "$slot" ] &&',
`old=$(find "$slot/${OWNER_FILE_NAME}" -prune -type f -mmin +${staleMinutes} -print 2>/dev/null) && [ -n "$old" ]; then`,
'mv "$slot" "$claim" 2>/dev/null || continue;',
'fi;',
`owner_file="$claim/${OWNER_FILE_NAME}"; identity_file="$claim/${IDENTITY_FILE_NAME}"; actual=$(cat "$owner_file" 2>/dev/null) || actual=; expected_identity=$(cat "$identity_file" 2>/dev/null) || expected_identity=; identity=$(ls -id "$claim" 2>/dev/null | awk '{print $1}') || identity=; old=$(find "$owner_file" -prune -type f -mmin +${staleMinutes} -print 2>/dev/null) || old=;`,
`if [ -n "$identity" ] && [ "$identity" = "$expected_identity" ] && [ -d "$claim" ] && [ ! -L "$claim" ] && [ -f "$owner_file" ] && [ ! -L "$owner_file" ] && [ -f "$identity_file" ] && [ ! -L "$identity_file" ] && printf '%s' "$actual" | grep -Eq '^\\.sftp-namespace-[0-9a-f]{32}$' && [ -n "$old" ]; then`,
'claim_identity="$identity";',
'[ ! -e "$deleting" ] && [ ! -L "$deleting" ] && mv "$claim" "$deleting" 2>/dev/null || continue;',
`owner_file="$deleting/${OWNER_FILE_NAME}"; identity_file="$deleting/${IDENTITY_FILE_NAME}"; deleting_actual=$(cat "$owner_file" 2>/dev/null) || deleting_actual=; expected_identity=$(cat "$identity_file" 2>/dev/null) || expected_identity=; deleting_identity=$(ls -id "$deleting" 2>/dev/null | awk '{print $1}') || deleting_identity=; deleting_old=$(find "$owner_file" -prune -type f -mmin +${staleMinutes} -print 2>/dev/null) || deleting_old=;`,
`if [ -n "$deleting_identity" ] && [ "$deleting_identity" = "$claim_identity" ] && [ "$deleting_identity" = "$expected_identity" ] && [ -d "$deleting" ] && [ ! -L "$deleting" ] && [ -f "$owner_file" ] && [ ! -L "$owner_file" ] && [ -f "$identity_file" ] && [ ! -L "$identity_file" ] && printf '%s' "$deleting_actual" | grep -Eq '^\\.sftp-namespace-[0-9a-f]{32}$' && [ -n "$deleting_old" ] && links=$(find "$deleting" -type l -print -quit 2>/dev/null) && [ -z "$links" ]; then`,
'rm -rf "$deleting"; exit 0;',
'fi;',
'if [ ! -e "$claim" ] && [ ! -L "$claim" ]; then mv "$deleting" "$claim" 2>/dev/null || true; fi;',
'continue;',
'fi;',
posixRestoreClaim(),
'done;',
'exit 0'
].join(' ')
}
@@ -0,0 +1,16 @@
export const RELAY_UPLOAD_STAGE_POOL_NAME = '.upload-stages'
export const RELAY_UPLOAD_STAGE_SLOT_COUNT = 8
export const RELAY_UPLOAD_STAGE_STALE_SECONDS = 40 * 60
export const RELAY_UPLOAD_OWNER_FILE_NAME = '.orca-upload-owner'
export const RELAY_UPLOAD_IDENTITY_FILE_NAME = '.orca-upload-identity'
export const RELAY_UPLOAD_SLOT_RESULT_PREFIX = '__ORCA_UPLOAD_STAGE_SLOT__'
export const RELAY_UPLOAD_PROMOTION_RESULT_PREFIX = '__ORCA_UPLOAD_STAGE_PROMOTION__'
export type RelayUploadStageSlot = {
claimDir: string
deleteDir: string
poolDir: string
slotDir: string
slotName: string
}
@@ -0,0 +1,247 @@
import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell'
import {
RELAY_UPLOAD_IDENTITY_FILE_NAME,
RELAY_UPLOAD_OWNER_FILE_NAME,
RELAY_UPLOAD_PROMOTION_RESULT_PREFIX,
RELAY_UPLOAD_SLOT_RESULT_PREFIX,
RELAY_UPLOAD_STAGE_POOL_NAME,
RELAY_UPLOAD_STAGE_SLOT_COUNT,
type RelayUploadStageSlot
} from './ssh-relay-upload-stage-contract'
export function reserveWindowsRelayUploadStageCommand(poolDir: string, owner: string): string {
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
...windowsFileIdentityScript(),
`$pool = ${powerShellLiteral(poolDir)}`,
`$owner = ${powerShellLiteral(owner)}`,
'$null = New-Item -ItemType Directory -Force -Path $pool',
'$poolItem = Get-Item -LiteralPath $pool -Force -ErrorAction Stop',
'if (-not $poolItem.PSIsContainer -or (($poolItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0)) { throw "Unsafe relay upload stage pool" }',
`foreach ($n in 0..${RELAY_UPLOAD_STAGE_SLOT_COUNT - 1}) {`,
'$slot = Join-Path $pool ("slot-" + $n)',
'$claim = Join-Path $pool ("claim-" + $n)',
'$deleting = Join-Path $pool ("delete-" + $n)',
'$slotExisting = Get-Item -LiteralPath $slot -Force -ErrorAction SilentlyContinue',
'$claimExisting = Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue',
'$deleteExisting = Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue',
'if (($null -ne $claimExisting) -or ($null -ne $slotExisting) -or ($null -ne $deleteExisting)) { continue }',
'try {',
'$null = New-Item -ItemType Directory -Path $slot -ErrorAction Stop',
'$null = New-Item -ItemType Directory -Path (Join-Path $slot "payload") -ErrorAction Stop',
'$identity = & $getFileIdentity $slot',
`[System.IO.File]::WriteAllText((Join-Path $slot ${powerShellLiteral(RELAY_UPLOAD_OWNER_FILE_NAME)}), $owner, [System.Text.UTF8Encoding]::new($false))`,
`[System.IO.File]::WriteAllText((Join-Path $slot ${powerShellLiteral(RELAY_UPLOAD_IDENTITY_FILE_NAME)}), $identity, [System.Text.UTF8Encoding]::new($false))`,
`[System.IO.File]::WriteAllText((Join-Path $slot ${powerShellLiteral(owner)}), '', [System.Text.UTF8Encoding]::new($false))`,
`Write-Output (${powerShellLiteral(RELAY_UPLOAD_SLOT_RESULT_PREFIX)} + $owner + ':slot-' + $n)`,
'exit 0',
'} catch { continue }',
'}',
`throw ${powerShellLiteral(`Orca relay upload staging quota is full; reconnect after 40 minutes or inspect .orca-remote/${RELAY_UPLOAD_STAGE_POOL_NAME}`)}`
].join('\n')
)
}
function windowsFileIdentityScript(): string[] {
return [
"if ($env:OS -eq 'Windows_NT') {",
"if ($null -eq ('OrcaRelayUploadFileIdentity' -as [type])) {",
"Add-Type -TypeDefinition @'",
'using System;',
'using System.ComponentModel;',
'using System.Runtime.InteropServices;',
'using Microsoft.Win32.SafeHandles;',
'public static class OrcaRelayUploadFileIdentity {',
'[StructLayout(LayoutKind.Sequential)] struct Info { public uint Attr; public System.Runtime.InteropServices.ComTypes.FILETIME C; public System.Runtime.InteropServices.ComTypes.FILETIME A; public System.Runtime.InteropServices.ComTypes.FILETIME W; public uint Vol; public uint SizeH; public uint SizeL; public uint Links; public uint IndexH; public uint IndexL; }',
'[DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern SafeFileHandle CreateFileW(string path, uint access, uint share, IntPtr sec, uint creation, uint flags, IntPtr template);',
'[DllImport("kernel32.dll", SetLastError=true)] static extern bool GetFileInformationByHandle(SafeFileHandle handle, out Info info);',
'public static string Read(string path) {',
'using (var handle = CreateFileW(path, 0, 7, IntPtr.Zero, 3, 0x02200000, IntPtr.Zero)) {',
'if (handle.IsInvalid) throw new Win32Exception(Marshal.GetLastWin32Error());',
'Info info; if (!GetFileInformationByHandle(handle, out info)) throw new Win32Exception(Marshal.GetLastWin32Error());',
'return info.Vol.ToString("x") + ":" + info.IndexH.ToString("x") + ":" + info.IndexL.ToString("x");',
'}',
'}',
'}',
"'@",
'}',
'$getFileIdentity = { param($path) [OrcaRelayUploadFileIdentity]::Read($path) }',
'} else {',
'$getFileIdentity = { param($path) $resolved = (Get-Item -LiteralPath $path -Force -ErrorAction Stop).FullName; $value = & /usr/bin/stat -f "%i" -- $resolved 2>$null; if ($LASTEXITCODE -ne 0) { $value = & /usr/bin/stat -c "%i" -- $resolved }; ([string]$value).Trim() }',
'}'
]
}
function windowsClaimPrelude(stage: RelayUploadStageSlot): string[] {
return [
`$slot = ${powerShellLiteral(stage.slotDir)}`,
`$claim = ${powerShellLiteral(stage.claimDir)}`,
`$deleting = ${powerShellLiteral(stage.deleteDir)}`,
`$pool = ${powerShellLiteral(stage.poolDir)}`,
'$poolItem = Get-Item -LiteralPath $pool -Force -ErrorAction SilentlyContinue',
'if (($null -eq $poolItem) -or -not $poolItem.PSIsContainer -or (($poolItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0)) { exit 0 }',
'$claimBefore = Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue',
'$deleteBefore = Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue',
'$slotBefore = Get-Item -LiteralPath $slot -Force -ErrorAction SilentlyContinue',
'if (($null -ne $claimBefore) -or ($null -ne $deleteBefore) -or ($null -eq $slotBefore)) { exit 0 }',
'Move-Item -LiteralPath $slot -Destination $claim -ErrorAction Stop'
]
}
function windowsOwnershipScript(
owner: string,
requirePayload: boolean,
pathExpression = '$claim'
): string[] {
return [
`$expectedOwner = ${powerShellLiteral(owner)}`,
`$ownedPath = ${pathExpression}`,
`$ownerPath = Join-Path $ownedPath ${powerShellLiteral(RELAY_UPLOAD_OWNER_FILE_NAME)}`,
`$identityPath = Join-Path $ownedPath ${powerShellLiteral(RELAY_UPLOAD_IDENTITY_FILE_NAME)}`,
'$claimItem = Get-Item -LiteralPath $ownedPath -Force -ErrorAction SilentlyContinue',
'$ownerItem = Get-Item -LiteralPath $ownerPath -Force -ErrorAction SilentlyContinue',
'$identityItem = Get-Item -LiteralPath $identityPath -Force -ErrorAction SilentlyContinue',
'$actualOwner = if ($null -ne $ownerItem) { [System.IO.File]::ReadAllText($ownerPath).Trim() } else { "" }',
'$expectedIdentity = if ($null -ne $identityItem) { [System.IO.File]::ReadAllText($identityPath).Trim() } else { "" }',
'$actualIdentity = if ($null -ne $claimItem) { & $getFileIdentity $ownedPath } else { "" }',
'$owned = ($null -ne $claimItem) -and $claimItem.PSIsContainer -and (($claimItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -ne $ownerItem) -and -not $ownerItem.PSIsContainer -and (($ownerItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -ne $identityItem) -and -not $identityItem.PSIsContainer -and (($identityItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($actualOwner -ceq $expectedOwner) -and ($actualIdentity -ceq $expectedIdentity)',
...(requirePayload
? [
'$payload = Join-Path $ownedPath "payload"',
'$payloadItem = Get-Item -LiteralPath $payload -Force -ErrorAction SilentlyContinue',
'$reparse = $null',
'if (($null -ne $payloadItem) -and $payloadItem.PSIsContainer -and (($payloadItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0)) {',
'$reparse = Get-ChildItem -LiteralPath $payload -Force -Recurse -ErrorAction Stop | Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | Select-Object -First 1',
'}',
'$owned = $owned -and ($null -ne $payloadItem) -and $payloadItem.PSIsContainer -and (($payloadItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -eq $reparse)'
]
: [])
]
}
function windowsRestoreClaim(): string {
return 'if (($null -eq (Get-Item -LiteralPath $slot -Force -ErrorAction SilentlyContinue)) -and ($null -ne (Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue))) { Move-Item -LiteralPath $claim -Destination $slot -ErrorAction SilentlyContinue }'
}
function windowsStaleOwnershipScript(pathExpression: string): string[] {
return [
`$ownedPath = ${pathExpression}`,
`$ownerPath = Join-Path $ownedPath ${powerShellLiteral(RELAY_UPLOAD_OWNER_FILE_NAME)}`,
`$identityPath = Join-Path $ownedPath ${powerShellLiteral(RELAY_UPLOAD_IDENTITY_FILE_NAME)}`,
'$ownedItem = Get-Item -LiteralPath $ownedPath -Force -ErrorAction SilentlyContinue',
'$ownerItem = Get-Item -LiteralPath $ownerPath -Force -ErrorAction SilentlyContinue',
'$identityItem = Get-Item -LiteralPath $identityPath -Force -ErrorAction SilentlyContinue',
'$actualOwner = if ($null -ne $ownerItem) { [System.IO.File]::ReadAllText($ownerPath).Trim() } else { "" }',
'$expectedIdentity = if ($null -ne $identityItem) { [System.IO.File]::ReadAllText($identityPath).Trim() } else { "" }',
'$actualIdentity = if ($null -ne $ownedItem) { & $getFileIdentity $ownedPath } else { "" }',
"$owned = ($null -ne $ownedItem) -and $ownedItem.PSIsContainer -and (($ownedItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -ne $ownerItem) -and -not $ownerItem.PSIsContainer -and (($ownerItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -ne $identityItem) -and -not $identityItem.PSIsContainer -and (($identityItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($actualIdentity -ceq $expectedIdentity) -and ($ownerItem.LastWriteTimeUtc -lt $cutoff) -and ($actualOwner -match '^\\.sftp-namespace-[0-9a-f]{32}$')",
'$reparse = $null',
'if ($owned) {',
'$reparse = Get-ChildItem -LiteralPath $ownedPath -Force -Recurse -ErrorAction Stop | Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | Select-Object -First 1',
'}',
'$owned = $owned -and ($null -eq $reparse)'
]
}
function windowsDeleteOwnedClaimScript(owner: string): string[] {
return [
'$deleteBefore = Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue',
'if ($null -ne $deleteBefore) { exit 0 }',
'Move-Item -LiteralPath $claim -Destination $deleting -ErrorAction Stop',
...windowsOwnershipScript(owner, true, '$deleting'),
'if ($owned) {',
'Remove-Item -LiteralPath $deleting -Recurse -Force -ErrorAction Stop',
'} elseif (($null -eq (Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue)) -and ($null -ne (Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue))) {',
'Move-Item -LiteralPath $deleting -Destination $claim -ErrorAction SilentlyContinue',
'}'
]
}
export function promoteWindowsRelayUploadStageCommand(
stage: RelayUploadStageSlot,
owner: string,
destinationDir: string
): string {
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
...windowsFileIdentityScript(),
...windowsClaimPrelude(stage),
...windowsOwnershipScript(owner, true),
'if (-not $owned) {',
windowsRestoreClaim(),
`Write-Output (${powerShellLiteral(RELAY_UPLOAD_PROMOTION_RESULT_PREFIX)} + $expectedOwner + ':OWNERSHIP_LOST')`,
'exit 0',
'}',
`Get-ChildItem -LiteralPath $payload -Force -ErrorAction Stop | Copy-Item -Destination ${powerShellLiteral(destinationDir)} -Recurse -Force -ErrorAction Stop`,
...windowsDeleteOwnedClaimScript(owner),
`Write-Output (${powerShellLiteral(RELAY_UPLOAD_PROMOTION_RESULT_PREFIX)} + $expectedOwner + ':PROMOTED')`
].join('\n')
)
}
export function cleanupWindowsRelayUploadStageCommand(
stage: RelayUploadStageSlot,
owner: string
): string {
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
...windowsFileIdentityScript(),
...windowsClaimPrelude(stage),
...windowsOwnershipScript(owner, true),
'if ($owned) {',
...windowsDeleteOwnedClaimScript(owner),
'} else {',
windowsRestoreClaim(),
'}'
].join('\n')
)
}
export function recoverWindowsRelayUploadStageCommand(
poolDir: string,
staleSeconds: number
): string {
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
...windowsFileIdentityScript(),
`$pool = ${powerShellLiteral(poolDir)}`,
`$cutoff = [DateTime]::UtcNow.AddSeconds(-${staleSeconds})`,
'$poolItem = Get-Item -LiteralPath $pool -Force -ErrorAction SilentlyContinue',
'if (($null -eq $poolItem) -or -not $poolItem.PSIsContainer -or (($poolItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0)) { exit 0 }',
`foreach ($n in 0..${RELAY_UPLOAD_STAGE_SLOT_COUNT - 1}) {`,
'$slot = Join-Path $pool ("slot-" + $n)',
'$claim = Join-Path $pool ("claim-" + $n)',
'$deleting = Join-Path $pool ("delete-" + $n)',
...windowsStaleOwnershipScript('$deleting'),
'if ($owned) {',
'Remove-Item -LiteralPath $deleting -Recurse -Force -ErrorAction Stop',
'exit 0',
'}',
`$slotOwner = Join-Path $slot ${powerShellLiteral(RELAY_UPLOAD_OWNER_FILE_NAME)}`,
'$slotItem = Get-Item -LiteralPath $slot -Force -ErrorAction SilentlyContinue',
'$slotOwnerItem = Get-Item -LiteralPath $slotOwner -Force -ErrorAction SilentlyContinue',
'$claimItem = Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue',
'$deleteItem = Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue',
'if (($null -eq $claimItem) -and ($null -eq $deleteItem) -and ($null -ne $slotItem) -and $slotItem.PSIsContainer -and (($slotItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and ($null -ne $slotOwnerItem) -and -not $slotOwnerItem.PSIsContainer -and (($slotOwnerItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0) -and $slotOwnerItem.LastWriteTimeUtc -lt $cutoff) {',
'Move-Item -LiteralPath $slot -Destination $claim -ErrorAction SilentlyContinue',
'}',
...windowsStaleOwnershipScript('$claim'),
'if ($owned -and ($null -eq (Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue))) {',
'Move-Item -LiteralPath $claim -Destination $deleting -ErrorAction Stop',
...windowsStaleOwnershipScript('$deleting'),
'if (-not $owned) {',
'if (($null -eq (Get-Item -LiteralPath $claim -Force -ErrorAction SilentlyContinue)) -and ($null -ne (Get-Item -LiteralPath $deleting -Force -ErrorAction SilentlyContinue))) { Move-Item -LiteralPath $deleting -Destination $claim -ErrorAction SilentlyContinue }',
'continue',
'}',
'Remove-Item -LiteralPath $deleting -Recurse -Force -ErrorAction Stop',
'exit 0',
'}',
windowsRestoreClaim(),
'}'
].join('\n')
)
}
@@ -20,6 +20,7 @@ import {
import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone'
import {
listRelayBaseDirsCommand,
MAX_RELAY_GC_LISTING_ENTRIES,
moveRemoteTreeCommand,
probeFileExistsCommand,
probeRelayInstalledCommand,
@@ -196,6 +197,7 @@ export async function gcOldRelayVersions(
.split('\n')
.map((s) => s.trim())
.filter(Boolean)
.slice(0, MAX_RELAY_GC_LISTING_ENTRIES)
await cleanupRelayGcTombstones(conn, baseDir, entries, host)
+62 -3
View File
@@ -20,8 +20,10 @@ import {
commandInRemoteDirectory,
commandWithNodePath,
listRelayBaseDirsCommand,
MAX_RELAY_GC_LISTING_ENTRIES,
makeRemoteDirectoryCommand,
moveRemoteTreeCommand,
promoteRemoteTreeContentsCommand,
probeDirectoryExistsCommand,
probeRelayInstalledCommand,
readRemoteHomeCommand,
@@ -31,9 +33,13 @@ import { getRemoteHostPlatform } from './ssh-remote-platform'
const posix = getRemoteHostPlatform('linux-x64')
const windows = getRemoteHostPlatform('win32-x64')
const powerShellExecutable = (
process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh']
).find((candidate) => {
const powerShellExecutable = [
process.env.ORCA_POWERSHELL_EXECUTABLE,
...(process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh'])
].find((candidate) => {
if (!candidate) {
return false
}
const result = spawnSync(candidate, ['-NoProfile', '-NonInteractive', '-Command', 'exit 0'], {
stdio: 'ignore'
})
@@ -151,6 +157,29 @@ describe('ssh remote command builders', () => {
expect(windowsScript).toContain("'MOVED'")
})
it('enumerates Windows staging children before copying', () => {
const script = decodePowerShellCommand(
promoteRemoteTreeContentsCommand(windows, 'C:/Users/me/relay.upload-123', 'C:/Users/me/relay')
)
expect(script).toContain('Get-ChildItem -LiteralPath')
expect(script).toContain(' -Force -ErrorAction Stop | Copy-Item -Destination')
expect(script).not.toContain('Copy-Item -LiteralPath')
expect(script).toContain('Remove-Item -LiteralPath')
expect(script).toContain("$ErrorActionPreference = 'Stop'")
expect(script).toContain('Copy-Item -Destination')
})
it('removes POSIX staging only after the copy succeeds', () => {
const command = promoteRemoteTreeContentsCommand(
posix,
'/home/u/relay.upload-123',
'/home/u/relay'
)
expect(command).toContain("cp -a '/home/u/relay.upload-123'/. '/home/u/relay'/")
expect(command).toContain("&& rm -rf '/home/u/relay.upload-123'")
expect(command.indexOf('cp -a')).toBeLessThan(command.indexOf('rm -rf'))
})
it('emits an explicit POSIX liveness result so GC can fail closed', () => {
const command = relayLivenessProbeCommand(posix, '/home/u/.orca-remote/relay-0.1.0')
@@ -177,6 +206,36 @@ describe('ssh remote command builders', () => {
)
})
it('bounds real POSIX GC output with more than the exec-cap stage population', async () => {
const root = mkdtempSync(join(tmpdir(), 'orca-relay-gc-scale-'))
try {
for (let index = 0; index < 15_197; index += 1) {
mkdirSync(join(root, `relay-0.1.0+abc.upload-${String(index).padStart(12, '0')}`))
}
mkdirSync(join(root, 'relay-0.1.0+aaa'))
mkdirSync(join(root, 'relay-0.1.0+bbb'))
const output = await runShellCommand(listRelayBaseDirsCommand(posix, root))
const entries = output.trim().split('\n')
expect(entries).toEqual(['relay-0.1.0+aaa', 'relay-0.1.0+bbb'])
expect(Buffer.byteLength(output)).toBeLessThan(1_024)
expect(entries.length).toBeLessThanOrEqual(MAX_RELAY_GC_LISTING_ENTRIES)
} finally {
rmSync(root, { recursive: true, force: true })
}
}, 30_000)
it('fails closed when real POSIX GC enumeration fails', async () => {
const root = mkdtempSync(join(tmpdir(), 'orca-relay-gc-failure-'))
try {
const command = `find() { return 23; }\n${listRelayBaseDirsCommand(posix, root)}`
await expect(runShellCommand(command)).rejects.toThrow('shell exited 1')
} finally {
rmSync(root, { recursive: true, force: true })
}
})
it('escapes double quotes before passing JavaScript to native Windows commands', () => {
const script = decodePowerShellCommand(
relayLivenessProbeCommand(windows, 'C:/Users/me/.orca-remote/relay-0.1.0', {
+25 -3
View File
@@ -63,6 +63,19 @@ export function moveRemoteTreeCommand(
)
}
export function promoteRemoteTreeContentsCommand(
host: RemoteHostPlatform,
sourcePath: string,
destinationPath: string
): string {
if (!isWindowsRemoteHost(host)) {
return `cp -a ${shellEscape(sourcePath)}/. ${shellEscape(destinationPath)}/ && rm -rf ${shellEscape(sourcePath)}`
}
return powerShellCommand(
`$ErrorActionPreference = 'Stop'; Get-ChildItem -LiteralPath ${powerShellLiteral(sourcePath)} -Force -ErrorAction Stop | Copy-Item -Destination ${powerShellLiteral(destinationPath)} -Recurse -Force -ErrorAction Stop; Remove-Item -LiteralPath ${powerShellLiteral(sourcePath)} -Recurse -Force -ErrorAction Stop`
)
}
export function writeRemoteEmptyFileCommand(host: RemoteHostPlatform, remotePath: string): string {
if (!isWindowsRemoteHost(host)) {
return `touch ${shellEscape(remotePath)}`
@@ -102,17 +115,26 @@ export function probeRelayInstalledCommand(
)
}
export const MAX_RELAY_GC_LISTING_ENTRIES = 64
export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string {
if (!isWindowsRemoteHost(host)) {
return `ls -1 ${shellEscape(baseDir)} 2>/dev/null || true`
const statusPrefix = '__ORCA_RELAY_GC_FIND_STATUS__'
return [
`base=${shellEscape(baseDir)}; [ -d "$base" ] || exit 0;`,
`{ find "$base" -mindepth 1 -maxdepth 1 -type d -name 'relay-*' -print; status=$?; printf '\n${statusPrefix}%s\n' "$status"; } |`,
String.raw`awk 'BEGIN { count=0; status=-1 } /^${statusPrefix}[0-9]+$/ { status=substr($0, ${statusPrefix.length + 1}); next } { name=$0; sub(/^.*\//, "", name); if (name ~ /^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$/ && count < ${MAX_RELAY_GC_LISTING_ENTRIES}) { entries[count++]=name } } END { if (status != 0) exit 1; for (i=0; i<count; i++) print entries[i] }'`
].join(' ')
}
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
`$base = ${powerShellLiteral(baseDir)}`,
'if (Test-Path -LiteralPath $base -PathType Container) {',
'Get-ChildItem -LiteralPath $base -Directory | ForEach-Object { $_.Name }',
"Get-ChildItem -LiteralPath $base -Directory -Filter 'relay-*' -ErrorAction Stop | Where-Object { $_.Name -match '^relay-(v?[0-9]+\\.[0-9]+\\.[0-9]+(\\+[0-9a-f]+)?)(\\.gc-tombstone\\.[0-9]+\\.[0-9]+)?$' } | Select-Object -First " +
`${MAX_RELAY_GC_LISTING_ENTRIES} | ForEach-Object { $_.Name }`,
'}'
].join(' ')
].join('\n')
)
}
@@ -37,7 +37,7 @@ export async function connectDockerSshRelayTarget(
target: {
label: `${viaProxyJump ? 'Docker SSH ProxyJump' : 'Docker SSH Relay'} E2E ${Date.now()}`,
...(viaProxyJump ? { configHost: 'orca-e2e-destination' } : {}),
host: '127.0.0.1',
host: target.host,
port: viaProxyJump ? 22 : target.port,
username: 'root',
identityFile: target.identityFile,
+12 -4
View File
@@ -14,6 +14,7 @@ export const DOCKER_SSH_SECOND_HUB_REMOTE_REPO_PATH = '/tmp/orca-docker-second-h
export type DockerSshRelayTarget = {
containerName: string
containerIp: string
host: string
identityFile: string
port: number
tempDir: string
@@ -58,7 +59,7 @@ function sshArgs(target: DockerSshRelayTarget, command: string): string[] {
'BatchMode=yes',
'-o',
'IdentitiesOnly=yes',
'root@127.0.0.1',
`root@${target.host}`,
command
]
}
@@ -122,6 +123,13 @@ export function writeDockerSshRelayTargetFile(
}
export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTarget {
const host = process.env.ORCA_E2E_SSH_TARGET_HOST?.trim() || '127.0.0.1'
if (host === 'localhost' || host === '::1' || host.startsWith('127.')) {
if (process.env.ORCA_E2E_SSH_TARGET_HOST) {
throw new Error(`ORCA_E2E_SSH_TARGET_HOST must be non-loopback: ${host}`)
}
}
const bindHost = host === '127.0.0.1' ? host : '0.0.0.0'
const tempDir = mkdtempSync(path.join(os.tmpdir(), 'orca-ssh-docker-'))
const identityFile = path.join(tempDir, 'id_ed25519')
run('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', identityFile, '-q'])
@@ -139,7 +147,7 @@ export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTar
'--name',
containerName,
'-p',
'127.0.0.1::22',
`${bindHost}::22`,
'-e',
`AUTHORIZED_KEY=${publicKey}`,
getDockerSshRelayImage(),
@@ -169,7 +177,7 @@ export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTar
if (!containerIp) {
throw new Error(`Unable to read container IP for ${containerName}`)
}
target = { containerName, containerIp, identityFile, port, tempDir }
target = { containerName, containerIp, host, identityFile, port, tempDir }
waitForSsh(target)
seedRemoteRepo(target, DOCKER_SSH_RELAY_REMOTE_REPO_PATH)
seedRemoteRepo(target, DOCKER_SSH_PROXY_JUMP_REMOTE_REPO_PATH)
@@ -177,7 +185,7 @@ export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTar
return target
} catch (error) {
cleanupDockerSshRelayTarget(
target ?? { containerName, containerIp: '', identityFile, port: 0, tempDir }
target ?? { containerName, containerIp: '', host, identityFile, port: 0, tempDir }
)
throw error
}