feat: compose bounded SSH relay SFTP transfer

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo-H
2026-07-16 11:52:28 -04:00
co-authored by Orca
parent d112617067
commit 32f78d720f
9 changed files with 936 additions and 30 deletions
@@ -13,6 +13,7 @@ on:
- 'package.json'
- 'pnpm-lock.yaml'
- 'src/main/ipc/parcel-watcher-process-entry.ts'
- 'src/main/ssh/ssh-relay-runtime-*'
- 'src/relay/**'
workflow_dispatch:
# Why: callers may gate on the native graph only after separate release wiring is reviewed.
@@ -286,6 +287,7 @@ jobs:
src/main/ssh/ssh-relay-runtime-source-scan.test.ts \
src/main/ssh/ssh-relay-runtime-source-stream.test.ts \
src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts \
src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts \
src/main/ssh/ssh-relay-runtime-sftp-session.test.ts \
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts \
src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts \
@@ -553,6 +555,88 @@ jobs:
src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts
test ! -e "$extraction_output"
- name: Start loopback OpenSSH SFTP fixture
if: runner.os == 'Linux'
shell: bash
run: |
set -euo pipefail
if ! test -x /usr/sbin/sshd; then
apt_options=(-o Acquire::Retries=3 -o Acquire::https::Timeout=15 -o Acquire::ForceIPv4=true)
timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" update -qq
timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" install -y -qq openssh-server
fi
fixture="$RUNNER_TEMP/orca-live-sftp"
mkdir -p "$fixture" "$HOME/.ssh" "$fixture/remote"
chmod 700 "$HOME/.ssh" "$fixture"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client-key"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host-key"
cp "$fixture/client-key.pub" "$HOME/.ssh/authorized_keys"
chmod 600 "$HOME/.ssh/authorized_keys" "$fixture/client-key" "$fixture/host-key"
sudo chown root:root "$fixture/host-key"
port=22222
server_version=$(/usr/sbin/sshd -V 2>&1 | head -1)
config="$fixture/sshd_config"
printf '%s\n' \
"Port $port" \
'ListenAddress 127.0.0.1' \
"HostKey $fixture/host-key" \
"PidFile $fixture/sshd.pid" \
"AuthorizedKeysFile $HOME/.ssh/authorized_keys" \
"AllowUsers $USER" \
'PasswordAuthentication no' \
'KbdInteractiveAuthentication no' \
'ChallengeResponseAuthentication no' \
'UsePAM no' \
'StrictModes yes' \
'LogLevel VERBOSE' \
'Subsystem sftp internal-sftp' > "$config"
sudo mkdir -p /run/sshd
sudo /usr/sbin/sshd -f "$config" -E "$fixture/sshd.log"
for attempt in {1..20}; do
if ssh -F /dev/null -i "$fixture/client-key" -p "$port" \
-o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
"$USER@127.0.0.1" true; then
break
fi
if test "$attempt" = 20; then
cat "$fixture/sshd.log"
exit 1
fi
sleep 0.25
done
{
printf 'ORCA_SSH_RELAY_LIVE_SFTP_HOST=127.0.0.1\n'
printf 'ORCA_SSH_RELAY_LIVE_SFTP_PORT=%s\n' "$port"
printf 'ORCA_SSH_RELAY_LIVE_SFTP_USER=%s\n' "$USER"
printf 'ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY=%s\n' "$fixture/client-key"
printf 'ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT=%s\n' "$fixture/remote"
printf 'ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION=%s\n' "$server_version"
} >> "$GITHUB_ENV"
- name: Measure exact full-size runtime over live OpenSSH SFTP
if: runner.os == 'Linux'
shell: bash
run: |
set -euo pipefail
first_output="$RUNNER_TEMP/ssh-relay-runtime/${{ matrix.tuple }}/first"
ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT="$first_output/runtime" \
ORCA_SSH_RELAY_FULL_SIZE_IDENTITY="$first_output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json" \
pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 --reporter=verbose \
src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts
- name: Stop loopback OpenSSH SFTP fixture
if: always() && runner.os == 'Linux'
shell: bash
run: |
set -euo pipefail
fixture="$RUNNER_TEMP/orca-live-sftp"
if test -s "$fixture/sshd.pid"; then
sudo kill "$(cat "$fixture/sshd.pid")" || true
fi
if test -f "$fixture/sshd.log"; then
sudo tail -200 "$fixture/sshd.log"
fi
- name: Upload unpublished artifact evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
@@ -844,6 +928,7 @@ jobs:
src/main/ssh/ssh-relay-runtime-source-scan.test.ts `
src/main/ssh/ssh-relay-runtime-source-stream.test.ts `
src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts `
src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts `
src/main/ssh/ssh-relay-runtime-sftp-session.test.ts `
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts `
src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts `
@@ -83,6 +83,25 @@ describe('SSH relay runtime artifact workflow', () => {
)
})
it('runs exact full-size Linux runtime transfer against loopback-only stock OpenSSH', async () => {
const workflow = parse(await readFile(workflowUrl, 'utf8'))
const steps = workflow.jobs['build-posix-runtime'].steps
const start = steps.find((step) => step.name === 'Start loopback OpenSSH SFTP fixture')
const measure = steps.find(
(step) => step.name === 'Measure exact full-size runtime over live OpenSSH SFTP'
)
const stop = steps.find((step) => step.name === 'Stop loopback OpenSSH SFTP fixture')
expect(start.if).toBe("runner.os == 'Linux'")
expect(start.run).toContain('ListenAddress 127.0.0.1')
expect(start.run).toContain('Subsystem sftp internal-sftp')
expect(start.run).toContain('timeout --signal=TERM --kill-after=15s 180s sudo apt-get')
expect(measure.if).toBe("runner.os == 'Linux'")
expect(measure.run).toContain('ssh-relay-runtime-sftp-openssh-full-size.test.ts')
expect(measure.run).toContain('first_output/runtime')
expect(stop.if).toBe("always() && runner.os == 'Linux'")
})
it('uploads only the first output after two clean builds verify and compare', async () => {
const source = await readFile(workflowUrl, 'utf8')
const workflow = parse(source)
@@ -144,6 +163,7 @@ describe('SSH relay runtime artifact workflow', () => {
'ssh-relay-runtime-source-scan.test.ts',
'ssh-relay-runtime-source-stream.test.ts',
'ssh-relay-runtime-sftp-file-destination.test.ts',
'ssh-relay-runtime-sftp-connection-transfer.test.ts',
'ssh-relay-runtime-sftp-session.test.ts',
'ssh-relay-runtime-sftp-tree-transfer.test.ts',
'ssh-relay-compiled-manifest-trust.test.ts',
@@ -9,9 +9,20 @@ keeps commands, hashes, runner identities, timings, and failure details.
A checked box means the work has evidence in the detailed ledger. Design approval alone does not
complete a box.
Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected raw `ssh2`
session adapter is exact-head green; live adapter/tree composition audit is next, 2026-07-15, Codex
implementation owner.** `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both
Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected live
OpenSSH SFTP composition is in progress, 2026-07-15, Codex implementation owner.**
`E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001` limits this package to one authenticated built-in
`SshConnection`, one raw SFTP channel, the existing one-to-four file transfer, captured-transport-
only force close with awaited teardown, and exact full-size Linux x64/arm64 runtime transfer against
target-native OpenSSH runners. It also requires ordinary cancellation cleanup to run before a short
retained-callback breaker closes the session. No product/default, system-SSH, Windows-remote,
high-RTT, `MaxSessions=1`, or tuple-enablement claim is made. SignPath remains deferred.
`E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` is locally green: 15/15 purpose cases, 189 focused cases
with two declared skips, 603 broad relay cases with six declared skips, 281/281 release contracts,
typecheck, full lint, formatting, max-lines, diff, protected-resolver, and isolation gates pass. The
full-size live suite is deliberately skipped without runner OpenSSH/runtime inputs, so exact-head
Linux x64/arm64 live CI remains the next gate.
`E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both
Linux supplements, Windows x64 floor, PR Checks, Golden E2E, and computer-use on exact commit
`fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100
rejection after complete runtime smoke. No live SFTP/tree or product/default claim is made.
@@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps
Date created: 2026-07-14<br>
Last updated: 2026-07-15<br>
Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected raw `ssh2` session adapter is exact-head green; live adapter/tree composition audit is next**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
Session checkpoint: **RAW SFTP SESSION ADAPTER EXACT-HEAD GREEN — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all-six primary native, both Linux supplement, Windows x64 floor, PR Checks, Golden E2E, and computer-use proof for exact implementation commit `fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100 rejection after complete runtime smoke. Audit the smallest disconnected live adapter/tree composition next, including a concrete real connection-force-close owner and OpenSSH SFTP server. Do not credit live transfer, server filesystem/modes, full-size network metrics, product/settings/fallback wiring, tuple enablement, publication, or default behavior from native callback mocks. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected authenticated-connection/SFTP-tree composition and live OpenSSH proof**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
Session checkpoint: **LIVE SFTP COMPOSITION LOCALLY GREEN / EXACT-HEAD RUNNER PROOF NEXT — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` proves the disconnected authenticated-connection owner, captured-client force close, cancellation cleanup/breaker ordering, workflow oracle, focused/broad/release suites, typecheck, full lint, formatting, max-lines, diff, protected-resolver, and no-product-import gates locally. The purpose suite is 15/15; the live full-size suite is deliberately skipped locally because no runner OpenSSH/runtime inputs are present and therefore earns no live cell. Push this isolated package and require exact-head Linux x64/arm64 stock-OpenSSH full-size metrics before crediting live SFTP. Do not credit Windows/POSIX system SSH, high-RTT, `MaxSessions=1`, product/settings/fallback wiring, tuple enablement, publication, or default behavior. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)<br>
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693)
@@ -17207,6 +17207,122 @@ config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 7/7 in 356 ms. The
publish/launch and every product/Beta/settings/fallback/tuple/publication/default path remain
absent; legacy remains the sole production/default path and SignPath remains deferred.
### E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001 — authenticated connection ownership and live OpenSSH boundary
- Date/owner/base: 2026-07-15, Codex implementation owner; exact clean evidence base `d11261706`
after `E-M6-SFTP-SESSION-ADAPTER-CI-001` closure.
- Audit commands: `sed -n '1,290p' src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts`,
`sed -n '1,280p' src/main/ssh/ssh-relay-runtime-sftp-session.ts`, relevant lifecycle ranges from
`ssh-connection.ts`, and searches for existing live-SSH/OpenSSH/workflow fixtures. Result: the raw
adapter and tree contracts compose without changing authentication or relay RPC, but no
non-test owner captures the exact `ssh2` transport that opened the raw SFTP channel or awaits
connection teardown if channel close exceeds its grace.
- Composition boundary: add one purpose-named module that accepts the existing authenticated
`SshConnection`, captures its exact current built-in `ssh2` client, opens exactly one signal-bound
raw SFTP session, rejects a connection-generation race after closing the returned channel, and
delegates to the existing session adapter and tree orchestrator. Force-close may destroy only the
captured client, must await that client's public `close` event with a bounded timeout, and must not
dispose the `SshConnection` or destroy a replacement client; the existing disconnect handler owns
reconnect state. System-SSH targets reject this composition without opening anything.
- Cancellation correction: current tree abort starts session close immediately. That is required to
release a retained raw callback, but it can race and prevent reverse cleanup when the active
callback settles normally. On abort, arm a short breaker below the 10-second cancellation-and-join
ceiling; when the stream reaches its catch boundary, cancel the breaker before reverse cleanup.
A genuinely retained callback is still released by timed session close, while normal cancellation
gets one opportunity to remove proved-owned paths before closing the session. Tests must prove
both orders and that no write begins after abort.
- Layer-A proof: purpose tests cover exact signal and connection identity, one SFTP open, system-SSH
rejection, connection replacement race, captured-client-only destroy/await-close, force-close
timeout/failure, normal owned-tree cancellation cleanup before close, and retained-callback breaker
settlement. Existing focused/broad/static/isolation gates remain mandatory.
- Layer-B proof: on target-native Ubuntu x64 and arm64 GitHub runners, start a loopback-only stock
OpenSSH `sshd` with key authentication and internal SFTP, then transfer the exact runtime produced
by that runner's build job. Validate the complete staged tree bytes/count/modes, exclusive-root
behavior, serial and four-file measurements, full-size incremental desktop RSS under 80 MiB,
cancellation settlement under 10 seconds, no later writes, and owned-stage cleanup. Record runner
image, architecture, OpenSSH version, runtime tuple/content/size/count, duration, RSS, and gaps.
- Explicit residual boundary: this package does not prove a Windows remote, cross-client matrix,
high-RTT comparison, `MaxSessions=1`, network interruption/disk/quota/inode/noexec/AV failures,
POSIX or Windows system-SSH transfer, remote bundled-Node verification/install/publish/launch,
product/Beta/settings/fallback wiring, tuple enablement, publication, or default behavior. Legacy
remains the sole production/default path and SignPath remains deferred.
### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-RED-001 — missing owner and cancellation ordering fail as audited
- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0,
pnpm 10.24.0, exact base `d11261706` plus audit/test/workflow-only diff. Exact Node 24 remains CI.
- Command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts
--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — EXPECTED FAIL: the five-case
connection suite collects zero tests because the purpose-named composition module is absent, and
the new ordinary-cancellation case observes the final owned-tree `rmdir` after session close
(event indices 23 versus 11). Nine prior tree cases pass. Vitest 1.87 seconds / 3.24 seconds wall,
141,524,992-byte maximum RSS. The temporary unhandled-rejection diagnostic came from attaching
the retained-callback test's rejection expectation after advancing fake time and was corrected in
the test before GREEN; it is not counted as product evidence.
- RED scope: tests and native workflow inclusion only. Protected resolver files remain untouched;
no owner module, live server, product importer, settings/fallback/tuple/default behavior, or
SignPath work exists.
### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001 — disconnected owner and live-runner oracle are locally green
- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0,
pnpm 10.24.0, atop exact evidence base `d11261706` plus this isolated diff. Exact Node 24 and stock
OpenSSH remain the native CI gate.
- Implementation: the 140-line connection module accepts only the existing authenticated built-in
connection boundary, captures the exact current `ssh2` client, opens one signal-bound raw SFTP
channel, and rejects/awaits cleanup if the connection changes during open. Its escalation destroys
only the captured client and awaits public client close for five seconds; a replacement client is
never touched, normal raw-session close leaves the connection open, and existing disconnect logic
retains reconnect ownership. No `SshConnection` auth/RPC/default behavior changes.
- Cancellation correction: the tree orchestrator arms a 250 ms raw-callback breaker on abort. If the
callback settles and the stream reaches its catch boundary, it cancels the breaker, reverse-cleans
proved-owned files/directories, then closes the session. If a callback remains retained, the
breaker closes the raw session to release the borrowed 64 KiB source buffer. Tests prove both
orders and no second write after abort; the overall 10-second cancellation-and-join ceiling is
unchanged.
- Purpose command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts
--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — PASS, 15/15 in 1.59 seconds Vitest /
2.68 seconds wall, 139,755,520-byte maximum RSS. Five owner cases cover exact signal/one open,
normal connection retention, system-SSH rejection, generation replacement cleanup, captured-only
force close, awaited raw/client close, and bounded missing-close failure. Ten tree cases include
ordinary cancellation cleanup before close and retained-callback breaker settlement.
- Focused command: connection, source tree/scan/stream, three SFTP boundaries, acquisition/cache
resolution/population/integration, live-suite skip oracle, and workflow contract — PASS, 14 files
plus one declared live-input skip; 189 passed / two declared skips in 14.22 seconds Vitest /
15.70 seconds wall, 172,146,688-byte maximum RSS.
- Broad commands: `src/main/ssh/ssh-relay-*.test.ts` — PASS, 46 files passed / four declared skipped,
603 tests passed / six declared skipped in 25.09 seconds Vitest / 27.10 seconds wall,
301,694,976-byte maximum RSS; `config/scripts/ssh-relay-runtime-*.test.mjs` — PASS, 50 files /
281 tests in 22.94 seconds Vitest / 24.87 seconds wall, 190,398,464-byte maximum RSS. Only the
established stale-lock diagnostic and local Node 26 module deprecation appear.
- Static/isolation commands: `pnpm typecheck` passes in 4.75 seconds wall with 1,249,492,992-byte
maximum RSS; targeted `oxlint`, targeted `oxfmt --check`, standalone max-lines ratchet,
`git diff --check`, protected-resolver empty diff, workflow YAML/oracle, and no non-test consumer
outside this disconnected module pass. Full `pnpm lint` passes in 17.44 seconds wall with
2,025,259,008-byte maximum RSS: switch exhaustiveness, 41 reliability gates, 355-entry max-lines
ratchet, bundled-skill verification, 9,837 localization references, locale parity, and zero-
candidate coverage pass with only pre-existing warnings. Module/purpose/live-test sizes are
140/170/311 lines, and the updated tree module/test are 271/426 lines, with no new bypass.
- Live runner oracle: the existing artifact workflow now triggers for runtime-transfer changes and
includes the five owner cases once in each native command family. Linux x64 and arm64 build jobs,
after producing and locally verifying the exact runtime, start a loopback-only stock OpenSSH
`sshd` with key authentication/internal SFTP. The purpose-named full-size suite scans the exact
identity/runtime, transfers it serially and with four files, validates every path/byte/hash/mode,
proves exclusive-root refusal leaves the tree intact, cancels after the first completed write,
requires cleanup/no later writes and sub-10-second settlement, enforces the 80 MiB desktop RSS
ceiling, and emits tuple/content/file/byte/server/runner/duration/RSS metrics. The fixture setup is
bounded and cleanup runs under `always()`.
- Local live result: deliberately SKIP, one case, because the required seven OpenSSH/runtime environment
inputs are absent. This proves the skip gate only and earns no live/server/filesystem/full-size
evidence. Exact-head Linux x64/arm64 CI is mandatory next.
- Residual gaps: no Windows remote, cross-client matrix, high-RTT comparison, `MaxSessions=1`, live
peak channel/open-handle instrumentation, interruption/disk/quota/inode/noexec/AV failure, POSIX
or Windows system-SSH path, remote bundled-Node verification/install/publish/launch, product/Beta/
settings/fallback wiring, tuple enablement, publication, or default behavior is proved. Legacy
remains the sole production/default path and SignPath remains deferred.
## Accepted Gaps
No product gap is accepted merely because it appears in this list. Each entry requires explicit
@@ -17265,12 +17381,10 @@ The project is not complete until every applicable item below is checked with ev
## Next Required Action
Commit/push the locally green Windows test-oracle correction, then require fresh all-six exact-head
native/full-size and adjacent CI before checking the package or advancing. The stream must create no
remote files or open SSH channels.
Do not open an SSH channel, transfer to or install on a remote, modify platform detection/selectors,
add an Electron/startup consumer, mode wiring, fallback, tuple enablement, release publication, or
default behavior.
Commit and push the locally green disconnected SFTP composition package, then require exact-head
target-native Linux x64/arm64 OpenSSH full-size evidence and all-six native/adjacent regression CI
before checking any Milestone 6 SFTP item or advancing. Do not add an Electron/startup/product
importer, per-target mode wiring, fallback, tuple enablement, release publication, or default behavior.
Keep Node upstream `.tar.xz` inputs, Windows ZIP, `ORCA_RELAY_PATH`, existing desktop required-assets
behavior, detached-signature byte encoding, Windows arm64 build 26100, macOS 13.5, Linux kernel 4.18,
release-cut, desktop builds, publication, and every tuple separately gated.
@@ -0,0 +1,170 @@
import { EventEmitter } from 'node:events'
import type { Client, SFTPWrapper } from 'ssh2'
import { describe, expect, it, vi } from 'vitest'
import type { SshConnection } from './ssh-connection'
import { openSshRelayRuntimeSftpTreeSessionForConnection } from './ssh-relay-runtime-sftp-connection-transfer'
type Callback = (error?: Error) => void
type ConnectionBoundary = Pick<SshConnection, 'getClient' | 'sftp' | 'usesSystemSshTransport'>
function createRawSession(options: { closeOnEnd?: boolean } = {}): SFTPWrapper {
const emitter = new EventEmitter()
const raw = Object.assign(emitter, {
mkdir: vi.fn((_path: string, _attributes: unknown, callback: Callback) => callback()),
rmdir: vi.fn((_path: string, callback: Callback) => callback()),
open: vi.fn(
(
_path: string,
_flags: string,
_attributes: unknown,
callback: (error: Error | undefined, handle: Buffer) => void
) => callback(undefined, Buffer.from('handle'))
),
write: vi.fn(
(
_handle: Buffer,
_buffer: Buffer,
_offset: number,
_length: number,
_position: number,
callback: Callback
) => callback()
),
fchmod: vi.fn((_handle: Buffer, _mode: number, callback: Callback) => callback()),
fstat: vi.fn(
(_handle: Buffer, callback: (error: Error | undefined, value: { mode: number }) => void) =>
callback(undefined, { mode: 0o100755 })
),
close: vi.fn((_handle: Buffer, callback: Callback) => callback()),
unlink: vi.fn((_path: string, callback: Callback) => callback()),
end: vi.fn(() => {
if (options.closeOnEnd) {
queueMicrotask(() => emitter.emit('close'))
}
})
})
return raw as unknown as SFTPWrapper
}
function createClient(onDestroy?: () => void): Client & { destroy: ReturnType<typeof vi.fn> } {
const emitter = new EventEmitter()
const client = Object.assign(emitter, {
destroy: vi.fn(() => onDestroy?.())
})
return client as unknown as Client & { destroy: ReturnType<typeof vi.fn> }
}
function createConnection(
client: Client | null,
raw: SFTPWrapper,
options: { systemSsh?: boolean; onSftp?: () => void } = {}
): ConnectionBoundary & {
setClient: (next: Client | null) => void
sftp: ReturnType<typeof vi.fn>
} {
let currentClient = client
const sftp = vi.fn(async (_signal?: AbortSignal) => {
options.onSftp?.()
return raw
})
return {
getClient: () => currentClient,
setClient: (next) => {
currentClient = next
},
usesSystemSshTransport: () => options.systemSsh ?? false,
sftp
}
}
describe('SSH relay runtime authenticated SFTP connection transfer', () => {
it('opens one raw session with the exact signal and leaves the connection open after normal close', async () => {
const raw = createRawSession({ closeOnEnd: true })
const client = createClient()
const connection = createConnection(client, raw)
const signal = new AbortController().signal
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal)
await session.close()
expect(connection.sftp).toHaveBeenCalledOnce()
expect(connection.sftp).toHaveBeenCalledWith(signal)
expect(raw.end).toHaveBeenCalledOnce()
expect(client.destroy).not.toHaveBeenCalled()
})
it('rejects system SSH without opening an SFTP channel', async () => {
const raw = createRawSession()
const connection = createConnection(null, raw, { systemSsh: true })
await expect(
openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal)
).rejects.toThrow(/built-in SSH/i)
expect(connection.sftp).not.toHaveBeenCalled()
})
it('closes a raw channel opened by a replaced connection generation', async () => {
const raw = createRawSession({ closeOnEnd: true })
const first = createClient()
const replacement = createClient()
const connection = createConnection(first, raw, {
onSftp: () => connection.setClient(replacement)
})
await expect(
openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal)
).rejects.toThrow(/connection changed/i)
expect(raw.end).toHaveBeenCalledOnce()
expect(first.destroy).not.toHaveBeenCalled()
expect(replacement.destroy).not.toHaveBeenCalled()
})
it('force-closes only the captured client and awaits both client and raw close', async () => {
vi.useFakeTimers()
try {
const raw = createRawSession()
const first = createClient(() => {
raw.emit('close')
first.emit('close')
})
const replacement = createClient()
const connection = createConnection(first, raw)
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(
connection,
new AbortController().signal
)
connection.setClient(replacement)
const close = session.close(new Error('cancelled transfer'))
await vi.advanceTimersByTimeAsync(5_000)
await close
expect(first.destroy).toHaveBeenCalledOnce()
expect(replacement.destroy).not.toHaveBeenCalled()
} finally {
vi.useRealTimers()
}
})
it('fails within the bounded force-close window when client teardown never confirms', async () => {
vi.useFakeTimers()
try {
const raw = createRawSession()
const client = createClient()
const connection = createConnection(client, raw)
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(
connection,
new AbortController().signal
)
const close = session.close()
const rejected = expect(close).rejects.toThrow(/connection close timed out/i)
await vi.advanceTimersByTimeAsync(10_000)
await rejected
expect(client.destroy).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
})
})
@@ -0,0 +1,140 @@
import type { Client, SFTPWrapper } from 'ssh2'
import type { SshConnection } from './ssh-connection'
import { openSshRelayRuntimeSftpTreeSession } from './ssh-relay-runtime-sftp-session'
import {
transferSshRelayRuntimeTreeViaSftp,
type SshRelayRuntimeSftpTreeSession,
type SshRelayRuntimeSftpTreeTransferOptions,
type SshRelayRuntimeSftpTreeTransferResult
} from './ssh-relay-runtime-sftp-tree-transfer'
const TRANSPORT_CLOSE_TIMEOUT_MS = 5_000
export type SshRelayRuntimeBuiltInSftpConnection = Pick<
SshConnection,
'getClient' | 'sftp' | 'usesSystemSshTransport'
>
export type SshRelayRuntimeSftpConnectionTransferOptions = Omit<
SshRelayRuntimeSftpTreeTransferOptions,
'openSession'
> &
Readonly<{ connection: SshRelayRuntimeBuiltInSftpConnection }>
function waitForClose(
emitter: Pick<NodeJS.EventEmitter, 'once' | 'removeListener'>,
beginClose: () => void,
timeoutMessage: string
): Promise<void> {
return new Promise((resolve, reject) => {
let settled = false
const finish = (error?: unknown): void => {
if (settled) {
return
}
settled = true
clearTimeout(timer)
emitter.removeListener('close', onClose)
if (error === undefined) {
resolve()
} else {
reject(error)
}
}
const onClose = (): void => finish()
const timer = setTimeout(() => finish(new Error(timeoutMessage)), TRANSPORT_CLOSE_TIMEOUT_MS)
emitter.once('close', onClose)
try {
beginClose()
} catch (error) {
finish(error)
}
})
}
async function forceCloseCapturedClient(client: Client): Promise<void> {
await waitForClose(
client,
() => client.destroy(),
'SSH relay runtime owning connection close timed out'
)
}
async function closeSessionFromReplacedConnection(raw: SFTPWrapper, client: Client): Promise<void> {
try {
await waitForClose(
raw,
() => raw.end(),
'SSH relay runtime replaced connection SFTP close timed out'
)
} catch (error) {
try {
await forceCloseCapturedClient(client)
} catch (forceError) {
throw new AggregateError(
[error, forceError],
'SSH relay runtime replaced connection cleanup failed'
)
}
throw error
}
}
function validateConnection(connection: SshRelayRuntimeBuiltInSftpConnection): void {
if (
!connection ||
typeof connection.getClient !== 'function' ||
typeof connection.sftp !== 'function' ||
typeof connection.usesSystemSshTransport !== 'function'
) {
throw new Error('SSH relay runtime SFTP connection boundary is invalid')
}
if (connection.usesSystemSshTransport()) {
throw new Error('SSH relay runtime SFTP transfer requires the built-in SSH transport')
}
}
export async function openSshRelayRuntimeSftpTreeSessionForConnection(
connection: SshRelayRuntimeBuiltInSftpConnection,
signal: AbortSignal
): Promise<SshRelayRuntimeSftpTreeSession> {
validateConnection(connection)
signal.throwIfAborted()
const capturedClient = connection.getClient()
if (!capturedClient) {
throw new Error('SSH relay runtime SFTP transfer requires a connected built-in SSH client')
}
return openSshRelayRuntimeSftpTreeSession({
signal,
openRawSession: async (exactSignal) => {
if (connection.getClient() !== capturedClient) {
throw new Error('SSH relay runtime SFTP connection changed before channel open')
}
const raw = await connection.sftp(exactSignal)
if (connection.getClient() !== capturedClient) {
// Why: never return a channel whose owning connection generation is no longer current.
await closeSessionFromReplacedConnection(raw, capturedClient)
throw new Error('SSH relay runtime SFTP connection changed during channel open')
}
return raw
},
// Why: escalation may tear down only the transport that owns the stuck raw callback.
forceCloseConnection: () => forceCloseCapturedClient(capturedClient)
})
}
export function transferSshRelayRuntimeTreeOverSftpConnection(
options: SshRelayRuntimeSftpConnectionTransferOptions
): Promise<SshRelayRuntimeSftpTreeTransferResult> {
const { connection, ...treeOptions } = options
return transferSshRelayRuntimeTreeViaSftp({
...treeOptions,
openSession: (signal) => openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal)
})
}
export const SSH_RELAY_RUNTIME_SFTP_CONNECTION_LIMITS = Object.freeze({
transportCloseTimeoutMs: TRANSPORT_CLOSE_TIMEOUT_MS
})
@@ -0,0 +1,311 @@
import { createHash } from 'node:crypto'
import { lstat, open, opendir, readFile, rm } from 'node:fs/promises'
import { join } from 'node:path'
import { performance } from 'node:perf_hooks'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } }))
import type { SshTarget } from '../../shared/ssh-types'
import { SshConnection } from './ssh-connection'
import { transferSshRelayRuntimeTreeOverSftpConnection } from './ssh-relay-runtime-sftp-connection-transfer'
import {
scanSshRelayRuntimeSourceTree,
type SshRelayRuntimeScannedSourceTree
} from './ssh-relay-runtime-source-scan'
import type { SshRelayRuntimeSourceTree } from './ssh-relay-runtime-source-tree'
import { SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS } from './ssh-relay-runtime-source-stream'
type MeasurementIdentity = Pick<
SshRelayRuntimeSourceTree,
'tupleId' | 'contentId' | 'os' | 'architecture'
> & {
entries: (
| { path: string; type: 'directory'; mode: 0o755 }
| {
path: string
type: 'file'
role: SshRelayRuntimeSourceTree['files'][number]['role']
size: number
mode: 0o644 | 0o755
sha256: SshRelayRuntimeSourceTree['files'][number]['sha256']
}
)[]
archive: { expandedSize: number; fileCount: number }
}
type Measurement<T> = {
result: T
elapsedMs: number
baselineRss: number
peakRss: number
incrementalRssBytes: number
}
const host = process.env.ORCA_SSH_RELAY_LIVE_SFTP_HOST
const user = process.env.ORCA_SSH_RELAY_LIVE_SFTP_USER
const identityFile = process.env.ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY
const remoteRoot = process.env.ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT
const runtimeRoot = process.env.ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT
const identityPath = process.env.ORCA_SSH_RELAY_FULL_SIZE_IDENTITY
const port = Number.parseInt(process.env.ORCA_SSH_RELAY_LIVE_SFTP_PORT ?? '', 10)
const hasLiveInput = Boolean(
host &&
user &&
identityFile &&
remoteRoot &&
runtimeRoot &&
identityPath &&
Number.isInteger(port)
)
function parseIdentity(value: unknown): MeasurementIdentity {
if (
typeof value !== 'object' ||
value === null ||
!('entries' in value) ||
!Array.isArray(value.entries) ||
!('archive' in value) ||
typeof value.archive !== 'object' ||
value.archive === null
) {
throw new Error('Live SFTP measurement identity is incomplete')
}
return value as MeasurementIdentity
}
function sourceTree(identity: MeasurementIdentity): SshRelayRuntimeSourceTree {
const directories = identity.entries
.filter((entry) => entry.type === 'directory')
.map((entry) => ({
...entry,
localPath: join(runtimeRoot as string, ...entry.path.split('/'))
}))
const files = identity.entries
.filter((entry) => entry.type === 'file')
.map((entry) => ({
...entry,
localPath: join(runtimeRoot as string, ...entry.path.split('/'))
}))
return Object.freeze({
tupleId: identity.tupleId,
contentId: identity.contentId,
releaseTag: 'measurement-only',
os: identity.os,
architecture: identity.architecture,
runtimeRoot: runtimeRoot as string,
directories,
files,
fileCount: identity.archive.fileCount,
expandedBytes: identity.archive.expandedSize,
assertLeaseOwned: async () => {}
})
}
async function measure<T>(operation: () => Promise<T>): Promise<Measurement<T>> {
const baselineRss = process.memoryUsage().rss
let peakRss = baselineRss
const sample = (): void => {
peakRss = Math.max(peakRss, process.memoryUsage().rss)
}
const sampler = setInterval(sample, 1)
const startedAt = performance.now()
try {
const result = await operation()
sample()
return {
result,
elapsedMs: performance.now() - startedAt,
baselineRss,
peakRss,
incrementalRssBytes: Math.max(0, peakRss - baselineRss)
}
} finally {
clearInterval(sampler)
}
}
async function hashFile(path: string): Promise<string> {
const handle = await open(path, 'r')
const digest = createHash('sha256')
const buffer = Buffer.allocUnsafe(64 * 1024)
let position = 0
try {
while (true) {
const { bytesRead } = await handle.read(buffer, 0, buffer.length, position)
if (bytesRead === 0) {
break
}
digest.update(buffer.subarray(0, bytesRead))
position += bytesRead
}
} finally {
await handle.close()
}
return `sha256:${digest.digest('hex')}`
}
async function collectRemotePaths(root: string): Promise<string[]> {
const paths: string[] = []
const pending = [{ localPath: root, relativePath: '' }]
while (pending.length > 0) {
const current = pending.pop()!
const directory = await opendir(current.localPath)
for await (const entry of directory) {
const relativePath = current.relativePath
? `${current.relativePath}/${entry.name}`
: entry.name
paths.push(relativePath)
if (entry.isDirectory()) {
pending.push({ localPath: join(current.localPath, entry.name), relativePath })
}
}
}
return paths.sort()
}
async function validateTransferredTree(
tree: SshRelayRuntimeScannedSourceTree,
stage: string
): Promise<void> {
const expectedPaths = [...tree.directories, ...tree.files].map((entry) => entry.path).sort()
// Why: the loopback runner can inspect sshd's filesystem without adding another proof channel.
expect(await collectRemotePaths(stage)).toEqual(expectedPaths)
for (const directory of tree.directories) {
const metadata = await lstat(join(stage, ...directory.path.split('/')))
expect(metadata.isDirectory()).toBe(true)
expect(metadata.mode & 0o777).toBe(directory.mode)
}
for (const file of tree.files) {
const path = join(stage, ...file.path.split('/'))
const metadata = await lstat(path)
expect(metadata.isFile()).toBe(true)
expect(metadata.size).toBe(file.size)
expect(metadata.mode & 0o777).toBe(file.mode)
expect(await hashFile(path)).toBe(file.sha256)
}
}
function stagePath(name: string, identity: MeasurementIdentity): string {
return join(remoteRoot as string, `${name}-${identity.contentId.slice('sha256:'.length, 23)}`)
}
async function transfer(
connection: SshConnection,
tree: SshRelayRuntimeScannedSourceTree,
stage: string,
maximumConcurrency: number,
signal = new AbortController().signal,
onProgress?: (bytes: number) => void
) {
return transferSshRelayRuntimeTreeOverSftpConnection({
connection,
tree,
remoteStagingRoot: stage,
enforcePosixMode: true,
maximumConcurrency,
signal,
onProgress: ({ bytesTransferred }) => onProgress?.(bytesTransferred)
})
}
describe.skipIf(!hasLiveInput)('SSH relay full-size transfer through live OpenSSH SFTP', () => {
it(
'preserves exact bytes and modes with bounded serial, concurrent, and cancellation behavior',
{ timeout: 20 * 60_000 },
async () => {
const identity = parseIdentity(JSON.parse(await readFile(identityPath as string, 'utf8')))
expect(identity.os).toBe('linux')
const tree = await scanSshRelayRuntimeSourceTree(
sourceTree(identity),
new AbortController().signal
)
const target: SshTarget = {
id: 'live-sftp-measurement',
label: 'live-sftp-measurement',
host: host as string,
port,
username: user as string,
identityFile: identityFile as string,
source: 'manual'
}
const connection = new SshConnection(target, { onStateChange: () => {} })
const serialStage = stagePath('serial', identity)
const concurrentStage = stagePath('concurrent', identity)
const cancelledStage = stagePath('cancelled', identity)
await Promise.all(
[serialStage, concurrentStage, cancelledStage].map((stage) =>
rm(stage, { recursive: true, force: true })
)
)
try {
await connection.connect()
const serial = await measure(() => transfer(connection, tree, serialStage, 1))
await validateTransferredTree(tree, serialStage)
const concurrent = await measure(() => transfer(connection, tree, concurrentStage, 4))
await validateTransferredTree(tree, concurrentStage)
await expect(transfer(connection, tree, concurrentStage, 1)).rejects.toBeTruthy()
await validateTransferredTree(tree, concurrentStage)
const controller = new AbortController()
let abortRequestedAt = 0
const cancelled = await measure(async () => {
const outcome = transfer(
connection,
tree,
cancelledStage,
4,
controller.signal,
(bytes) => {
if (bytes > 0 && !controller.signal.aborted) {
abortRequestedAt = performance.now()
controller.abort(new Error('live SFTP cancellation'))
}
}
)
await expect(outcome).rejects.toThrow('live SFTP cancellation')
return performance.now() - abortRequestedAt
})
await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' })
await new Promise((resolve) => setTimeout(resolve, 500))
await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' })
const metrics = {
tupleId: identity.tupleId,
contentId: identity.contentId,
files: tree.fileCount,
bytes: tree.expandedBytes,
serverVersion: process.env.ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION,
runnerImage: process.env.ImageOS,
runnerVersion: process.env.ImageVersion,
runnerArchitecture: process.env.RUNNER_ARCH,
serialElapsedMs: serial.elapsedMs,
serialIncrementalRssBytes: serial.incrementalRssBytes,
concurrentElapsedMs: concurrent.elapsedMs,
concurrentIncrementalRssBytes: concurrent.incrementalRssBytes,
cancellationSettlementMs: cancelled.result,
cancellationIncrementalRssBytes: cancelled.incrementalRssBytes
}
console.log(`ssh_relay_live_sftp=${JSON.stringify(metrics)}`)
expect(serial.result.bytesTransferred).toBe(tree.expandedBytes)
expect(concurrent.result.bytesTransferred).toBe(tree.expandedBytes)
expect(serial.incrementalRssBytes).toBeLessThanOrEqual(
SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes
)
expect(concurrent.incrementalRssBytes).toBeLessThanOrEqual(
SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes
)
expect(cancelled.result).toBeLessThan(10_000)
expect(connection.getState().status).toBe('connected')
} finally {
await connection.disconnect()
await Promise.all(
[serialStage, concurrentStage, cancelledStage].map((stage) =>
rm(stage, { recursive: true, force: true })
)
)
}
}
)
})
@@ -325,33 +325,68 @@ describe('SSH relay runtime SFTP tree transfer', () => {
})
it('closes the session to settle a retained write callback on cancellation', async () => {
vi.useFakeTimers()
try {
const tree = await treeFixture()
const { session } = createSession()
const controller = new AbortController()
let retainedCallback: Callback | undefined
let writes = 0
vi.mocked(session.operations.write).mockImplementationOnce(
(_handle, _buffer, _offset, _length, _position, callback) => {
writes += 1
retainedCallback = callback
}
)
vi.mocked(session.close).mockImplementationOnce(async () => {
retainedCallback?.(new Error('session closed'))
})
const transfer = transferSshRelayRuntimeTreeViaSftp({
tree,
remoteStagingRoot: '/staging/content',
enforcePosixMode: true,
maximumConcurrency: 1,
signal: controller.signal,
openSession: async () => session
})
await vi.waitFor(() => expect(writes).toBe(1))
controller.abort(new Error('cancelled'))
const rejected = expect(transfer).rejects.toThrow(/cancelled|session closed/)
await vi.advanceTimersByTimeAsync(250)
await rejected
expect(session.close).toHaveBeenCalledOnce()
expect(writes).toBe(1)
} finally {
vi.useRealTimers()
}
})
it('reverse-cleans owned paths before session close when cancellation is not callback-stuck', async () => {
const tree = await treeFixture()
const { session } = createSession()
const { session, events } = createSession()
const controller = new AbortController()
let retainedCallback: Callback | undefined
let writes = 0
vi.mocked(session.operations.write).mockImplementationOnce(
(_handle, _buffer, _offset, _length, _position, callback) => {
writes += 1
retainedCallback = callback
}
)
vi.mocked(session.close).mockImplementationOnce(async () => {
retainedCallback?.(new Error('session closed'))
})
const transfer = transferSshRelayRuntimeTreeViaSftp({
tree,
remoteStagingRoot: '/staging/content',
remoteStagingRoot: '/staging/cancelled-content',
enforcePosixMode: true,
maximumConcurrency: 1,
signal: controller.signal,
openSession: async () => session
openSession: async () => session,
onProgress: ({ bytesTransferred }) => {
writes += 1
if (bytesTransferred > 0) {
controller.abort(new Error('ordinary cancellation'))
}
}
})
await vi.waitFor(() => expect(writes).toBe(1))
controller.abort(new Error('cancelled'))
await expect(transfer).rejects.toThrow(/cancelled|session closed/)
expect(session.close).toHaveBeenCalledOnce()
await expect(transfer).rejects.toThrow('ordinary cancellation')
expect(events.findLastIndex((event) => event.startsWith('rmdir:'))).toBeLessThan(
events.indexOf('close-session')
)
expect(events).toContain('rmdir:/staging/cancelled-content')
expect(writes).toBe(1)
})
@@ -39,6 +39,7 @@ export type SshRelayRuntimeSftpTreeTransferResult = Readonly<
>
const MAXIMUM_CONCURRENT_FILES = 4
const ABORTED_CALLBACK_BREAKER_MS = 250
function waitForSftpCallback(register: (callback: SftpCallback) => void): Promise<void> {
return new Promise((resolve, reject) => {
@@ -169,13 +170,24 @@ export async function transferSshRelayRuntimeTreeViaSftp(
signal.throwIfAborted()
const session = await openSession(signal)
let closePromise: Promise<void> | undefined
let abortedCallbackBreaker: ReturnType<typeof setTimeout> | undefined
const closeSession = (reason?: unknown): Promise<void> => {
closePromise ??= Promise.resolve().then(() => session.close(reason))
return closePromise
}
const clearAbortedCallbackBreaker = (): void => {
clearTimeout(abortedCallbackBreaker)
abortedCallbackBreaker = undefined
}
const onAbort = (): void => {
// Why: closing the session is what releases callbacks and their borrowed source buffers.
void closeSession(signal.reason).catch(() => {})
if (abortedCallbackBreaker || closePromise) {
return
}
// Why: normal cancellation gets a cleanup opportunity, but a retained raw callback needs close.
abortedCallbackBreaker = setTimeout(() => {
abortedCallbackBreaker = undefined
void closeSession(signal.reason).catch(() => {})
}, ABORTED_CALLBACK_BREAKER_MS)
}
signal.addEventListener('abort', onAbort, { once: true })
if (signal.aborted) {
@@ -239,6 +251,8 @@ export async function transferSshRelayRuntimeTreeViaSftp(
signal.throwIfAborted()
return Object.freeze({ remoteStagingRoot, ...result })
} catch (error) {
// The stream reached a settled callback boundary, so reverse cleanup can safely own shutdown.
clearAbortedCallbackBreaker()
const cleanupFailures =
ownedDirectories.length === 0
? []
@@ -246,6 +260,12 @@ export async function transferSshRelayRuntimeTreeViaSftp(
await closeSession(error).catch((closeError) => cleanupFailures.push(closeError))
throw joinedFailure(error, cleanupFailures)
} finally {
clearAbortedCallbackBreaker()
signal.removeEventListener('abort', onAbort)
}
}
export const SSH_RELAY_RUNTIME_SFTP_TREE_TRANSFER_LIMITS = Object.freeze({
maximumConcurrentFiles: MAXIMUM_CONCURRENT_FILES,
abortedCallbackBreakerMs: ABORTED_CALLBACK_BREAKER_MS
})