From 32f78d720febdb5581282cfbc64b6aae374930f4 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Thu, 16 Jul 2026 11:52:28 -0400 Subject: [PATCH] feat: compose bounded SSH relay SFTP transfer Co-authored-by: Orca --- .../workflows/ssh-relay-runtime-artifacts.yml | 85 +++++ .../ssh-relay-runtime-workflow.test.mjs | 20 ++ ...elease-implementation-checklist-summary.md | 17 +- ...github-release-implementation-checklist.md | 130 +++++++- ...y-runtime-sftp-connection-transfer.test.ts | 170 ++++++++++ ...-relay-runtime-sftp-connection-transfer.ts | 140 ++++++++ ...lay-runtime-sftp-openssh-full-size.test.ts | 311 ++++++++++++++++++ ...h-relay-runtime-sftp-tree-transfer.test.ts | 69 +++- .../ssh-relay-runtime-sftp-tree-transfer.ts | 24 +- 9 files changed, 936 insertions(+), 30 deletions(-) create mode 100644 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts create mode 100644 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.ts create mode 100644 src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts diff --git a/.github/workflows/ssh-relay-runtime-artifacts.yml b/.github/workflows/ssh-relay-runtime-artifacts.yml index 60aaea0a288..6287cd1da6c 100644 --- a/.github/workflows/ssh-relay-runtime-artifacts.yml +++ b/.github/workflows/ssh-relay-runtime-artifacts.yml @@ -13,6 +13,7 @@ on: - 'package.json' - 'pnpm-lock.yaml' - 'src/main/ipc/parcel-watcher-process-entry.ts' + - 'src/main/ssh/ssh-relay-runtime-*' - 'src/relay/**' workflow_dispatch: # Why: callers may gate on the native graph only after separate release wiring is reviewed. @@ -286,6 +287,7 @@ jobs: src/main/ssh/ssh-relay-runtime-source-scan.test.ts \ src/main/ssh/ssh-relay-runtime-source-stream.test.ts \ src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts \ + src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts \ src/main/ssh/ssh-relay-runtime-sftp-session.test.ts \ src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts \ src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts \ @@ -553,6 +555,88 @@ jobs: src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts test ! -e "$extraction_output" + - name: Start loopback OpenSSH SFTP fixture + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + if ! test -x /usr/sbin/sshd; then + apt_options=(-o Acquire::Retries=3 -o Acquire::https::Timeout=15 -o Acquire::ForceIPv4=true) + timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" update -qq + timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" install -y -qq openssh-server + fi + fixture="$RUNNER_TEMP/orca-live-sftp" + mkdir -p "$fixture" "$HOME/.ssh" "$fixture/remote" + chmod 700 "$HOME/.ssh" "$fixture" + ssh-keygen -q -t ed25519 -N '' -f "$fixture/client-key" + ssh-keygen -q -t ed25519 -N '' -f "$fixture/host-key" + cp "$fixture/client-key.pub" "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" "$fixture/client-key" "$fixture/host-key" + sudo chown root:root "$fixture/host-key" + port=22222 + server_version=$(/usr/sbin/sshd -V 2>&1 | head -1) + config="$fixture/sshd_config" + printf '%s\n' \ + "Port $port" \ + 'ListenAddress 127.0.0.1' \ + "HostKey $fixture/host-key" \ + "PidFile $fixture/sshd.pid" \ + "AuthorizedKeysFile $HOME/.ssh/authorized_keys" \ + "AllowUsers $USER" \ + 'PasswordAuthentication no' \ + 'KbdInteractiveAuthentication no' \ + 'ChallengeResponseAuthentication no' \ + 'UsePAM no' \ + 'StrictModes yes' \ + 'LogLevel VERBOSE' \ + 'Subsystem sftp internal-sftp' > "$config" + sudo mkdir -p /run/sshd + sudo /usr/sbin/sshd -f "$config" -E "$fixture/sshd.log" + for attempt in {1..20}; do + if ssh -F /dev/null -i "$fixture/client-key" -p "$port" \ + -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "$USER@127.0.0.1" true; then + break + fi + if test "$attempt" = 20; then + cat "$fixture/sshd.log" + exit 1 + fi + sleep 0.25 + done + { + printf 'ORCA_SSH_RELAY_LIVE_SFTP_HOST=127.0.0.1\n' + printf 'ORCA_SSH_RELAY_LIVE_SFTP_PORT=%s\n' "$port" + printf 'ORCA_SSH_RELAY_LIVE_SFTP_USER=%s\n' "$USER" + printf 'ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY=%s\n' "$fixture/client-key" + printf 'ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT=%s\n' "$fixture/remote" + printf 'ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION=%s\n' "$server_version" + } >> "$GITHUB_ENV" + + - name: Measure exact full-size runtime over live OpenSSH SFTP + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + first_output="$RUNNER_TEMP/ssh-relay-runtime/${{ matrix.tuple }}/first" + ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT="$first_output/runtime" \ + ORCA_SSH_RELAY_FULL_SIZE_IDENTITY="$first_output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json" \ + pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 --reporter=verbose \ + src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts + + - name: Stop loopback OpenSSH SFTP fixture + if: always() && runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + fixture="$RUNNER_TEMP/orca-live-sftp" + if test -s "$fixture/sshd.pid"; then + sudo kill "$(cat "$fixture/sshd.pid")" || true + fi + if test -f "$fixture/sshd.log"; then + sudo tail -200 "$fixture/sshd.log" + fi + - name: Upload unpublished artifact evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -844,6 +928,7 @@ jobs: src/main/ssh/ssh-relay-runtime-source-scan.test.ts ` src/main/ssh/ssh-relay-runtime-source-stream.test.ts ` src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts ` + src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts ` src/main/ssh/ssh-relay-runtime-sftp-session.test.ts ` src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts ` src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts ` diff --git a/config/scripts/ssh-relay-runtime-workflow.test.mjs b/config/scripts/ssh-relay-runtime-workflow.test.mjs index 990f4ae5fa7..97bf9a9a7aa 100644 --- a/config/scripts/ssh-relay-runtime-workflow.test.mjs +++ b/config/scripts/ssh-relay-runtime-workflow.test.mjs @@ -83,6 +83,25 @@ describe('SSH relay runtime artifact workflow', () => { ) }) + it('runs exact full-size Linux runtime transfer against loopback-only stock OpenSSH', async () => { + const workflow = parse(await readFile(workflowUrl, 'utf8')) + const steps = workflow.jobs['build-posix-runtime'].steps + const start = steps.find((step) => step.name === 'Start loopback OpenSSH SFTP fixture') + const measure = steps.find( + (step) => step.name === 'Measure exact full-size runtime over live OpenSSH SFTP' + ) + const stop = steps.find((step) => step.name === 'Stop loopback OpenSSH SFTP fixture') + + expect(start.if).toBe("runner.os == 'Linux'") + expect(start.run).toContain('ListenAddress 127.0.0.1') + expect(start.run).toContain('Subsystem sftp internal-sftp') + expect(start.run).toContain('timeout --signal=TERM --kill-after=15s 180s sudo apt-get') + expect(measure.if).toBe("runner.os == 'Linux'") + expect(measure.run).toContain('ssh-relay-runtime-sftp-openssh-full-size.test.ts') + expect(measure.run).toContain('first_output/runtime') + expect(stop.if).toBe("always() && runner.os == 'Linux'") + }) + it('uploads only the first output after two clean builds verify and compare', async () => { const source = await readFile(workflowUrl, 'utf8') const workflow = parse(source) @@ -144,6 +163,7 @@ describe('SSH relay runtime artifact workflow', () => { 'ssh-relay-runtime-source-scan.test.ts', 'ssh-relay-runtime-source-stream.test.ts', 'ssh-relay-runtime-sftp-file-destination.test.ts', + 'ssh-relay-runtime-sftp-connection-transfer.test.ts', 'ssh-relay-runtime-sftp-session.test.ts', 'ssh-relay-runtime-sftp-tree-transfer.test.ts', 'ssh-relay-compiled-manifest-trust.test.ts', diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md index 72eea955141..1f38d7ffc10 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md @@ -9,9 +9,20 @@ keeps commands, hashes, runner identities, timings, and failure details. A checked box means the work has evidence in the detailed ledger. Design approval alone does not complete a box. -Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected raw `ssh2` -session adapter is exact-head green; live adapter/tree composition audit is next, 2026-07-15, Codex -implementation owner.** `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both +Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected live +OpenSSH SFTP composition is in progress, 2026-07-15, Codex implementation owner.** +`E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001` limits this package to one authenticated built-in +`SshConnection`, one raw SFTP channel, the existing one-to-four file transfer, captured-transport- +only force close with awaited teardown, and exact full-size Linux x64/arm64 runtime transfer against +target-native OpenSSH runners. It also requires ordinary cancellation cleanup to run before a short +retained-callback breaker closes the session. No product/default, system-SSH, Windows-remote, +high-RTT, `MaxSessions=1`, or tuple-enablement claim is made. SignPath remains deferred. +`E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` is locally green: 15/15 purpose cases, 189 focused cases +with two declared skips, 603 broad relay cases with six declared skips, 281/281 release contracts, +typecheck, full lint, formatting, max-lines, diff, protected-resolver, and isolation gates pass. The +full-size live suite is deliberately skipped without runner OpenSSH/runtime inputs, so exact-head +Linux x64/arm64 live CI remains the next gate. +`E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both Linux supplements, Windows x64 floor, PR Checks, Golden E2E, and computer-use on exact commit `fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100 rejection after complete runtime smoke. No live SFTP/tree or product/default claim is made. diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md index 4eb155a3c41..8df452fb05e 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md @@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps Date created: 2026-07-14
Last updated: 2026-07-15
-Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected raw `ssh2` session adapter is exact-head green; live adapter/tree composition audit is next**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.
-Session checkpoint: **RAW SFTP SESSION ADAPTER EXACT-HEAD GREEN — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all-six primary native, both Linux supplement, Windows x64 floor, PR Checks, Golden E2E, and computer-use proof for exact implementation commit `fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100 rejection after complete runtime smoke. Audit the smallest disconnected live adapter/tree composition next, including a concrete real connection-force-close owner and OpenSSH SFTP server. Do not credit live transfer, server filesystem/modes, full-size network metrics, product/settings/fallback wiring, tuple enablement, publication, or default behavior from native callback mocks. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.
+Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected authenticated-connection/SFTP-tree composition and live OpenSSH proof**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.
+Session checkpoint: **LIVE SFTP COMPOSITION LOCALLY GREEN / EXACT-HEAD RUNNER PROOF NEXT — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` proves the disconnected authenticated-connection owner, captured-client force close, cancellation cleanup/breaker ordering, workflow oracle, focused/broad/release suites, typecheck, full lint, formatting, max-lines, diff, protected-resolver, and no-product-import gates locally. The purpose suite is 15/15; the live full-size suite is deliberately skipped locally because no runner OpenSSH/runtime inputs are present and therefore earns no live cell. Push this isolated package and require exact-head Linux x64/arm64 stock-OpenSSH full-size metrics before crediting live SFTP. Do not credit Windows/POSIX system SSH, high-RTT, `MaxSessions=1`, product/settings/fallback wiring, tuple enablement, publication, or default behavior. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693) @@ -17207,6 +17207,122 @@ config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 7/7 in 356 ms. The publish/launch and every product/Beta/settings/fallback/tuple/publication/default path remain absent; legacy remains the sole production/default path and SignPath remains deferred. +### E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001 — authenticated connection ownership and live OpenSSH boundary + +- Date/owner/base: 2026-07-15, Codex implementation owner; exact clean evidence base `d11261706` + after `E-M6-SFTP-SESSION-ADAPTER-CI-001` closure. +- Audit commands: `sed -n '1,290p' src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts`, + `sed -n '1,280p' src/main/ssh/ssh-relay-runtime-sftp-session.ts`, relevant lifecycle ranges from + `ssh-connection.ts`, and searches for existing live-SSH/OpenSSH/workflow fixtures. Result: the raw + adapter and tree contracts compose without changing authentication or relay RPC, but no + non-test owner captures the exact `ssh2` transport that opened the raw SFTP channel or awaits + connection teardown if channel close exceeds its grace. +- Composition boundary: add one purpose-named module that accepts the existing authenticated + `SshConnection`, captures its exact current built-in `ssh2` client, opens exactly one signal-bound + raw SFTP session, rejects a connection-generation race after closing the returned channel, and + delegates to the existing session adapter and tree orchestrator. Force-close may destroy only the + captured client, must await that client's public `close` event with a bounded timeout, and must not + dispose the `SshConnection` or destroy a replacement client; the existing disconnect handler owns + reconnect state. System-SSH targets reject this composition without opening anything. +- Cancellation correction: current tree abort starts session close immediately. That is required to + release a retained raw callback, but it can race and prevent reverse cleanup when the active + callback settles normally. On abort, arm a short breaker below the 10-second cancellation-and-join + ceiling; when the stream reaches its catch boundary, cancel the breaker before reverse cleanup. + A genuinely retained callback is still released by timed session close, while normal cancellation + gets one opportunity to remove proved-owned paths before closing the session. Tests must prove + both orders and that no write begins after abort. +- Layer-A proof: purpose tests cover exact signal and connection identity, one SFTP open, system-SSH + rejection, connection replacement race, captured-client-only destroy/await-close, force-close + timeout/failure, normal owned-tree cancellation cleanup before close, and retained-callback breaker + settlement. Existing focused/broad/static/isolation gates remain mandatory. +- Layer-B proof: on target-native Ubuntu x64 and arm64 GitHub runners, start a loopback-only stock + OpenSSH `sshd` with key authentication and internal SFTP, then transfer the exact runtime produced + by that runner's build job. Validate the complete staged tree bytes/count/modes, exclusive-root + behavior, serial and four-file measurements, full-size incremental desktop RSS under 80 MiB, + cancellation settlement under 10 seconds, no later writes, and owned-stage cleanup. Record runner + image, architecture, OpenSSH version, runtime tuple/content/size/count, duration, RSS, and gaps. +- Explicit residual boundary: this package does not prove a Windows remote, cross-client matrix, + high-RTT comparison, `MaxSessions=1`, network interruption/disk/quota/inode/noexec/AV failures, + POSIX or Windows system-SSH transfer, remote bundled-Node verification/install/publish/launch, + product/Beta/settings/fallback wiring, tuple enablement, publication, or default behavior. Legacy + remains the sole production/default path and SignPath remains deferred. + +### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-RED-001 — missing owner and cancellation ordering fail as audited + +- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0, + pnpm 10.24.0, exact base `d11261706` plus audit/test/workflow-only diff. Exact Node 24 remains CI. +- Command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts +--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts +src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — EXPECTED FAIL: the five-case + connection suite collects zero tests because the purpose-named composition module is absent, and + the new ordinary-cancellation case observes the final owned-tree `rmdir` after session close + (event indices 23 versus 11). Nine prior tree cases pass. Vitest 1.87 seconds / 3.24 seconds wall, + 141,524,992-byte maximum RSS. The temporary unhandled-rejection diagnostic came from attaching + the retained-callback test's rejection expectation after advancing fake time and was corrected in + the test before GREEN; it is not counted as product evidence. +- RED scope: tests and native workflow inclusion only. Protected resolver files remain untouched; + no owner module, live server, product importer, settings/fallback/tuple/default behavior, or + SignPath work exists. + +### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001 — disconnected owner and live-runner oracle are locally green + +- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0, + pnpm 10.24.0, atop exact evidence base `d11261706` plus this isolated diff. Exact Node 24 and stock + OpenSSH remain the native CI gate. +- Implementation: the 140-line connection module accepts only the existing authenticated built-in + connection boundary, captures the exact current `ssh2` client, opens one signal-bound raw SFTP + channel, and rejects/awaits cleanup if the connection changes during open. Its escalation destroys + only the captured client and awaits public client close for five seconds; a replacement client is + never touched, normal raw-session close leaves the connection open, and existing disconnect logic + retains reconnect ownership. No `SshConnection` auth/RPC/default behavior changes. +- Cancellation correction: the tree orchestrator arms a 250 ms raw-callback breaker on abort. If the + callback settles and the stream reaches its catch boundary, it cancels the breaker, reverse-cleans + proved-owned files/directories, then closes the session. If a callback remains retained, the + breaker closes the raw session to release the borrowed 64 KiB source buffer. Tests prove both + orders and no second write after abort; the overall 10-second cancellation-and-join ceiling is + unchanged. +- Purpose command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts +--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts +src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — PASS, 15/15 in 1.59 seconds Vitest / + 2.68 seconds wall, 139,755,520-byte maximum RSS. Five owner cases cover exact signal/one open, + normal connection retention, system-SSH rejection, generation replacement cleanup, captured-only + force close, awaited raw/client close, and bounded missing-close failure. Ten tree cases include + ordinary cancellation cleanup before close and retained-callback breaker settlement. +- Focused command: connection, source tree/scan/stream, three SFTP boundaries, acquisition/cache + resolution/population/integration, live-suite skip oracle, and workflow contract — PASS, 14 files + plus one declared live-input skip; 189 passed / two declared skips in 14.22 seconds Vitest / + 15.70 seconds wall, 172,146,688-byte maximum RSS. +- Broad commands: `src/main/ssh/ssh-relay-*.test.ts` — PASS, 46 files passed / four declared skipped, + 603 tests passed / six declared skipped in 25.09 seconds Vitest / 27.10 seconds wall, + 301,694,976-byte maximum RSS; `config/scripts/ssh-relay-runtime-*.test.mjs` — PASS, 50 files / + 281 tests in 22.94 seconds Vitest / 24.87 seconds wall, 190,398,464-byte maximum RSS. Only the + established stale-lock diagnostic and local Node 26 module deprecation appear. +- Static/isolation commands: `pnpm typecheck` passes in 4.75 seconds wall with 1,249,492,992-byte + maximum RSS; targeted `oxlint`, targeted `oxfmt --check`, standalone max-lines ratchet, + `git diff --check`, protected-resolver empty diff, workflow YAML/oracle, and no non-test consumer + outside this disconnected module pass. Full `pnpm lint` passes in 17.44 seconds wall with + 2,025,259,008-byte maximum RSS: switch exhaustiveness, 41 reliability gates, 355-entry max-lines + ratchet, bundled-skill verification, 9,837 localization references, locale parity, and zero- + candidate coverage pass with only pre-existing warnings. Module/purpose/live-test sizes are + 140/170/311 lines, and the updated tree module/test are 271/426 lines, with no new bypass. +- Live runner oracle: the existing artifact workflow now triggers for runtime-transfer changes and + includes the five owner cases once in each native command family. Linux x64 and arm64 build jobs, + after producing and locally verifying the exact runtime, start a loopback-only stock OpenSSH + `sshd` with key authentication/internal SFTP. The purpose-named full-size suite scans the exact + identity/runtime, transfers it serially and with four files, validates every path/byte/hash/mode, + proves exclusive-root refusal leaves the tree intact, cancels after the first completed write, + requires cleanup/no later writes and sub-10-second settlement, enforces the 80 MiB desktop RSS + ceiling, and emits tuple/content/file/byte/server/runner/duration/RSS metrics. The fixture setup is + bounded and cleanup runs under `always()`. +- Local live result: deliberately SKIP, one case, because the required seven OpenSSH/runtime environment + inputs are absent. This proves the skip gate only and earns no live/server/filesystem/full-size + evidence. Exact-head Linux x64/arm64 CI is mandatory next. +- Residual gaps: no Windows remote, cross-client matrix, high-RTT comparison, `MaxSessions=1`, live + peak channel/open-handle instrumentation, interruption/disk/quota/inode/noexec/AV failure, POSIX + or Windows system-SSH path, remote bundled-Node verification/install/publish/launch, product/Beta/ + settings/fallback wiring, tuple enablement, publication, or default behavior is proved. Legacy + remains the sole production/default path and SignPath remains deferred. + ## Accepted Gaps No product gap is accepted merely because it appears in this list. Each entry requires explicit @@ -17265,12 +17381,10 @@ The project is not complete until every applicable item below is checked with ev ## Next Required Action -Commit/push the locally green Windows test-oracle correction, then require fresh all-six exact-head -native/full-size and adjacent CI before checking the package or advancing. The stream must create no -remote files or open SSH channels. -Do not open an SSH channel, transfer to or install on a remote, modify platform detection/selectors, -add an Electron/startup consumer, mode wiring, fallback, tuple enablement, release publication, or -default behavior. +Commit and push the locally green disconnected SFTP composition package, then require exact-head +target-native Linux x64/arm64 OpenSSH full-size evidence and all-six native/adjacent regression CI +before checking any Milestone 6 SFTP item or advancing. Do not add an Electron/startup/product +importer, per-target mode wiring, fallback, tuple enablement, release publication, or default behavior. Keep Node upstream `.tar.xz` inputs, Windows ZIP, `ORCA_RELAY_PATH`, existing desktop required-assets behavior, detached-signature byte encoding, Windows arm64 build 26100, macOS 13.5, Linux kernel 4.18, release-cut, desktop builds, publication, and every tuple separately gated. diff --git a/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts b/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts new file mode 100644 index 00000000000..c382cfe2f46 --- /dev/null +++ b/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts @@ -0,0 +1,170 @@ +import { EventEmitter } from 'node:events' + +import type { Client, SFTPWrapper } from 'ssh2' +import { describe, expect, it, vi } from 'vitest' + +import type { SshConnection } from './ssh-connection' +import { openSshRelayRuntimeSftpTreeSessionForConnection } from './ssh-relay-runtime-sftp-connection-transfer' + +type Callback = (error?: Error) => void +type ConnectionBoundary = Pick + +function createRawSession(options: { closeOnEnd?: boolean } = {}): SFTPWrapper { + const emitter = new EventEmitter() + const raw = Object.assign(emitter, { + mkdir: vi.fn((_path: string, _attributes: unknown, callback: Callback) => callback()), + rmdir: vi.fn((_path: string, callback: Callback) => callback()), + open: vi.fn( + ( + _path: string, + _flags: string, + _attributes: unknown, + callback: (error: Error | undefined, handle: Buffer) => void + ) => callback(undefined, Buffer.from('handle')) + ), + write: vi.fn( + ( + _handle: Buffer, + _buffer: Buffer, + _offset: number, + _length: number, + _position: number, + callback: Callback + ) => callback() + ), + fchmod: vi.fn((_handle: Buffer, _mode: number, callback: Callback) => callback()), + fstat: vi.fn( + (_handle: Buffer, callback: (error: Error | undefined, value: { mode: number }) => void) => + callback(undefined, { mode: 0o100755 }) + ), + close: vi.fn((_handle: Buffer, callback: Callback) => callback()), + unlink: vi.fn((_path: string, callback: Callback) => callback()), + end: vi.fn(() => { + if (options.closeOnEnd) { + queueMicrotask(() => emitter.emit('close')) + } + }) + }) + return raw as unknown as SFTPWrapper +} + +function createClient(onDestroy?: () => void): Client & { destroy: ReturnType } { + const emitter = new EventEmitter() + const client = Object.assign(emitter, { + destroy: vi.fn(() => onDestroy?.()) + }) + return client as unknown as Client & { destroy: ReturnType } +} + +function createConnection( + client: Client | null, + raw: SFTPWrapper, + options: { systemSsh?: boolean; onSftp?: () => void } = {} +): ConnectionBoundary & { + setClient: (next: Client | null) => void + sftp: ReturnType +} { + let currentClient = client + const sftp = vi.fn(async (_signal?: AbortSignal) => { + options.onSftp?.() + return raw + }) + return { + getClient: () => currentClient, + setClient: (next) => { + currentClient = next + }, + usesSystemSshTransport: () => options.systemSsh ?? false, + sftp + } +} + +describe('SSH relay runtime authenticated SFTP connection transfer', () => { + it('opens one raw session with the exact signal and leaves the connection open after normal close', async () => { + const raw = createRawSession({ closeOnEnd: true }) + const client = createClient() + const connection = createConnection(client, raw) + const signal = new AbortController().signal + + const session = await openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal) + await session.close() + + expect(connection.sftp).toHaveBeenCalledOnce() + expect(connection.sftp).toHaveBeenCalledWith(signal) + expect(raw.end).toHaveBeenCalledOnce() + expect(client.destroy).not.toHaveBeenCalled() + }) + + it('rejects system SSH without opening an SFTP channel', async () => { + const raw = createRawSession() + const connection = createConnection(null, raw, { systemSsh: true }) + + await expect( + openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal) + ).rejects.toThrow(/built-in SSH/i) + expect(connection.sftp).not.toHaveBeenCalled() + }) + + it('closes a raw channel opened by a replaced connection generation', async () => { + const raw = createRawSession({ closeOnEnd: true }) + const first = createClient() + const replacement = createClient() + const connection = createConnection(first, raw, { + onSftp: () => connection.setClient(replacement) + }) + + await expect( + openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal) + ).rejects.toThrow(/connection changed/i) + expect(raw.end).toHaveBeenCalledOnce() + expect(first.destroy).not.toHaveBeenCalled() + expect(replacement.destroy).not.toHaveBeenCalled() + }) + + it('force-closes only the captured client and awaits both client and raw close', async () => { + vi.useFakeTimers() + try { + const raw = createRawSession() + const first = createClient(() => { + raw.emit('close') + first.emit('close') + }) + const replacement = createClient() + const connection = createConnection(first, raw) + const session = await openSshRelayRuntimeSftpTreeSessionForConnection( + connection, + new AbortController().signal + ) + connection.setClient(replacement) + const close = session.close(new Error('cancelled transfer')) + + await vi.advanceTimersByTimeAsync(5_000) + await close + expect(first.destroy).toHaveBeenCalledOnce() + expect(replacement.destroy).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('fails within the bounded force-close window when client teardown never confirms', async () => { + vi.useFakeTimers() + try { + const raw = createRawSession() + const client = createClient() + const connection = createConnection(client, raw) + const session = await openSshRelayRuntimeSftpTreeSessionForConnection( + connection, + new AbortController().signal + ) + const close = session.close() + const rejected = expect(close).rejects.toThrow(/connection close timed out/i) + + await vi.advanceTimersByTimeAsync(10_000) + await rejected + expect(client.destroy).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.ts b/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.ts new file mode 100644 index 00000000000..cb7193fd1aa --- /dev/null +++ b/src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.ts @@ -0,0 +1,140 @@ +import type { Client, SFTPWrapper } from 'ssh2' + +import type { SshConnection } from './ssh-connection' +import { openSshRelayRuntimeSftpTreeSession } from './ssh-relay-runtime-sftp-session' +import { + transferSshRelayRuntimeTreeViaSftp, + type SshRelayRuntimeSftpTreeSession, + type SshRelayRuntimeSftpTreeTransferOptions, + type SshRelayRuntimeSftpTreeTransferResult +} from './ssh-relay-runtime-sftp-tree-transfer' + +const TRANSPORT_CLOSE_TIMEOUT_MS = 5_000 + +export type SshRelayRuntimeBuiltInSftpConnection = Pick< + SshConnection, + 'getClient' | 'sftp' | 'usesSystemSshTransport' +> + +export type SshRelayRuntimeSftpConnectionTransferOptions = Omit< + SshRelayRuntimeSftpTreeTransferOptions, + 'openSession' +> & + Readonly<{ connection: SshRelayRuntimeBuiltInSftpConnection }> + +function waitForClose( + emitter: Pick, + beginClose: () => void, + timeoutMessage: string +): Promise { + return new Promise((resolve, reject) => { + let settled = false + const finish = (error?: unknown): void => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + emitter.removeListener('close', onClose) + if (error === undefined) { + resolve() + } else { + reject(error) + } + } + const onClose = (): void => finish() + const timer = setTimeout(() => finish(new Error(timeoutMessage)), TRANSPORT_CLOSE_TIMEOUT_MS) + emitter.once('close', onClose) + try { + beginClose() + } catch (error) { + finish(error) + } + }) +} + +async function forceCloseCapturedClient(client: Client): Promise { + await waitForClose( + client, + () => client.destroy(), + 'SSH relay runtime owning connection close timed out' + ) +} + +async function closeSessionFromReplacedConnection(raw: SFTPWrapper, client: Client): Promise { + try { + await waitForClose( + raw, + () => raw.end(), + 'SSH relay runtime replaced connection SFTP close timed out' + ) + } catch (error) { + try { + await forceCloseCapturedClient(client) + } catch (forceError) { + throw new AggregateError( + [error, forceError], + 'SSH relay runtime replaced connection cleanup failed' + ) + } + throw error + } +} + +function validateConnection(connection: SshRelayRuntimeBuiltInSftpConnection): void { + if ( + !connection || + typeof connection.getClient !== 'function' || + typeof connection.sftp !== 'function' || + typeof connection.usesSystemSshTransport !== 'function' + ) { + throw new Error('SSH relay runtime SFTP connection boundary is invalid') + } + if (connection.usesSystemSshTransport()) { + throw new Error('SSH relay runtime SFTP transfer requires the built-in SSH transport') + } +} + +export async function openSshRelayRuntimeSftpTreeSessionForConnection( + connection: SshRelayRuntimeBuiltInSftpConnection, + signal: AbortSignal +): Promise { + validateConnection(connection) + signal.throwIfAborted() + const capturedClient = connection.getClient() + if (!capturedClient) { + throw new Error('SSH relay runtime SFTP transfer requires a connected built-in SSH client') + } + + return openSshRelayRuntimeSftpTreeSession({ + signal, + openRawSession: async (exactSignal) => { + if (connection.getClient() !== capturedClient) { + throw new Error('SSH relay runtime SFTP connection changed before channel open') + } + const raw = await connection.sftp(exactSignal) + if (connection.getClient() !== capturedClient) { + // Why: never return a channel whose owning connection generation is no longer current. + await closeSessionFromReplacedConnection(raw, capturedClient) + throw new Error('SSH relay runtime SFTP connection changed during channel open') + } + return raw + }, + // Why: escalation may tear down only the transport that owns the stuck raw callback. + forceCloseConnection: () => forceCloseCapturedClient(capturedClient) + }) +} + +export function transferSshRelayRuntimeTreeOverSftpConnection( + options: SshRelayRuntimeSftpConnectionTransferOptions +): Promise { + const { connection, ...treeOptions } = options + return transferSshRelayRuntimeTreeViaSftp({ + ...treeOptions, + openSession: (signal) => openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal) + }) +} + +export const SSH_RELAY_RUNTIME_SFTP_CONNECTION_LIMITS = Object.freeze({ + transportCloseTimeoutMs: TRANSPORT_CLOSE_TIMEOUT_MS +}) diff --git a/src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts b/src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts new file mode 100644 index 00000000000..20a18db9e63 --- /dev/null +++ b/src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts @@ -0,0 +1,311 @@ +import { createHash } from 'node:crypto' +import { lstat, open, opendir, readFile, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' + +import { describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } })) + +import type { SshTarget } from '../../shared/ssh-types' +import { SshConnection } from './ssh-connection' +import { transferSshRelayRuntimeTreeOverSftpConnection } from './ssh-relay-runtime-sftp-connection-transfer' +import { + scanSshRelayRuntimeSourceTree, + type SshRelayRuntimeScannedSourceTree +} from './ssh-relay-runtime-source-scan' +import type { SshRelayRuntimeSourceTree } from './ssh-relay-runtime-source-tree' +import { SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS } from './ssh-relay-runtime-source-stream' + +type MeasurementIdentity = Pick< + SshRelayRuntimeSourceTree, + 'tupleId' | 'contentId' | 'os' | 'architecture' +> & { + entries: ( + | { path: string; type: 'directory'; mode: 0o755 } + | { + path: string + type: 'file' + role: SshRelayRuntimeSourceTree['files'][number]['role'] + size: number + mode: 0o644 | 0o755 + sha256: SshRelayRuntimeSourceTree['files'][number]['sha256'] + } + )[] + archive: { expandedSize: number; fileCount: number } +} + +type Measurement = { + result: T + elapsedMs: number + baselineRss: number + peakRss: number + incrementalRssBytes: number +} + +const host = process.env.ORCA_SSH_RELAY_LIVE_SFTP_HOST +const user = process.env.ORCA_SSH_RELAY_LIVE_SFTP_USER +const identityFile = process.env.ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY +const remoteRoot = process.env.ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT +const runtimeRoot = process.env.ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT +const identityPath = process.env.ORCA_SSH_RELAY_FULL_SIZE_IDENTITY +const port = Number.parseInt(process.env.ORCA_SSH_RELAY_LIVE_SFTP_PORT ?? '', 10) +const hasLiveInput = Boolean( + host && + user && + identityFile && + remoteRoot && + runtimeRoot && + identityPath && + Number.isInteger(port) +) + +function parseIdentity(value: unknown): MeasurementIdentity { + if ( + typeof value !== 'object' || + value === null || + !('entries' in value) || + !Array.isArray(value.entries) || + !('archive' in value) || + typeof value.archive !== 'object' || + value.archive === null + ) { + throw new Error('Live SFTP measurement identity is incomplete') + } + return value as MeasurementIdentity +} + +function sourceTree(identity: MeasurementIdentity): SshRelayRuntimeSourceTree { + const directories = identity.entries + .filter((entry) => entry.type === 'directory') + .map((entry) => ({ + ...entry, + localPath: join(runtimeRoot as string, ...entry.path.split('/')) + })) + const files = identity.entries + .filter((entry) => entry.type === 'file') + .map((entry) => ({ + ...entry, + localPath: join(runtimeRoot as string, ...entry.path.split('/')) + })) + return Object.freeze({ + tupleId: identity.tupleId, + contentId: identity.contentId, + releaseTag: 'measurement-only', + os: identity.os, + architecture: identity.architecture, + runtimeRoot: runtimeRoot as string, + directories, + files, + fileCount: identity.archive.fileCount, + expandedBytes: identity.archive.expandedSize, + assertLeaseOwned: async () => {} + }) +} + +async function measure(operation: () => Promise): Promise> { + const baselineRss = process.memoryUsage().rss + let peakRss = baselineRss + const sample = (): void => { + peakRss = Math.max(peakRss, process.memoryUsage().rss) + } + const sampler = setInterval(sample, 1) + const startedAt = performance.now() + try { + const result = await operation() + sample() + return { + result, + elapsedMs: performance.now() - startedAt, + baselineRss, + peakRss, + incrementalRssBytes: Math.max(0, peakRss - baselineRss) + } + } finally { + clearInterval(sampler) + } +} + +async function hashFile(path: string): Promise { + const handle = await open(path, 'r') + const digest = createHash('sha256') + const buffer = Buffer.allocUnsafe(64 * 1024) + let position = 0 + try { + while (true) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position) + if (bytesRead === 0) { + break + } + digest.update(buffer.subarray(0, bytesRead)) + position += bytesRead + } + } finally { + await handle.close() + } + return `sha256:${digest.digest('hex')}` +} + +async function collectRemotePaths(root: string): Promise { + const paths: string[] = [] + const pending = [{ localPath: root, relativePath: '' }] + while (pending.length > 0) { + const current = pending.pop()! + const directory = await opendir(current.localPath) + for await (const entry of directory) { + const relativePath = current.relativePath + ? `${current.relativePath}/${entry.name}` + : entry.name + paths.push(relativePath) + if (entry.isDirectory()) { + pending.push({ localPath: join(current.localPath, entry.name), relativePath }) + } + } + } + return paths.sort() +} + +async function validateTransferredTree( + tree: SshRelayRuntimeScannedSourceTree, + stage: string +): Promise { + const expectedPaths = [...tree.directories, ...tree.files].map((entry) => entry.path).sort() + // Why: the loopback runner can inspect sshd's filesystem without adding another proof channel. + expect(await collectRemotePaths(stage)).toEqual(expectedPaths) + for (const directory of tree.directories) { + const metadata = await lstat(join(stage, ...directory.path.split('/'))) + expect(metadata.isDirectory()).toBe(true) + expect(metadata.mode & 0o777).toBe(directory.mode) + } + for (const file of tree.files) { + const path = join(stage, ...file.path.split('/')) + const metadata = await lstat(path) + expect(metadata.isFile()).toBe(true) + expect(metadata.size).toBe(file.size) + expect(metadata.mode & 0o777).toBe(file.mode) + expect(await hashFile(path)).toBe(file.sha256) + } +} + +function stagePath(name: string, identity: MeasurementIdentity): string { + return join(remoteRoot as string, `${name}-${identity.contentId.slice('sha256:'.length, 23)}`) +} + +async function transfer( + connection: SshConnection, + tree: SshRelayRuntimeScannedSourceTree, + stage: string, + maximumConcurrency: number, + signal = new AbortController().signal, + onProgress?: (bytes: number) => void +) { + return transferSshRelayRuntimeTreeOverSftpConnection({ + connection, + tree, + remoteStagingRoot: stage, + enforcePosixMode: true, + maximumConcurrency, + signal, + onProgress: ({ bytesTransferred }) => onProgress?.(bytesTransferred) + }) +} + +describe.skipIf(!hasLiveInput)('SSH relay full-size transfer through live OpenSSH SFTP', () => { + it( + 'preserves exact bytes and modes with bounded serial, concurrent, and cancellation behavior', + { timeout: 20 * 60_000 }, + async () => { + const identity = parseIdentity(JSON.parse(await readFile(identityPath as string, 'utf8'))) + expect(identity.os).toBe('linux') + const tree = await scanSshRelayRuntimeSourceTree( + sourceTree(identity), + new AbortController().signal + ) + const target: SshTarget = { + id: 'live-sftp-measurement', + label: 'live-sftp-measurement', + host: host as string, + port, + username: user as string, + identityFile: identityFile as string, + source: 'manual' + } + const connection = new SshConnection(target, { onStateChange: () => {} }) + const serialStage = stagePath('serial', identity) + const concurrentStage = stagePath('concurrent', identity) + const cancelledStage = stagePath('cancelled', identity) + await Promise.all( + [serialStage, concurrentStage, cancelledStage].map((stage) => + rm(stage, { recursive: true, force: true }) + ) + ) + try { + await connection.connect() + const serial = await measure(() => transfer(connection, tree, serialStage, 1)) + await validateTransferredTree(tree, serialStage) + const concurrent = await measure(() => transfer(connection, tree, concurrentStage, 4)) + await validateTransferredTree(tree, concurrentStage) + await expect(transfer(connection, tree, concurrentStage, 1)).rejects.toBeTruthy() + await validateTransferredTree(tree, concurrentStage) + + const controller = new AbortController() + let abortRequestedAt = 0 + const cancelled = await measure(async () => { + const outcome = transfer( + connection, + tree, + cancelledStage, + 4, + controller.signal, + (bytes) => { + if (bytes > 0 && !controller.signal.aborted) { + abortRequestedAt = performance.now() + controller.abort(new Error('live SFTP cancellation')) + } + } + ) + await expect(outcome).rejects.toThrow('live SFTP cancellation') + return performance.now() - abortRequestedAt + }) + await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' }) + await new Promise((resolve) => setTimeout(resolve, 500)) + await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' }) + + const metrics = { + tupleId: identity.tupleId, + contentId: identity.contentId, + files: tree.fileCount, + bytes: tree.expandedBytes, + serverVersion: process.env.ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION, + runnerImage: process.env.ImageOS, + runnerVersion: process.env.ImageVersion, + runnerArchitecture: process.env.RUNNER_ARCH, + serialElapsedMs: serial.elapsedMs, + serialIncrementalRssBytes: serial.incrementalRssBytes, + concurrentElapsedMs: concurrent.elapsedMs, + concurrentIncrementalRssBytes: concurrent.incrementalRssBytes, + cancellationSettlementMs: cancelled.result, + cancellationIncrementalRssBytes: cancelled.incrementalRssBytes + } + console.log(`ssh_relay_live_sftp=${JSON.stringify(metrics)}`) + expect(serial.result.bytesTransferred).toBe(tree.expandedBytes) + expect(concurrent.result.bytesTransferred).toBe(tree.expandedBytes) + expect(serial.incrementalRssBytes).toBeLessThanOrEqual( + SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes + ) + expect(concurrent.incrementalRssBytes).toBeLessThanOrEqual( + SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes + ) + expect(cancelled.result).toBeLessThan(10_000) + expect(connection.getState().status).toBe('connected') + } finally { + await connection.disconnect() + await Promise.all( + [serialStage, concurrentStage, cancelledStage].map((stage) => + rm(stage, { recursive: true, force: true }) + ) + ) + } + } + ) +}) diff --git a/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts b/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts index 5b86ff8f1bd..ef113b81d7c 100644 --- a/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts +++ b/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts @@ -325,33 +325,68 @@ describe('SSH relay runtime SFTP tree transfer', () => { }) it('closes the session to settle a retained write callback on cancellation', async () => { + vi.useFakeTimers() + try { + const tree = await treeFixture() + const { session } = createSession() + const controller = new AbortController() + let retainedCallback: Callback | undefined + let writes = 0 + vi.mocked(session.operations.write).mockImplementationOnce( + (_handle, _buffer, _offset, _length, _position, callback) => { + writes += 1 + retainedCallback = callback + } + ) + vi.mocked(session.close).mockImplementationOnce(async () => { + retainedCallback?.(new Error('session closed')) + }) + const transfer = transferSshRelayRuntimeTreeViaSftp({ + tree, + remoteStagingRoot: '/staging/content', + enforcePosixMode: true, + maximumConcurrency: 1, + signal: controller.signal, + openSession: async () => session + }) + + await vi.waitFor(() => expect(writes).toBe(1)) + controller.abort(new Error('cancelled')) + const rejected = expect(transfer).rejects.toThrow(/cancelled|session closed/) + await vi.advanceTimersByTimeAsync(250) + await rejected + expect(session.close).toHaveBeenCalledOnce() + expect(writes).toBe(1) + } finally { + vi.useRealTimers() + } + }) + + it('reverse-cleans owned paths before session close when cancellation is not callback-stuck', async () => { const tree = await treeFixture() - const { session } = createSession() + const { session, events } = createSession() const controller = new AbortController() - let retainedCallback: Callback | undefined let writes = 0 - vi.mocked(session.operations.write).mockImplementationOnce( - (_handle, _buffer, _offset, _length, _position, callback) => { - writes += 1 - retainedCallback = callback - } - ) - vi.mocked(session.close).mockImplementationOnce(async () => { - retainedCallback?.(new Error('session closed')) - }) const transfer = transferSshRelayRuntimeTreeViaSftp({ tree, - remoteStagingRoot: '/staging/content', + remoteStagingRoot: '/staging/cancelled-content', enforcePosixMode: true, maximumConcurrency: 1, signal: controller.signal, - openSession: async () => session + openSession: async () => session, + onProgress: ({ bytesTransferred }) => { + writes += 1 + if (bytesTransferred > 0) { + controller.abort(new Error('ordinary cancellation')) + } + } }) - await vi.waitFor(() => expect(writes).toBe(1)) - controller.abort(new Error('cancelled')) - await expect(transfer).rejects.toThrow(/cancelled|session closed/) - expect(session.close).toHaveBeenCalledOnce() + await expect(transfer).rejects.toThrow('ordinary cancellation') + expect(events.findLastIndex((event) => event.startsWith('rmdir:'))).toBeLessThan( + events.indexOf('close-session') + ) + expect(events).toContain('rmdir:/staging/cancelled-content') expect(writes).toBe(1) }) diff --git a/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts b/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts index cf9523827e2..abb92b2033b 100644 --- a/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts +++ b/src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts @@ -39,6 +39,7 @@ export type SshRelayRuntimeSftpTreeTransferResult = Readonly< > const MAXIMUM_CONCURRENT_FILES = 4 +const ABORTED_CALLBACK_BREAKER_MS = 250 function waitForSftpCallback(register: (callback: SftpCallback) => void): Promise { return new Promise((resolve, reject) => { @@ -169,13 +170,24 @@ export async function transferSshRelayRuntimeTreeViaSftp( signal.throwIfAborted() const session = await openSession(signal) let closePromise: Promise | undefined + let abortedCallbackBreaker: ReturnType | undefined const closeSession = (reason?: unknown): Promise => { closePromise ??= Promise.resolve().then(() => session.close(reason)) return closePromise } + const clearAbortedCallbackBreaker = (): void => { + clearTimeout(abortedCallbackBreaker) + abortedCallbackBreaker = undefined + } const onAbort = (): void => { - // Why: closing the session is what releases callbacks and their borrowed source buffers. - void closeSession(signal.reason).catch(() => {}) + if (abortedCallbackBreaker || closePromise) { + return + } + // Why: normal cancellation gets a cleanup opportunity, but a retained raw callback needs close. + abortedCallbackBreaker = setTimeout(() => { + abortedCallbackBreaker = undefined + void closeSession(signal.reason).catch(() => {}) + }, ABORTED_CALLBACK_BREAKER_MS) } signal.addEventListener('abort', onAbort, { once: true }) if (signal.aborted) { @@ -239,6 +251,8 @@ export async function transferSshRelayRuntimeTreeViaSftp( signal.throwIfAborted() return Object.freeze({ remoteStagingRoot, ...result }) } catch (error) { + // The stream reached a settled callback boundary, so reverse cleanup can safely own shutdown. + clearAbortedCallbackBreaker() const cleanupFailures = ownedDirectories.length === 0 ? [] @@ -246,6 +260,12 @@ export async function transferSshRelayRuntimeTreeViaSftp( await closeSession(error).catch((closeError) => cleanupFailures.push(closeError)) throw joinedFailure(error, cleanupFailures) } finally { + clearAbortedCallbackBreaker() signal.removeEventListener('abort', onAbort) } } + +export const SSH_RELAY_RUNTIME_SFTP_TREE_TRANSFER_LIMITS = Object.freeze({ + maximumConcurrentFiles: MAXIMUM_CONCURRENT_FILES, + abortedCallbackBreakerMs: ABORTED_CALLBACK_BREAKER_MS +})