mirror of
https://github.com/stablyai/orca.git
synced 2026-10-18 08:03:22 +00:00
feat: compose bounded SSH relay SFTP transfer
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -13,6 +13,7 @@ on:
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
- 'src/main/ipc/parcel-watcher-process-entry.ts'
|
||||
- 'src/main/ssh/ssh-relay-runtime-*'
|
||||
- 'src/relay/**'
|
||||
workflow_dispatch:
|
||||
# Why: callers may gate on the native graph only after separate release wiring is reviewed.
|
||||
@@ -286,6 +287,7 @@ jobs:
|
||||
src/main/ssh/ssh-relay-runtime-source-scan.test.ts \
|
||||
src/main/ssh/ssh-relay-runtime-source-stream.test.ts \
|
||||
src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts \
|
||||
src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts \
|
||||
src/main/ssh/ssh-relay-runtime-sftp-session.test.ts \
|
||||
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts \
|
||||
src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts \
|
||||
@@ -553,6 +555,88 @@ jobs:
|
||||
src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts
|
||||
test ! -e "$extraction_output"
|
||||
|
||||
- name: Start loopback OpenSSH SFTP fixture
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! test -x /usr/sbin/sshd; then
|
||||
apt_options=(-o Acquire::Retries=3 -o Acquire::https::Timeout=15 -o Acquire::ForceIPv4=true)
|
||||
timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" update -qq
|
||||
timeout --signal=TERM --kill-after=15s 180s sudo apt-get "${apt_options[@]}" install -y -qq openssh-server
|
||||
fi
|
||||
fixture="$RUNNER_TEMP/orca-live-sftp"
|
||||
mkdir -p "$fixture" "$HOME/.ssh" "$fixture/remote"
|
||||
chmod 700 "$HOME/.ssh" "$fixture"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client-key"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host-key"
|
||||
cp "$fixture/client-key.pub" "$HOME/.ssh/authorized_keys"
|
||||
chmod 600 "$HOME/.ssh/authorized_keys" "$fixture/client-key" "$fixture/host-key"
|
||||
sudo chown root:root "$fixture/host-key"
|
||||
port=22222
|
||||
server_version=$(/usr/sbin/sshd -V 2>&1 | head -1)
|
||||
config="$fixture/sshd_config"
|
||||
printf '%s\n' \
|
||||
"Port $port" \
|
||||
'ListenAddress 127.0.0.1' \
|
||||
"HostKey $fixture/host-key" \
|
||||
"PidFile $fixture/sshd.pid" \
|
||||
"AuthorizedKeysFile $HOME/.ssh/authorized_keys" \
|
||||
"AllowUsers $USER" \
|
||||
'PasswordAuthentication no' \
|
||||
'KbdInteractiveAuthentication no' \
|
||||
'ChallengeResponseAuthentication no' \
|
||||
'UsePAM no' \
|
||||
'StrictModes yes' \
|
||||
'LogLevel VERBOSE' \
|
||||
'Subsystem sftp internal-sftp' > "$config"
|
||||
sudo mkdir -p /run/sshd
|
||||
sudo /usr/sbin/sshd -f "$config" -E "$fixture/sshd.log"
|
||||
for attempt in {1..20}; do
|
||||
if ssh -F /dev/null -i "$fixture/client-key" -p "$port" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
"$USER@127.0.0.1" true; then
|
||||
break
|
||||
fi
|
||||
if test "$attempt" = 20; then
|
||||
cat "$fixture/sshd.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
{
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_HOST=127.0.0.1\n'
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_PORT=%s\n' "$port"
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_USER=%s\n' "$USER"
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY=%s\n' "$fixture/client-key"
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT=%s\n' "$fixture/remote"
|
||||
printf 'ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION=%s\n' "$server_version"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Measure exact full-size runtime over live OpenSSH SFTP
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
first_output="$RUNNER_TEMP/ssh-relay-runtime/${{ matrix.tuple }}/first"
|
||||
ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT="$first_output/runtime" \
|
||||
ORCA_SSH_RELAY_FULL_SIZE_IDENTITY="$first_output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json" \
|
||||
pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 --reporter=verbose \
|
||||
src/main/ssh/ssh-relay-runtime-sftp-openssh-full-size.test.ts
|
||||
|
||||
- name: Stop loopback OpenSSH SFTP fixture
|
||||
if: always() && runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
fixture="$RUNNER_TEMP/orca-live-sftp"
|
||||
if test -s "$fixture/sshd.pid"; then
|
||||
sudo kill "$(cat "$fixture/sshd.pid")" || true
|
||||
fi
|
||||
if test -f "$fixture/sshd.log"; then
|
||||
sudo tail -200 "$fixture/sshd.log"
|
||||
fi
|
||||
|
||||
- name: Upload unpublished artifact evidence
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
@@ -844,6 +928,7 @@ jobs:
|
||||
src/main/ssh/ssh-relay-runtime-source-scan.test.ts `
|
||||
src/main/ssh/ssh-relay-runtime-source-stream.test.ts `
|
||||
src/main/ssh/ssh-relay-runtime-sftp-file-destination.test.ts `
|
||||
src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts `
|
||||
src/main/ssh/ssh-relay-runtime-sftp-session.test.ts `
|
||||
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts `
|
||||
src/main/ssh/ssh-relay-compiled-manifest-trust.test.ts `
|
||||
|
||||
@@ -83,6 +83,25 @@ describe('SSH relay runtime artifact workflow', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('runs exact full-size Linux runtime transfer against loopback-only stock OpenSSH', async () => {
|
||||
const workflow = parse(await readFile(workflowUrl, 'utf8'))
|
||||
const steps = workflow.jobs['build-posix-runtime'].steps
|
||||
const start = steps.find((step) => step.name === 'Start loopback OpenSSH SFTP fixture')
|
||||
const measure = steps.find(
|
||||
(step) => step.name === 'Measure exact full-size runtime over live OpenSSH SFTP'
|
||||
)
|
||||
const stop = steps.find((step) => step.name === 'Stop loopback OpenSSH SFTP fixture')
|
||||
|
||||
expect(start.if).toBe("runner.os == 'Linux'")
|
||||
expect(start.run).toContain('ListenAddress 127.0.0.1')
|
||||
expect(start.run).toContain('Subsystem sftp internal-sftp')
|
||||
expect(start.run).toContain('timeout --signal=TERM --kill-after=15s 180s sudo apt-get')
|
||||
expect(measure.if).toBe("runner.os == 'Linux'")
|
||||
expect(measure.run).toContain('ssh-relay-runtime-sftp-openssh-full-size.test.ts')
|
||||
expect(measure.run).toContain('first_output/runtime')
|
||||
expect(stop.if).toBe("always() && runner.os == 'Linux'")
|
||||
})
|
||||
|
||||
it('uploads only the first output after two clean builds verify and compare', async () => {
|
||||
const source = await readFile(workflowUrl, 'utf8')
|
||||
const workflow = parse(source)
|
||||
@@ -144,6 +163,7 @@ describe('SSH relay runtime artifact workflow', () => {
|
||||
'ssh-relay-runtime-source-scan.test.ts',
|
||||
'ssh-relay-runtime-source-stream.test.ts',
|
||||
'ssh-relay-runtime-sftp-file-destination.test.ts',
|
||||
'ssh-relay-runtime-sftp-connection-transfer.test.ts',
|
||||
'ssh-relay-runtime-sftp-session.test.ts',
|
||||
'ssh-relay-runtime-sftp-tree-transfer.test.ts',
|
||||
'ssh-relay-compiled-manifest-trust.test.ts',
|
||||
|
||||
+14
-3
@@ -9,9 +9,20 @@ keeps commands, hashes, runner identities, timings, and failure details.
|
||||
A checked box means the work has evidence in the detailed ledger. Design approval alone does not
|
||||
complete a box.
|
||||
|
||||
Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected raw `ssh2`
|
||||
session adapter is exact-head green; live adapter/tree composition audit is next, 2026-07-15, Codex
|
||||
implementation owner.** `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both
|
||||
Active checkpoint: **Milestone 6 / Work Package 5 bounded runtime transfer — disconnected live
|
||||
OpenSSH SFTP composition is in progress, 2026-07-15, Codex implementation owner.**
|
||||
`E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001` limits this package to one authenticated built-in
|
||||
`SshConnection`, one raw SFTP channel, the existing one-to-four file transfer, captured-transport-
|
||||
only force close with awaited teardown, and exact full-size Linux x64/arm64 runtime transfer against
|
||||
target-native OpenSSH runners. It also requires ordinary cancellation cleanup to run before a short
|
||||
retained-callback breaker closes the session. No product/default, system-SSH, Windows-remote,
|
||||
high-RTT, `MaxSessions=1`, or tuple-enablement claim is made. SignPath remains deferred.
|
||||
`E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` is locally green: 15/15 purpose cases, 189 focused cases
|
||||
with two declared skips, 603 broad relay cases with six declared skips, 281/281 release contracts,
|
||||
typecheck, full lint, formatting, max-lines, diff, protected-resolver, and isolation gates pass. The
|
||||
full-size live suite is deliberately skipped without runner OpenSSH/runtime inputs, so exact-head
|
||||
Linux x64/arm64 live CI remains the next gate.
|
||||
`E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all six primary native jobs, both
|
||||
Linux supplements, Windows x64 floor, PR Checks, Golden E2E, and computer-use on exact commit
|
||||
`fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100
|
||||
rejection after complete runtime smoke. No live SFTP/tree or product/default claim is made.
|
||||
|
||||
+122
-8
@@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps
|
||||
|
||||
Date created: 2026-07-14<br>
|
||||
Last updated: 2026-07-15<br>
|
||||
Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected raw `ssh2` session adapter is exact-head green; live adapter/tree composition audit is next**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
|
||||
Session checkpoint: **RAW SFTP SESSION ADAPTER EXACT-HEAD GREEN — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-SESSION-ADAPTER-CI-001` closes all-six primary native, both Linux supplement, Windows x64 floor, PR Checks, Golden E2E, and computer-use proof for exact implementation commit `fbec34cf1`; Windows arm64 retains only the declared hosted build-26200 versus required-26100 rejection after complete runtime smoke. Audit the smallest disconnected live adapter/tree composition next, including a concrete real connection-force-close owner and OpenSSH SFTP server. Do not credit live transfer, server filesystem/modes, full-size network metrics, product/settings/fallback wiring, tuple enablement, publication, or default behavior from native callback mocks. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
|
||||
Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: disconnected authenticated-connection/SFTP-tree composition and live OpenSSH proof**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
|
||||
Session checkpoint: **LIVE SFTP COMPOSITION LOCALLY GREEN / EXACT-HEAD RUNNER PROOF NEXT — 2026-07-15, Codex implementation owner** — `E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001` proves the disconnected authenticated-connection owner, captured-client force close, cancellation cleanup/breaker ordering, workflow oracle, focused/broad/release suites, typecheck, full lint, formatting, max-lines, diff, protected-resolver, and no-product-import gates locally. The purpose suite is 15/15; the live full-size suite is deliberately skipped locally because no runner OpenSSH/runtime inputs are present and therefore earns no live cell. Push this isolated package and require exact-head Linux x64/arm64 stock-OpenSSH full-size metrics before crediting live SFTP. Do not credit Windows/POSIX system SSH, high-RTT, `MaxSessions=1`, product/settings/fallback wiring, tuple enablement, publication, or default behavior. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
|
||||
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)<br>
|
||||
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693)
|
||||
|
||||
@@ -17207,6 +17207,122 @@ config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 7/7 in 356 ms. The
|
||||
publish/launch and every product/Beta/settings/fallback/tuple/publication/default path remain
|
||||
absent; legacy remains the sole production/default path and SignPath remains deferred.
|
||||
|
||||
### E-M6-SFTP-LIVE-COMPOSITION-AUDIT-001 — authenticated connection ownership and live OpenSSH boundary
|
||||
|
||||
- Date/owner/base: 2026-07-15, Codex implementation owner; exact clean evidence base `d11261706`
|
||||
after `E-M6-SFTP-SESSION-ADAPTER-CI-001` closure.
|
||||
- Audit commands: `sed -n '1,290p' src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.ts`,
|
||||
`sed -n '1,280p' src/main/ssh/ssh-relay-runtime-sftp-session.ts`, relevant lifecycle ranges from
|
||||
`ssh-connection.ts`, and searches for existing live-SSH/OpenSSH/workflow fixtures. Result: the raw
|
||||
adapter and tree contracts compose without changing authentication or relay RPC, but no
|
||||
non-test owner captures the exact `ssh2` transport that opened the raw SFTP channel or awaits
|
||||
connection teardown if channel close exceeds its grace.
|
||||
- Composition boundary: add one purpose-named module that accepts the existing authenticated
|
||||
`SshConnection`, captures its exact current built-in `ssh2` client, opens exactly one signal-bound
|
||||
raw SFTP session, rejects a connection-generation race after closing the returned channel, and
|
||||
delegates to the existing session adapter and tree orchestrator. Force-close may destroy only the
|
||||
captured client, must await that client's public `close` event with a bounded timeout, and must not
|
||||
dispose the `SshConnection` or destroy a replacement client; the existing disconnect handler owns
|
||||
reconnect state. System-SSH targets reject this composition without opening anything.
|
||||
- Cancellation correction: current tree abort starts session close immediately. That is required to
|
||||
release a retained raw callback, but it can race and prevent reverse cleanup when the active
|
||||
callback settles normally. On abort, arm a short breaker below the 10-second cancellation-and-join
|
||||
ceiling; when the stream reaches its catch boundary, cancel the breaker before reverse cleanup.
|
||||
A genuinely retained callback is still released by timed session close, while normal cancellation
|
||||
gets one opportunity to remove proved-owned paths before closing the session. Tests must prove
|
||||
both orders and that no write begins after abort.
|
||||
- Layer-A proof: purpose tests cover exact signal and connection identity, one SFTP open, system-SSH
|
||||
rejection, connection replacement race, captured-client-only destroy/await-close, force-close
|
||||
timeout/failure, normal owned-tree cancellation cleanup before close, and retained-callback breaker
|
||||
settlement. Existing focused/broad/static/isolation gates remain mandatory.
|
||||
- Layer-B proof: on target-native Ubuntu x64 and arm64 GitHub runners, start a loopback-only stock
|
||||
OpenSSH `sshd` with key authentication and internal SFTP, then transfer the exact runtime produced
|
||||
by that runner's build job. Validate the complete staged tree bytes/count/modes, exclusive-root
|
||||
behavior, serial and four-file measurements, full-size incremental desktop RSS under 80 MiB,
|
||||
cancellation settlement under 10 seconds, no later writes, and owned-stage cleanup. Record runner
|
||||
image, architecture, OpenSSH version, runtime tuple/content/size/count, duration, RSS, and gaps.
|
||||
- Explicit residual boundary: this package does not prove a Windows remote, cross-client matrix,
|
||||
high-RTT comparison, `MaxSessions=1`, network interruption/disk/quota/inode/noexec/AV failures,
|
||||
POSIX or Windows system-SSH transfer, remote bundled-Node verification/install/publish/launch,
|
||||
product/Beta/settings/fallback wiring, tuple enablement, publication, or default behavior. Legacy
|
||||
remains the sole production/default path and SignPath remains deferred.
|
||||
|
||||
### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-RED-001 — missing owner and cancellation ordering fail as audited
|
||||
|
||||
- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0,
|
||||
pnpm 10.24.0, exact base `d11261706` plus audit/test/workflow-only diff. Exact Node 24 remains CI.
|
||||
- Command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts
|
||||
--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts
|
||||
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — EXPECTED FAIL: the five-case
|
||||
connection suite collects zero tests because the purpose-named composition module is absent, and
|
||||
the new ordinary-cancellation case observes the final owned-tree `rmdir` after session close
|
||||
(event indices 23 versus 11). Nine prior tree cases pass. Vitest 1.87 seconds / 3.24 seconds wall,
|
||||
141,524,992-byte maximum RSS. The temporary unhandled-rejection diagnostic came from attaching
|
||||
the retained-callback test's rejection expectation after advancing fake time and was corrected in
|
||||
the test before GREEN; it is not counted as product evidence.
|
||||
- RED scope: tests and native workflow inclusion only. Protected resolver files remain untouched;
|
||||
no owner module, live server, product importer, settings/fallback/tuple/default behavior, or
|
||||
SignPath work exists.
|
||||
|
||||
### E-M6-SFTP-LIVE-COMPOSITION-LOCAL-001 — disconnected owner and live-runner oracle are locally green
|
||||
|
||||
- Date/owner/environment: 2026-07-15, Codex implementation owner; local macOS arm64, Node 26.0.0,
|
||||
pnpm 10.24.0, atop exact evidence base `d11261706` plus this isolated diff. Exact Node 24 and stock
|
||||
OpenSSH remain the native CI gate.
|
||||
- Implementation: the 140-line connection module accepts only the existing authenticated built-in
|
||||
connection boundary, captures the exact current `ssh2` client, opens one signal-bound raw SFTP
|
||||
channel, and rejects/awaits cleanup if the connection changes during open. Its escalation destroys
|
||||
only the captured client and awaits public client close for five seconds; a replacement client is
|
||||
never touched, normal raw-session close leaves the connection open, and existing disconnect logic
|
||||
retains reconnect ownership. No `SshConnection` auth/RPC/default behavior changes.
|
||||
- Cancellation correction: the tree orchestrator arms a 250 ms raw-callback breaker on abort. If the
|
||||
callback settles and the stream reaches its catch boundary, it cancels the breaker, reverse-cleans
|
||||
proved-owned files/directories, then closes the session. If a callback remains retained, the
|
||||
breaker closes the raw session to release the borrowed 64 KiB source buffer. Tests prove both
|
||||
orders and no second write after abort; the overall 10-second cancellation-and-join ceiling is
|
||||
unchanged.
|
||||
- Purpose command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts
|
||||
--maxWorkers=1 src/main/ssh/ssh-relay-runtime-sftp-connection-transfer.test.ts
|
||||
src/main/ssh/ssh-relay-runtime-sftp-tree-transfer.test.ts` — PASS, 15/15 in 1.59 seconds Vitest /
|
||||
2.68 seconds wall, 139,755,520-byte maximum RSS. Five owner cases cover exact signal/one open,
|
||||
normal connection retention, system-SSH rejection, generation replacement cleanup, captured-only
|
||||
force close, awaited raw/client close, and bounded missing-close failure. Ten tree cases include
|
||||
ordinary cancellation cleanup before close and retained-callback breaker settlement.
|
||||
- Focused command: connection, source tree/scan/stream, three SFTP boundaries, acquisition/cache
|
||||
resolution/population/integration, live-suite skip oracle, and workflow contract — PASS, 14 files
|
||||
plus one declared live-input skip; 189 passed / two declared skips in 14.22 seconds Vitest /
|
||||
15.70 seconds wall, 172,146,688-byte maximum RSS.
|
||||
- Broad commands: `src/main/ssh/ssh-relay-*.test.ts` — PASS, 46 files passed / four declared skipped,
|
||||
603 tests passed / six declared skipped in 25.09 seconds Vitest / 27.10 seconds wall,
|
||||
301,694,976-byte maximum RSS; `config/scripts/ssh-relay-runtime-*.test.mjs` — PASS, 50 files /
|
||||
281 tests in 22.94 seconds Vitest / 24.87 seconds wall, 190,398,464-byte maximum RSS. Only the
|
||||
established stale-lock diagnostic and local Node 26 module deprecation appear.
|
||||
- Static/isolation commands: `pnpm typecheck` passes in 4.75 seconds wall with 1,249,492,992-byte
|
||||
maximum RSS; targeted `oxlint`, targeted `oxfmt --check`, standalone max-lines ratchet,
|
||||
`git diff --check`, protected-resolver empty diff, workflow YAML/oracle, and no non-test consumer
|
||||
outside this disconnected module pass. Full `pnpm lint` passes in 17.44 seconds wall with
|
||||
2,025,259,008-byte maximum RSS: switch exhaustiveness, 41 reliability gates, 355-entry max-lines
|
||||
ratchet, bundled-skill verification, 9,837 localization references, locale parity, and zero-
|
||||
candidate coverage pass with only pre-existing warnings. Module/purpose/live-test sizes are
|
||||
140/170/311 lines, and the updated tree module/test are 271/426 lines, with no new bypass.
|
||||
- Live runner oracle: the existing artifact workflow now triggers for runtime-transfer changes and
|
||||
includes the five owner cases once in each native command family. Linux x64 and arm64 build jobs,
|
||||
after producing and locally verifying the exact runtime, start a loopback-only stock OpenSSH
|
||||
`sshd` with key authentication/internal SFTP. The purpose-named full-size suite scans the exact
|
||||
identity/runtime, transfers it serially and with four files, validates every path/byte/hash/mode,
|
||||
proves exclusive-root refusal leaves the tree intact, cancels after the first completed write,
|
||||
requires cleanup/no later writes and sub-10-second settlement, enforces the 80 MiB desktop RSS
|
||||
ceiling, and emits tuple/content/file/byte/server/runner/duration/RSS metrics. The fixture setup is
|
||||
bounded and cleanup runs under `always()`.
|
||||
- Local live result: deliberately SKIP, one case, because the required seven OpenSSH/runtime environment
|
||||
inputs are absent. This proves the skip gate only and earns no live/server/filesystem/full-size
|
||||
evidence. Exact-head Linux x64/arm64 CI is mandatory next.
|
||||
- Residual gaps: no Windows remote, cross-client matrix, high-RTT comparison, `MaxSessions=1`, live
|
||||
peak channel/open-handle instrumentation, interruption/disk/quota/inode/noexec/AV failure, POSIX
|
||||
or Windows system-SSH path, remote bundled-Node verification/install/publish/launch, product/Beta/
|
||||
settings/fallback wiring, tuple enablement, publication, or default behavior is proved. Legacy
|
||||
remains the sole production/default path and SignPath remains deferred.
|
||||
|
||||
## Accepted Gaps
|
||||
|
||||
No product gap is accepted merely because it appears in this list. Each entry requires explicit
|
||||
@@ -17265,12 +17381,10 @@ The project is not complete until every applicable item below is checked with ev
|
||||
|
||||
## Next Required Action
|
||||
|
||||
Commit/push the locally green Windows test-oracle correction, then require fresh all-six exact-head
|
||||
native/full-size and adjacent CI before checking the package or advancing. The stream must create no
|
||||
remote files or open SSH channels.
|
||||
Do not open an SSH channel, transfer to or install on a remote, modify platform detection/selectors,
|
||||
add an Electron/startup consumer, mode wiring, fallback, tuple enablement, release publication, or
|
||||
default behavior.
|
||||
Commit and push the locally green disconnected SFTP composition package, then require exact-head
|
||||
target-native Linux x64/arm64 OpenSSH full-size evidence and all-six native/adjacent regression CI
|
||||
before checking any Milestone 6 SFTP item or advancing. Do not add an Electron/startup/product
|
||||
importer, per-target mode wiring, fallback, tuple enablement, release publication, or default behavior.
|
||||
Keep Node upstream `.tar.xz` inputs, Windows ZIP, `ORCA_RELAY_PATH`, existing desktop required-assets
|
||||
behavior, detached-signature byte encoding, Windows arm64 build 26100, macOS 13.5, Linux kernel 4.18,
|
||||
release-cut, desktop builds, publication, and every tuple separately gated.
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
|
||||
import type { Client, SFTPWrapper } from 'ssh2'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { openSshRelayRuntimeSftpTreeSessionForConnection } from './ssh-relay-runtime-sftp-connection-transfer'
|
||||
|
||||
type Callback = (error?: Error) => void
|
||||
type ConnectionBoundary = Pick<SshConnection, 'getClient' | 'sftp' | 'usesSystemSshTransport'>
|
||||
|
||||
function createRawSession(options: { closeOnEnd?: boolean } = {}): SFTPWrapper {
|
||||
const emitter = new EventEmitter()
|
||||
const raw = Object.assign(emitter, {
|
||||
mkdir: vi.fn((_path: string, _attributes: unknown, callback: Callback) => callback()),
|
||||
rmdir: vi.fn((_path: string, callback: Callback) => callback()),
|
||||
open: vi.fn(
|
||||
(
|
||||
_path: string,
|
||||
_flags: string,
|
||||
_attributes: unknown,
|
||||
callback: (error: Error | undefined, handle: Buffer) => void
|
||||
) => callback(undefined, Buffer.from('handle'))
|
||||
),
|
||||
write: vi.fn(
|
||||
(
|
||||
_handle: Buffer,
|
||||
_buffer: Buffer,
|
||||
_offset: number,
|
||||
_length: number,
|
||||
_position: number,
|
||||
callback: Callback
|
||||
) => callback()
|
||||
),
|
||||
fchmod: vi.fn((_handle: Buffer, _mode: number, callback: Callback) => callback()),
|
||||
fstat: vi.fn(
|
||||
(_handle: Buffer, callback: (error: Error | undefined, value: { mode: number }) => void) =>
|
||||
callback(undefined, { mode: 0o100755 })
|
||||
),
|
||||
close: vi.fn((_handle: Buffer, callback: Callback) => callback()),
|
||||
unlink: vi.fn((_path: string, callback: Callback) => callback()),
|
||||
end: vi.fn(() => {
|
||||
if (options.closeOnEnd) {
|
||||
queueMicrotask(() => emitter.emit('close'))
|
||||
}
|
||||
})
|
||||
})
|
||||
return raw as unknown as SFTPWrapper
|
||||
}
|
||||
|
||||
function createClient(onDestroy?: () => void): Client & { destroy: ReturnType<typeof vi.fn> } {
|
||||
const emitter = new EventEmitter()
|
||||
const client = Object.assign(emitter, {
|
||||
destroy: vi.fn(() => onDestroy?.())
|
||||
})
|
||||
return client as unknown as Client & { destroy: ReturnType<typeof vi.fn> }
|
||||
}
|
||||
|
||||
function createConnection(
|
||||
client: Client | null,
|
||||
raw: SFTPWrapper,
|
||||
options: { systemSsh?: boolean; onSftp?: () => void } = {}
|
||||
): ConnectionBoundary & {
|
||||
setClient: (next: Client | null) => void
|
||||
sftp: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
let currentClient = client
|
||||
const sftp = vi.fn(async (_signal?: AbortSignal) => {
|
||||
options.onSftp?.()
|
||||
return raw
|
||||
})
|
||||
return {
|
||||
getClient: () => currentClient,
|
||||
setClient: (next) => {
|
||||
currentClient = next
|
||||
},
|
||||
usesSystemSshTransport: () => options.systemSsh ?? false,
|
||||
sftp
|
||||
}
|
||||
}
|
||||
|
||||
describe('SSH relay runtime authenticated SFTP connection transfer', () => {
|
||||
it('opens one raw session with the exact signal and leaves the connection open after normal close', async () => {
|
||||
const raw = createRawSession({ closeOnEnd: true })
|
||||
const client = createClient()
|
||||
const connection = createConnection(client, raw)
|
||||
const signal = new AbortController().signal
|
||||
|
||||
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal)
|
||||
await session.close()
|
||||
|
||||
expect(connection.sftp).toHaveBeenCalledOnce()
|
||||
expect(connection.sftp).toHaveBeenCalledWith(signal)
|
||||
expect(raw.end).toHaveBeenCalledOnce()
|
||||
expect(client.destroy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects system SSH without opening an SFTP channel', async () => {
|
||||
const raw = createRawSession()
|
||||
const connection = createConnection(null, raw, { systemSsh: true })
|
||||
|
||||
await expect(
|
||||
openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal)
|
||||
).rejects.toThrow(/built-in SSH/i)
|
||||
expect(connection.sftp).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('closes a raw channel opened by a replaced connection generation', async () => {
|
||||
const raw = createRawSession({ closeOnEnd: true })
|
||||
const first = createClient()
|
||||
const replacement = createClient()
|
||||
const connection = createConnection(first, raw, {
|
||||
onSftp: () => connection.setClient(replacement)
|
||||
})
|
||||
|
||||
await expect(
|
||||
openSshRelayRuntimeSftpTreeSessionForConnection(connection, new AbortController().signal)
|
||||
).rejects.toThrow(/connection changed/i)
|
||||
expect(raw.end).toHaveBeenCalledOnce()
|
||||
expect(first.destroy).not.toHaveBeenCalled()
|
||||
expect(replacement.destroy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('force-closes only the captured client and awaits both client and raw close', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const raw = createRawSession()
|
||||
const first = createClient(() => {
|
||||
raw.emit('close')
|
||||
first.emit('close')
|
||||
})
|
||||
const replacement = createClient()
|
||||
const connection = createConnection(first, raw)
|
||||
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(
|
||||
connection,
|
||||
new AbortController().signal
|
||||
)
|
||||
connection.setClient(replacement)
|
||||
const close = session.close(new Error('cancelled transfer'))
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
await close
|
||||
expect(first.destroy).toHaveBeenCalledOnce()
|
||||
expect(replacement.destroy).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('fails within the bounded force-close window when client teardown never confirms', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const raw = createRawSession()
|
||||
const client = createClient()
|
||||
const connection = createConnection(client, raw)
|
||||
const session = await openSshRelayRuntimeSftpTreeSessionForConnection(
|
||||
connection,
|
||||
new AbortController().signal
|
||||
)
|
||||
const close = session.close()
|
||||
const rejected = expect(close).rejects.toThrow(/connection close timed out/i)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
await rejected
|
||||
expect(client.destroy).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,140 @@
|
||||
import type { Client, SFTPWrapper } from 'ssh2'
|
||||
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { openSshRelayRuntimeSftpTreeSession } from './ssh-relay-runtime-sftp-session'
|
||||
import {
|
||||
transferSshRelayRuntimeTreeViaSftp,
|
||||
type SshRelayRuntimeSftpTreeSession,
|
||||
type SshRelayRuntimeSftpTreeTransferOptions,
|
||||
type SshRelayRuntimeSftpTreeTransferResult
|
||||
} from './ssh-relay-runtime-sftp-tree-transfer'
|
||||
|
||||
const TRANSPORT_CLOSE_TIMEOUT_MS = 5_000
|
||||
|
||||
export type SshRelayRuntimeBuiltInSftpConnection = Pick<
|
||||
SshConnection,
|
||||
'getClient' | 'sftp' | 'usesSystemSshTransport'
|
||||
>
|
||||
|
||||
export type SshRelayRuntimeSftpConnectionTransferOptions = Omit<
|
||||
SshRelayRuntimeSftpTreeTransferOptions,
|
||||
'openSession'
|
||||
> &
|
||||
Readonly<{ connection: SshRelayRuntimeBuiltInSftpConnection }>
|
||||
|
||||
function waitForClose(
|
||||
emitter: Pick<NodeJS.EventEmitter, 'once' | 'removeListener'>,
|
||||
beginClose: () => void,
|
||||
timeoutMessage: string
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false
|
||||
const finish = (error?: unknown): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
emitter.removeListener('close', onClose)
|
||||
if (error === undefined) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(error)
|
||||
}
|
||||
}
|
||||
const onClose = (): void => finish()
|
||||
const timer = setTimeout(() => finish(new Error(timeoutMessage)), TRANSPORT_CLOSE_TIMEOUT_MS)
|
||||
emitter.once('close', onClose)
|
||||
try {
|
||||
beginClose()
|
||||
} catch (error) {
|
||||
finish(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function forceCloseCapturedClient(client: Client): Promise<void> {
|
||||
await waitForClose(
|
||||
client,
|
||||
() => client.destroy(),
|
||||
'SSH relay runtime owning connection close timed out'
|
||||
)
|
||||
}
|
||||
|
||||
async function closeSessionFromReplacedConnection(raw: SFTPWrapper, client: Client): Promise<void> {
|
||||
try {
|
||||
await waitForClose(
|
||||
raw,
|
||||
() => raw.end(),
|
||||
'SSH relay runtime replaced connection SFTP close timed out'
|
||||
)
|
||||
} catch (error) {
|
||||
try {
|
||||
await forceCloseCapturedClient(client)
|
||||
} catch (forceError) {
|
||||
throw new AggregateError(
|
||||
[error, forceError],
|
||||
'SSH relay runtime replaced connection cleanup failed'
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function validateConnection(connection: SshRelayRuntimeBuiltInSftpConnection): void {
|
||||
if (
|
||||
!connection ||
|
||||
typeof connection.getClient !== 'function' ||
|
||||
typeof connection.sftp !== 'function' ||
|
||||
typeof connection.usesSystemSshTransport !== 'function'
|
||||
) {
|
||||
throw new Error('SSH relay runtime SFTP connection boundary is invalid')
|
||||
}
|
||||
if (connection.usesSystemSshTransport()) {
|
||||
throw new Error('SSH relay runtime SFTP transfer requires the built-in SSH transport')
|
||||
}
|
||||
}
|
||||
|
||||
export async function openSshRelayRuntimeSftpTreeSessionForConnection(
|
||||
connection: SshRelayRuntimeBuiltInSftpConnection,
|
||||
signal: AbortSignal
|
||||
): Promise<SshRelayRuntimeSftpTreeSession> {
|
||||
validateConnection(connection)
|
||||
signal.throwIfAborted()
|
||||
const capturedClient = connection.getClient()
|
||||
if (!capturedClient) {
|
||||
throw new Error('SSH relay runtime SFTP transfer requires a connected built-in SSH client')
|
||||
}
|
||||
|
||||
return openSshRelayRuntimeSftpTreeSession({
|
||||
signal,
|
||||
openRawSession: async (exactSignal) => {
|
||||
if (connection.getClient() !== capturedClient) {
|
||||
throw new Error('SSH relay runtime SFTP connection changed before channel open')
|
||||
}
|
||||
const raw = await connection.sftp(exactSignal)
|
||||
if (connection.getClient() !== capturedClient) {
|
||||
// Why: never return a channel whose owning connection generation is no longer current.
|
||||
await closeSessionFromReplacedConnection(raw, capturedClient)
|
||||
throw new Error('SSH relay runtime SFTP connection changed during channel open')
|
||||
}
|
||||
return raw
|
||||
},
|
||||
// Why: escalation may tear down only the transport that owns the stuck raw callback.
|
||||
forceCloseConnection: () => forceCloseCapturedClient(capturedClient)
|
||||
})
|
||||
}
|
||||
|
||||
export function transferSshRelayRuntimeTreeOverSftpConnection(
|
||||
options: SshRelayRuntimeSftpConnectionTransferOptions
|
||||
): Promise<SshRelayRuntimeSftpTreeTransferResult> {
|
||||
const { connection, ...treeOptions } = options
|
||||
return transferSshRelayRuntimeTreeViaSftp({
|
||||
...treeOptions,
|
||||
openSession: (signal) => openSshRelayRuntimeSftpTreeSessionForConnection(connection, signal)
|
||||
})
|
||||
}
|
||||
|
||||
export const SSH_RELAY_RUNTIME_SFTP_CONNECTION_LIMITS = Object.freeze({
|
||||
transportCloseTimeoutMs: TRANSPORT_CLOSE_TIMEOUT_MS
|
||||
})
|
||||
@@ -0,0 +1,311 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { lstat, open, opendir, readFile, rm } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } }))
|
||||
|
||||
import type { SshTarget } from '../../shared/ssh-types'
|
||||
import { SshConnection } from './ssh-connection'
|
||||
import { transferSshRelayRuntimeTreeOverSftpConnection } from './ssh-relay-runtime-sftp-connection-transfer'
|
||||
import {
|
||||
scanSshRelayRuntimeSourceTree,
|
||||
type SshRelayRuntimeScannedSourceTree
|
||||
} from './ssh-relay-runtime-source-scan'
|
||||
import type { SshRelayRuntimeSourceTree } from './ssh-relay-runtime-source-tree'
|
||||
import { SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS } from './ssh-relay-runtime-source-stream'
|
||||
|
||||
type MeasurementIdentity = Pick<
|
||||
SshRelayRuntimeSourceTree,
|
||||
'tupleId' | 'contentId' | 'os' | 'architecture'
|
||||
> & {
|
||||
entries: (
|
||||
| { path: string; type: 'directory'; mode: 0o755 }
|
||||
| {
|
||||
path: string
|
||||
type: 'file'
|
||||
role: SshRelayRuntimeSourceTree['files'][number]['role']
|
||||
size: number
|
||||
mode: 0o644 | 0o755
|
||||
sha256: SshRelayRuntimeSourceTree['files'][number]['sha256']
|
||||
}
|
||||
)[]
|
||||
archive: { expandedSize: number; fileCount: number }
|
||||
}
|
||||
|
||||
type Measurement<T> = {
|
||||
result: T
|
||||
elapsedMs: number
|
||||
baselineRss: number
|
||||
peakRss: number
|
||||
incrementalRssBytes: number
|
||||
}
|
||||
|
||||
const host = process.env.ORCA_SSH_RELAY_LIVE_SFTP_HOST
|
||||
const user = process.env.ORCA_SSH_RELAY_LIVE_SFTP_USER
|
||||
const identityFile = process.env.ORCA_SSH_RELAY_LIVE_SFTP_IDENTITY
|
||||
const remoteRoot = process.env.ORCA_SSH_RELAY_LIVE_SFTP_REMOTE_ROOT
|
||||
const runtimeRoot = process.env.ORCA_SSH_RELAY_FULL_SIZE_RUNTIME_ROOT
|
||||
const identityPath = process.env.ORCA_SSH_RELAY_FULL_SIZE_IDENTITY
|
||||
const port = Number.parseInt(process.env.ORCA_SSH_RELAY_LIVE_SFTP_PORT ?? '', 10)
|
||||
const hasLiveInput = Boolean(
|
||||
host &&
|
||||
user &&
|
||||
identityFile &&
|
||||
remoteRoot &&
|
||||
runtimeRoot &&
|
||||
identityPath &&
|
||||
Number.isInteger(port)
|
||||
)
|
||||
|
||||
function parseIdentity(value: unknown): MeasurementIdentity {
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
!('entries' in value) ||
|
||||
!Array.isArray(value.entries) ||
|
||||
!('archive' in value) ||
|
||||
typeof value.archive !== 'object' ||
|
||||
value.archive === null
|
||||
) {
|
||||
throw new Error('Live SFTP measurement identity is incomplete')
|
||||
}
|
||||
return value as MeasurementIdentity
|
||||
}
|
||||
|
||||
function sourceTree(identity: MeasurementIdentity): SshRelayRuntimeSourceTree {
|
||||
const directories = identity.entries
|
||||
.filter((entry) => entry.type === 'directory')
|
||||
.map((entry) => ({
|
||||
...entry,
|
||||
localPath: join(runtimeRoot as string, ...entry.path.split('/'))
|
||||
}))
|
||||
const files = identity.entries
|
||||
.filter((entry) => entry.type === 'file')
|
||||
.map((entry) => ({
|
||||
...entry,
|
||||
localPath: join(runtimeRoot as string, ...entry.path.split('/'))
|
||||
}))
|
||||
return Object.freeze({
|
||||
tupleId: identity.tupleId,
|
||||
contentId: identity.contentId,
|
||||
releaseTag: 'measurement-only',
|
||||
os: identity.os,
|
||||
architecture: identity.architecture,
|
||||
runtimeRoot: runtimeRoot as string,
|
||||
directories,
|
||||
files,
|
||||
fileCount: identity.archive.fileCount,
|
||||
expandedBytes: identity.archive.expandedSize,
|
||||
assertLeaseOwned: async () => {}
|
||||
})
|
||||
}
|
||||
|
||||
async function measure<T>(operation: () => Promise<T>): Promise<Measurement<T>> {
|
||||
const baselineRss = process.memoryUsage().rss
|
||||
let peakRss = baselineRss
|
||||
const sample = (): void => {
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss)
|
||||
}
|
||||
const sampler = setInterval(sample, 1)
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
const result = await operation()
|
||||
sample()
|
||||
return {
|
||||
result,
|
||||
elapsedMs: performance.now() - startedAt,
|
||||
baselineRss,
|
||||
peakRss,
|
||||
incrementalRssBytes: Math.max(0, peakRss - baselineRss)
|
||||
}
|
||||
} finally {
|
||||
clearInterval(sampler)
|
||||
}
|
||||
}
|
||||
|
||||
async function hashFile(path: string): Promise<string> {
|
||||
const handle = await open(path, 'r')
|
||||
const digest = createHash('sha256')
|
||||
const buffer = Buffer.allocUnsafe(64 * 1024)
|
||||
let position = 0
|
||||
try {
|
||||
while (true) {
|
||||
const { bytesRead } = await handle.read(buffer, 0, buffer.length, position)
|
||||
if (bytesRead === 0) {
|
||||
break
|
||||
}
|
||||
digest.update(buffer.subarray(0, bytesRead))
|
||||
position += bytesRead
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return `sha256:${digest.digest('hex')}`
|
||||
}
|
||||
|
||||
async function collectRemotePaths(root: string): Promise<string[]> {
|
||||
const paths: string[] = []
|
||||
const pending = [{ localPath: root, relativePath: '' }]
|
||||
while (pending.length > 0) {
|
||||
const current = pending.pop()!
|
||||
const directory = await opendir(current.localPath)
|
||||
for await (const entry of directory) {
|
||||
const relativePath = current.relativePath
|
||||
? `${current.relativePath}/${entry.name}`
|
||||
: entry.name
|
||||
paths.push(relativePath)
|
||||
if (entry.isDirectory()) {
|
||||
pending.push({ localPath: join(current.localPath, entry.name), relativePath })
|
||||
}
|
||||
}
|
||||
}
|
||||
return paths.sort()
|
||||
}
|
||||
|
||||
async function validateTransferredTree(
|
||||
tree: SshRelayRuntimeScannedSourceTree,
|
||||
stage: string
|
||||
): Promise<void> {
|
||||
const expectedPaths = [...tree.directories, ...tree.files].map((entry) => entry.path).sort()
|
||||
// Why: the loopback runner can inspect sshd's filesystem without adding another proof channel.
|
||||
expect(await collectRemotePaths(stage)).toEqual(expectedPaths)
|
||||
for (const directory of tree.directories) {
|
||||
const metadata = await lstat(join(stage, ...directory.path.split('/')))
|
||||
expect(metadata.isDirectory()).toBe(true)
|
||||
expect(metadata.mode & 0o777).toBe(directory.mode)
|
||||
}
|
||||
for (const file of tree.files) {
|
||||
const path = join(stage, ...file.path.split('/'))
|
||||
const metadata = await lstat(path)
|
||||
expect(metadata.isFile()).toBe(true)
|
||||
expect(metadata.size).toBe(file.size)
|
||||
expect(metadata.mode & 0o777).toBe(file.mode)
|
||||
expect(await hashFile(path)).toBe(file.sha256)
|
||||
}
|
||||
}
|
||||
|
||||
function stagePath(name: string, identity: MeasurementIdentity): string {
|
||||
return join(remoteRoot as string, `${name}-${identity.contentId.slice('sha256:'.length, 23)}`)
|
||||
}
|
||||
|
||||
async function transfer(
|
||||
connection: SshConnection,
|
||||
tree: SshRelayRuntimeScannedSourceTree,
|
||||
stage: string,
|
||||
maximumConcurrency: number,
|
||||
signal = new AbortController().signal,
|
||||
onProgress?: (bytes: number) => void
|
||||
) {
|
||||
return transferSshRelayRuntimeTreeOverSftpConnection({
|
||||
connection,
|
||||
tree,
|
||||
remoteStagingRoot: stage,
|
||||
enforcePosixMode: true,
|
||||
maximumConcurrency,
|
||||
signal,
|
||||
onProgress: ({ bytesTransferred }) => onProgress?.(bytesTransferred)
|
||||
})
|
||||
}
|
||||
|
||||
describe.skipIf(!hasLiveInput)('SSH relay full-size transfer through live OpenSSH SFTP', () => {
|
||||
it(
|
||||
'preserves exact bytes and modes with bounded serial, concurrent, and cancellation behavior',
|
||||
{ timeout: 20 * 60_000 },
|
||||
async () => {
|
||||
const identity = parseIdentity(JSON.parse(await readFile(identityPath as string, 'utf8')))
|
||||
expect(identity.os).toBe('linux')
|
||||
const tree = await scanSshRelayRuntimeSourceTree(
|
||||
sourceTree(identity),
|
||||
new AbortController().signal
|
||||
)
|
||||
const target: SshTarget = {
|
||||
id: 'live-sftp-measurement',
|
||||
label: 'live-sftp-measurement',
|
||||
host: host as string,
|
||||
port,
|
||||
username: user as string,
|
||||
identityFile: identityFile as string,
|
||||
source: 'manual'
|
||||
}
|
||||
const connection = new SshConnection(target, { onStateChange: () => {} })
|
||||
const serialStage = stagePath('serial', identity)
|
||||
const concurrentStage = stagePath('concurrent', identity)
|
||||
const cancelledStage = stagePath('cancelled', identity)
|
||||
await Promise.all(
|
||||
[serialStage, concurrentStage, cancelledStage].map((stage) =>
|
||||
rm(stage, { recursive: true, force: true })
|
||||
)
|
||||
)
|
||||
try {
|
||||
await connection.connect()
|
||||
const serial = await measure(() => transfer(connection, tree, serialStage, 1))
|
||||
await validateTransferredTree(tree, serialStage)
|
||||
const concurrent = await measure(() => transfer(connection, tree, concurrentStage, 4))
|
||||
await validateTransferredTree(tree, concurrentStage)
|
||||
await expect(transfer(connection, tree, concurrentStage, 1)).rejects.toBeTruthy()
|
||||
await validateTransferredTree(tree, concurrentStage)
|
||||
|
||||
const controller = new AbortController()
|
||||
let abortRequestedAt = 0
|
||||
const cancelled = await measure(async () => {
|
||||
const outcome = transfer(
|
||||
connection,
|
||||
tree,
|
||||
cancelledStage,
|
||||
4,
|
||||
controller.signal,
|
||||
(bytes) => {
|
||||
if (bytes > 0 && !controller.signal.aborted) {
|
||||
abortRequestedAt = performance.now()
|
||||
controller.abort(new Error('live SFTP cancellation'))
|
||||
}
|
||||
}
|
||||
)
|
||||
await expect(outcome).rejects.toThrow('live SFTP cancellation')
|
||||
return performance.now() - abortRequestedAt
|
||||
})
|
||||
await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
await expect(lstat(cancelledStage)).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
|
||||
const metrics = {
|
||||
tupleId: identity.tupleId,
|
||||
contentId: identity.contentId,
|
||||
files: tree.fileCount,
|
||||
bytes: tree.expandedBytes,
|
||||
serverVersion: process.env.ORCA_SSH_RELAY_LIVE_SFTP_SERVER_VERSION,
|
||||
runnerImage: process.env.ImageOS,
|
||||
runnerVersion: process.env.ImageVersion,
|
||||
runnerArchitecture: process.env.RUNNER_ARCH,
|
||||
serialElapsedMs: serial.elapsedMs,
|
||||
serialIncrementalRssBytes: serial.incrementalRssBytes,
|
||||
concurrentElapsedMs: concurrent.elapsedMs,
|
||||
concurrentIncrementalRssBytes: concurrent.incrementalRssBytes,
|
||||
cancellationSettlementMs: cancelled.result,
|
||||
cancellationIncrementalRssBytes: cancelled.incrementalRssBytes
|
||||
}
|
||||
console.log(`ssh_relay_live_sftp=${JSON.stringify(metrics)}`)
|
||||
expect(serial.result.bytesTransferred).toBe(tree.expandedBytes)
|
||||
expect(concurrent.result.bytesTransferred).toBe(tree.expandedBytes)
|
||||
expect(serial.incrementalRssBytes).toBeLessThanOrEqual(
|
||||
SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes
|
||||
)
|
||||
expect(concurrent.incrementalRssBytes).toBeLessThanOrEqual(
|
||||
SSH_RELAY_RUNTIME_SOURCE_STREAM_LIMITS.maximumIncrementalMemoryBytes
|
||||
)
|
||||
expect(cancelled.result).toBeLessThan(10_000)
|
||||
expect(connection.getState().status).toBe('connected')
|
||||
} finally {
|
||||
await connection.disconnect()
|
||||
await Promise.all(
|
||||
[serialStage, concurrentStage, cancelledStage].map((stage) =>
|
||||
rm(stage, { recursive: true, force: true })
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -325,33 +325,68 @@ describe('SSH relay runtime SFTP tree transfer', () => {
|
||||
})
|
||||
|
||||
it('closes the session to settle a retained write callback on cancellation', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const tree = await treeFixture()
|
||||
const { session } = createSession()
|
||||
const controller = new AbortController()
|
||||
let retainedCallback: Callback | undefined
|
||||
let writes = 0
|
||||
vi.mocked(session.operations.write).mockImplementationOnce(
|
||||
(_handle, _buffer, _offset, _length, _position, callback) => {
|
||||
writes += 1
|
||||
retainedCallback = callback
|
||||
}
|
||||
)
|
||||
vi.mocked(session.close).mockImplementationOnce(async () => {
|
||||
retainedCallback?.(new Error('session closed'))
|
||||
})
|
||||
const transfer = transferSshRelayRuntimeTreeViaSftp({
|
||||
tree,
|
||||
remoteStagingRoot: '/staging/content',
|
||||
enforcePosixMode: true,
|
||||
maximumConcurrency: 1,
|
||||
signal: controller.signal,
|
||||
openSession: async () => session
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(writes).toBe(1))
|
||||
controller.abort(new Error('cancelled'))
|
||||
const rejected = expect(transfer).rejects.toThrow(/cancelled|session closed/)
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await rejected
|
||||
expect(session.close).toHaveBeenCalledOnce()
|
||||
expect(writes).toBe(1)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('reverse-cleans owned paths before session close when cancellation is not callback-stuck', async () => {
|
||||
const tree = await treeFixture()
|
||||
const { session } = createSession()
|
||||
const { session, events } = createSession()
|
||||
const controller = new AbortController()
|
||||
let retainedCallback: Callback | undefined
|
||||
let writes = 0
|
||||
vi.mocked(session.operations.write).mockImplementationOnce(
|
||||
(_handle, _buffer, _offset, _length, _position, callback) => {
|
||||
writes += 1
|
||||
retainedCallback = callback
|
||||
}
|
||||
)
|
||||
vi.mocked(session.close).mockImplementationOnce(async () => {
|
||||
retainedCallback?.(new Error('session closed'))
|
||||
})
|
||||
const transfer = transferSshRelayRuntimeTreeViaSftp({
|
||||
tree,
|
||||
remoteStagingRoot: '/staging/content',
|
||||
remoteStagingRoot: '/staging/cancelled-content',
|
||||
enforcePosixMode: true,
|
||||
maximumConcurrency: 1,
|
||||
signal: controller.signal,
|
||||
openSession: async () => session
|
||||
openSession: async () => session,
|
||||
onProgress: ({ bytesTransferred }) => {
|
||||
writes += 1
|
||||
if (bytesTransferred > 0) {
|
||||
controller.abort(new Error('ordinary cancellation'))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(writes).toBe(1))
|
||||
controller.abort(new Error('cancelled'))
|
||||
await expect(transfer).rejects.toThrow(/cancelled|session closed/)
|
||||
expect(session.close).toHaveBeenCalledOnce()
|
||||
await expect(transfer).rejects.toThrow('ordinary cancellation')
|
||||
expect(events.findLastIndex((event) => event.startsWith('rmdir:'))).toBeLessThan(
|
||||
events.indexOf('close-session')
|
||||
)
|
||||
expect(events).toContain('rmdir:/staging/cancelled-content')
|
||||
expect(writes).toBe(1)
|
||||
})
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ export type SshRelayRuntimeSftpTreeTransferResult = Readonly<
|
||||
>
|
||||
|
||||
const MAXIMUM_CONCURRENT_FILES = 4
|
||||
const ABORTED_CALLBACK_BREAKER_MS = 250
|
||||
|
||||
function waitForSftpCallback(register: (callback: SftpCallback) => void): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -169,13 +170,24 @@ export async function transferSshRelayRuntimeTreeViaSftp(
|
||||
signal.throwIfAborted()
|
||||
const session = await openSession(signal)
|
||||
let closePromise: Promise<void> | undefined
|
||||
let abortedCallbackBreaker: ReturnType<typeof setTimeout> | undefined
|
||||
const closeSession = (reason?: unknown): Promise<void> => {
|
||||
closePromise ??= Promise.resolve().then(() => session.close(reason))
|
||||
return closePromise
|
||||
}
|
||||
const clearAbortedCallbackBreaker = (): void => {
|
||||
clearTimeout(abortedCallbackBreaker)
|
||||
abortedCallbackBreaker = undefined
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
// Why: closing the session is what releases callbacks and their borrowed source buffers.
|
||||
void closeSession(signal.reason).catch(() => {})
|
||||
if (abortedCallbackBreaker || closePromise) {
|
||||
return
|
||||
}
|
||||
// Why: normal cancellation gets a cleanup opportunity, but a retained raw callback needs close.
|
||||
abortedCallbackBreaker = setTimeout(() => {
|
||||
abortedCallbackBreaker = undefined
|
||||
void closeSession(signal.reason).catch(() => {})
|
||||
}, ABORTED_CALLBACK_BREAKER_MS)
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true })
|
||||
if (signal.aborted) {
|
||||
@@ -239,6 +251,8 @@ export async function transferSshRelayRuntimeTreeViaSftp(
|
||||
signal.throwIfAborted()
|
||||
return Object.freeze({ remoteStagingRoot, ...result })
|
||||
} catch (error) {
|
||||
// The stream reached a settled callback boundary, so reverse cleanup can safely own shutdown.
|
||||
clearAbortedCallbackBreaker()
|
||||
const cleanupFailures =
|
||||
ownedDirectories.length === 0
|
||||
? []
|
||||
@@ -246,6 +260,12 @@ export async function transferSshRelayRuntimeTreeViaSftp(
|
||||
await closeSession(error).catch((closeError) => cleanupFailures.push(closeError))
|
||||
throw joinedFailure(error, cleanupFailures)
|
||||
} finally {
|
||||
clearAbortedCallbackBreaker()
|
||||
signal.removeEventListener('abort', onAbort)
|
||||
}
|
||||
}
|
||||
|
||||
export const SSH_RELAY_RUNTIME_SFTP_TREE_TRANSFER_LIMITS = Object.freeze({
|
||||
maximumConcurrentFiles: MAXIMUM_CONCURRENT_FILES,
|
||||
abortedCallbackBreakerMs: ABORTED_CALLBACK_BREAKER_MS
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user