mirror of
https://github.com/stablyai/orca.git
synced 2026-10-18 00:03:20 +00:00
fix(ssh): settle Windows fixture sessions
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo-H
Orca
@@ -1697,6 +1697,36 @@ jobs:
|
||||
if (Test-Path -LiteralPath $log) { Get-Content -LiteralPath $log -Tail 200 }
|
||||
try {
|
||||
if ($ownsAccount -and $userSid) {
|
||||
function Get-FixtureOwnedProcesses([string]$fixtureSid) {
|
||||
return @(
|
||||
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object {
|
||||
$owner = Invoke-CimMethod -InputObject $_ -MethodName GetOwnerSid `
|
||||
-ErrorAction SilentlyContinue
|
||||
if ($owner.Sid -eq $fixtureSid) { $_ }
|
||||
}
|
||||
)
|
||||
}
|
||||
function Stop-FixtureOwnedProcesses([string]$fixtureSid) {
|
||||
for ($attempt = 1; $attempt -le 20; $attempt += 1) {
|
||||
$ownedProcesses = @(Get-FixtureOwnedProcesses $fixtureSid)
|
||||
if ($ownedProcesses.Count -eq 0) { return }
|
||||
foreach ($ownedProcess in $ownedProcesses) {
|
||||
Write-Output "fixture_owned_process=$($ownedProcess.Name):$($ownedProcess.ProcessId)"
|
||||
try {
|
||||
[void](Invoke-CimMethod -InputObject $ownedProcess -MethodName Terminate `
|
||||
-ErrorAction Stop)
|
||||
} catch {
|
||||
# The bounded ownership recheck handles processes that exit during termination.
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
$remaining = @(Get-FixtureOwnedProcesses $fixtureSid)
|
||||
if ($remaining.Count -gt 0) {
|
||||
$identities = $remaining | ForEach-Object { "$($_.Name):$($_.ProcessId)" }
|
||||
throw "Fixture account processes did not settle within 10 seconds: $($identities -join ',')"
|
||||
}
|
||||
}
|
||||
function Dismount-FixtureRegistryHive([string]$hiveName) {
|
||||
$registryPath = "Registry::HKEY_USERS\$hiveName"
|
||||
if (-not (Test-Path -LiteralPath $registryPath)) { return }
|
||||
@@ -1718,6 +1748,9 @@ jobs:
|
||||
$process.Dispose()
|
||||
}
|
||||
}
|
||||
# Why: sshd's service process can exit before a fixture-user
|
||||
# session descendant releases the profile registry handles.
|
||||
Stop-FixtureOwnedProcesses $userSid
|
||||
$escapedSid = $userSid.Replace("'", "''")
|
||||
$profileFailure = $null
|
||||
for ($attempt = 1; $attempt -le 20; $attempt += 1) {
|
||||
|
||||
@@ -74,6 +74,13 @@ export function assertWindowsOpenSshWorkflow(workflow, expect) {
|
||||
expect(stop.run).not.toContain('service-backup.json')
|
||||
expect(stop.run).not.toContain('sc.exe config')
|
||||
expect(stop.run).toContain('sc.exe delete')
|
||||
expect(stop.run).toContain('Get-FixtureOwnedProcesses $fixtureSid')
|
||||
expect(stop.run).toContain('$owner.Sid -eq $fixtureSid')
|
||||
expect(stop.run).toContain('-MethodName Terminate')
|
||||
expect(stop.run).toContain('Fixture account processes did not settle within 10 seconds')
|
||||
expect(stop.run.indexOf('Stop-FixtureOwnedProcesses $userSid')).toBeLessThan(
|
||||
stop.run.indexOf('Dismount-FixtureRegistryHive "${userSid}_Classes"')
|
||||
)
|
||||
expect(stop.run).toContain('Dismount-FixtureRegistryHive "${userSid}_Classes"')
|
||||
expect(stop.run).toContain('Dismount-FixtureRegistryHive $userSid')
|
||||
expect(stop.run).toContain("Join-Path $env:SystemRoot 'System32/reg.exe'")
|
||||
|
||||
+20
-2
@@ -86,8 +86,26 @@ fixture SID's `_Classes` and primary hives before retrying native profile deleti
|
||||
workflow oracle is split by concrete domain responsibility without a max-lines bypass. Focused
|
||||
tests pass 115 cases with one declared native skip, all three PowerShell blocks parse, 283/283
|
||||
artifact contracts pass, typecheck and full lint/reliability/max-lines pass, formatting/diff are
|
||||
clean, and the protected resolver files remain unchanged. Commit/push this isolated package and
|
||||
require fresh native x64/arm64 proof before advancing.
|
||||
clean, and the protected resolver files remain unchanged. Exact head
|
||||
`8ae6dcf6450b5f081788357dd8fe776293fb0cfe` is pushed; artifact run `29543292239` is in progress
|
||||
with native x64 job `87769913124` and ARM64 job `87769913148` assigned. Fresh service/auth/
|
||||
full-size/cancellation/collision/reuse/teardown proof remains required before advancing.
|
||||
X64 job `87769913124` is RED after preserving every earlier gate and starting official OpenSSH:
|
||||
the probe still reports `System SSH connection timed out` after 31.337 seconds despite `-n`.
|
||||
Teardown proves the owned `_Classes` hive exists but `reg.exe unload` returns exit 1, then catches
|
||||
the same locked `UsrClass.dat`. Record native process exit/stdout-end/sentinel/channel-close state
|
||||
and settle fixture-user session processes before the hive loop. ARM64 job `87769913148`
|
||||
independently reproduces the same boundary: probe timeout after 31.06 seconds, `_Classes` unload
|
||||
exit 1, locked `UsrClass.dat`, and retained profile directory. Do not advance the milestone on
|
||||
this result.
|
||||
`E-M6-WINDOWS-SYSTEM-SSH-TREE-LIFECYCLE-DIAGNOSTIC-LOCAL-001` is locally green: timeout errors
|
||||
record only sentinel/stdout-end/process-exit/channel-close state, and fixture teardown enumerates
|
||||
and boundedly terminates only processes whose native owner SID exactly matches the account created
|
||||
by the fixture before attempting hive unload. Focused tests pass 115 cases with one native skip,
|
||||
all three PowerShell blocks parse, 283/283 artifact contracts pass, typecheck and full
|
||||
lint/reliability/max-lines pass, formatting/diff are clean, and protected resolvers are unchanged.
|
||||
Commit/push and require fresh x64/arm64 lifecycle evidence; do not treat this local oracle as native
|
||||
proof.
|
||||
`E-M6-WINDOWS-SYSTEM-SSH-TREE-LIVE-AUDIT-001` fixes the loopback-only official Microsoft
|
||||
server, fixture-owned non-admin account/ACL, exact host-key trust, Windows PowerShell 5.1,
|
||||
serial/default and four-channel metrics, cancellation/collision/cleanup, and deterministic teardown
|
||||
|
||||
+75
-1
@@ -9,7 +9,7 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps
|
||||
Date created: 2026-07-14<br>
|
||||
Last updated: 2026-07-15<br>
|
||||
Current phase: Milestone 6 / Work Package 5 bounded runtime transfer — **In progress — 2026-07-15, Codex implementation owner: exact-head Windows OpenSSH live-proof diagnosis and fixture correction**. Keep every capability disconnected from every production/default SSH path. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
|
||||
Session checkpoint: **WINDOWS NATIVE NO-INPUT/PROFILE-HIVE CORRECTION LOCALLY GREEN / EXACT-HEAD PROOF NEXT — 2026-07-15, Codex implementation owner** — exact head `b652c1ebcce3803443d84c89811146366cdbdd5b`, artifact run `29542292712`, x64 job `87766845800`, and ARM64 job `87766845741` preserve all prior artifact/build/cache/hash/closure/signer gates and independently expose the same native probe/profile gaps. `E-M6-WINDOWS-SYSTEM-SSH-TREE-LIVE-NOINPUT-PROFILE-CORRECTION-LOCAL-001` adds probe-only OpenSSH `-n` before `--` while retaining connection reuse and excluding `-n` from ordinary commands; boundedly unloads only the fixture-owned SID's `_Classes` and primary hives before retrying native profile deletion; retains exact teardown assertions; and splits the Windows workflow oracle by concrete domain responsibility without a max-lines bypass. Focused tests, all three PowerShell parses, 283 artifact contracts, typecheck, full lint/reliability/max-lines, formatting, diff, and protected-resolver isolation pass locally. Commit/push this isolated package and require fresh x64/arm64 proof. Production runtime bytes still cross authenticated SSH and no product remote downloads anything. Do not connect a product caller, change legacy upload/fallback/default behavior, enable a tuple, publish an artifact, or perform SignPath work. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
|
||||
Session checkpoint: **WINDOWS NATIVE PROCESS/SESSION LIFECYCLE DIAGNOSTIC LOCALLY GREEN / EXACT-HEAD PROOF NEXT — 2026-07-15, Codex implementation owner** — exact head `b652c1ebcce3803443d84c89811146366cdbdd5b`, artifact run `29542292712`, x64 job `87766845800`, and ARM64 job `87766845741` preserve all prior artifact/build/cache/hash/closure/signer gates and independently expose the same native probe/profile gaps. Exact corrected head `8ae6dcf6450b5f081788357dd8fe776293fb0cfe`, run `29543292239`, x64 job `87769913124`, and ARM64 job `87769913148` remain RED: official OpenSSH starts, but probes still time out after 31.337/31.06 seconds and both fixture `_Classes` hive unloads return exit 1 before exact `UsrClass.dat`/directory failures. Win32 has no Orca control socket, so this is not an Orca ControlMaster retry path. `E-M6-WINDOWS-SYSTEM-SSH-TREE-LIFECYCLE-DIAGNOSTIC-LOCAL-001` adds non-sensitive sentinel/stdout-end/process-exit/channel-close timeout evidence and bounded exact-fixture-SID process settlement before hive removal; focused, PowerShell parse, 283-contract, typecheck, full lint/reliability/max-lines, formatting, diff, and protected-resolver gates pass locally. Commit/push and require fresh x64/arm64 lifecycle evidence. Do not advance Milestone 6. Production runtime bytes still cross authenticated SSH and no product remote downloads anything. Do not connect a product caller, change legacy upload/fallback/default behavior, enable a tuple, publish an artifact, or perform SignPath work. Merging to `main` remains prohibited, SignPath is deferred, and legacy remains the production default.<br>
|
||||
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)<br>
|
||||
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693)
|
||||
|
||||
@@ -19171,6 +19171,80 @@ config/scripts/ssh-relay-runtime-*.test.mjs` passes 50 files and 283/283 cases i
|
||||
the live Windows gate. Legacy remains the sole production/default path; SignPath and product
|
||||
caller/Beta/settings/fallback/tuple/publication work remain deferred.
|
||||
|
||||
### E-M6-WINDOWS-SYSTEM-SSH-TREE-LIVE-NOINPUT-LIFECYCLE-RED-001 — native no-input option alone does not settle x64 probe or owned profile
|
||||
|
||||
- Date/owner/head/run: 2026-07-15, Codex implementation owner; exact pushed head
|
||||
`8ae6dcf6450b5f081788357dd8fe776293fb0cfe`, artifact run `29543292239`, native x64 job
|
||||
`87769913124` (`windows-2022`, Windows Server 2022) and native ARM64 job `87769913148`
|
||||
(`windows-11-arm`).
|
||||
- Preserved evidence: 100 files / 854 tests pass with 18 declared skips in the native contract
|
||||
suite; exact build identity/smoke, full-size client cache, portable archive/library hashes and
|
||||
signer closure, exact fixture ACLs, official OpenSSH 10.0p2 startup, authentication, and
|
||||
PowerShell 5.1 all pass before the live transfer suite.
|
||||
- Probe RED: the exact full-size test fails after 31.337 seconds with
|
||||
`System SSH connection timed out`; total Vitest duration is 32.28 seconds. Probe-only `-n` did
|
||||
not establish child/channel settlement. `getControlSocketPath()` deliberately returns null on
|
||||
win32, so Orca does not inject ControlMaster/ControlPersist for this client and disabling reuse
|
||||
would not diagnose this failure.
|
||||
- Teardown RED: the owned fixture SID is
|
||||
`S-1-5-21-4137868374-391603265-4009536220-1003`; bounded `reg.exe unload` sees its `_Classes`
|
||||
hive but returns exit 1 throughout the ten-second retry. Exact filesystem deletion then identifies
|
||||
`C:\Users\orca_ssh_fixture\AppData\Local\Microsoft\Windows\UsrClass.dat` as in use and the
|
||||
profile directory remains. The service PID settlement check alone is insufficient evidence that
|
||||
every fixture-user `sshd-session.exe` or descendant has exited.
|
||||
- ARM64 independent RED: official OpenSSH startup passes, then the exact full-size test times out
|
||||
after 31.06 seconds with the same probe error. Its fixture SID
|
||||
`S-1-5-21-1882319117-3219095328-2125279949-1001` has the same `_Classes` unload exit 1,
|
||||
`UsrClass.dat` lock, and retained profile directory. The lifecycle gap is therefore shared by
|
||||
both native Windows architectures.
|
||||
- Correction gate: first add bounded, non-sensitive lifecycle evidence for native process exit,
|
||||
stdout end, sentinel observation, and channel close. Before hive unload, enumerate and boundedly
|
||||
settle only processes owned by the fixture account SID, retaining exact service/account/profile/
|
||||
directory absence assertions. Update the workflow oracle, parse all PowerShell blocks, rerun the
|
||||
focused/283/static/format/isolation gates, then require fresh x64/arm64 proof. Legacy remains the
|
||||
sole production/default path and SignPath remains deferred.
|
||||
|
||||
### E-M6-WINDOWS-SYSTEM-SSH-TREE-LIFECYCLE-DIAGNOSTIC-LOCAL-001 — native probe state and exact-SID session settlement are locally green
|
||||
|
||||
- Date/owner/base/runner: 2026-07-15, Codex implementation owner; uncommitted diagnostic package
|
||||
atop exact RED head `8ae6dcf6450b5f081788357dd8fe776293fb0cfe` on Apple arm64, macOS 26.2
|
||||
build 25C56, Node 26.5.0, pnpm 10.24.0. Native Windows execution remains required.
|
||||
- Probe evidence: the existing fail-closed 30-second timeout now reports whether the structured
|
||||
sentinel was observed, stdout ended, the native child emitted an exit code, and the channel close
|
||||
remained absent. The fields are booleans/an exit code only and expose no target, identity, path,
|
||||
command, or credential. The focused test proves exact values and listener cleanup without changing
|
||||
the success condition or legacy/default behavior.
|
||||
- Fixture correction: after service PID settlement and before registry/profile removal, teardown
|
||||
enumerates `Win32_Process`, calls native `GetOwnerSid`, and acts only when the result exactly
|
||||
equals the SID of the fixture-created account. Termination and ownership rechecks are bounded to
|
||||
twenty 500ms attempts; remaining process names/PIDs cause failure. `_Classes`/primary hive unload,
|
||||
native profile deletion, account deletion, filesystem deletion, and exact absence assertions all
|
||||
remain mandatory.
|
||||
- Focused command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts
|
||||
--maxWorkers=1 src/main/ssh/ssh-connection.test.ts src/main/ssh/ssh-system-fallback.test.ts
|
||||
src/main/ssh/ssh-relay-runtime-windows-system-ssh-openssh-full-size.test.ts
|
||||
config/scripts/ssh-relay-runtime-workflow.test.mjs` passes 3 files plus one declared native skip,
|
||||
115 passed tests plus one skip, in 4.34 seconds real; maximum RSS is 166,199,296 bytes and peak
|
||||
footprint is 98,408,208 bytes. Independent YAML extraction plus PowerShell
|
||||
`[scriptblock]::Create()` parses all three start/measure/stop blocks.
|
||||
- Release-contract command: `/usr/bin/time -l pnpm exec vitest run --config
|
||||
config/vitest.config.ts --maxWorkers=1 --reporter=dot
|
||||
config/scripts/ssh-relay-runtime-*.test.mjs` passes 50 files and 283/283 cases in 24.94 seconds
|
||||
real; maximum RSS is 195,788,800 bytes and peak footprint is 97,654,544 bytes.
|
||||
- Static commands: `/usr/bin/time -l pnpm typecheck` passes in 5.02 seconds real with maximum RSS
|
||||
1,236,762,624 bytes and peak footprint 97,965,912 bytes. `/usr/bin/time -l pnpm lint` passes in
|
||||
24.59 seconds real with only the 26 pre-existing warnings, all 41 reliability gates, the
|
||||
355-entry max-lines ratchet, bundled guides, and localization checks; maximum RSS is
|
||||
1,979,744,256 bytes and peak footprint is 97,572,672 bytes.
|
||||
- Isolation evidence: targeted `oxfmt --check`, `git diff --check`, and exact working-tree equality
|
||||
to `HEAD` for `ssh-remote-node-resolution.ts` and its test pass. Only the artifact workflow,
|
||||
purpose-named Windows workflow oracle, system-SSH probe diagnostics/test, and three planning
|
||||
artifacts are touched.
|
||||
- Residual boundary: local macOS proof cannot execute native Windows process ownership/termination,
|
||||
`ssh.exe` lifecycle, registry hive unload, profile removal, or the live transfer. Commit/push and
|
||||
require fresh exact-head x64/arm64 proof. Legacy remains the sole production/default path;
|
||||
SignPath and product caller/Beta/settings/fallback/tuple/publication work remain deferred.
|
||||
|
||||
## Accepted Gaps
|
||||
|
||||
No product gap is accepted merely because it appears in this list. Each entry requires explicit
|
||||
|
||||
@@ -1642,10 +1642,10 @@
|
||||
<td>
|
||||
Full-size runtime over SFTP and POSIX/Windows system SSH; tar/no-tar; no remote
|
||||
hash tools; corrupted upload; bundled-Node corruption/failure; bounded memory
|
||||
and concurrency; no-input system-SSH probe uses native OpenSSH <code>-n</code>
|
||||
before the destination terminator and proves child settlement without disabling
|
||||
connection reuse; fixture-owned Windows profile hives are boundedly unloaded
|
||||
before native profile deletion;
|
||||
and concurrency; no-input system-SSH probe records bounded native process exit,
|
||||
stdout-end, sentinel, and channel-close settlement rather than treating OpenSSH
|
||||
<code>-n</code> alone as proof; fixture-owned Windows session processes settle
|
||||
before profile hives are boundedly unloaded and native profile deletion begins;
|
||||
<code>MaxSessions=1</code>; interrupted upload; stale lock
|
||||
</td>
|
||||
<td>
|
||||
|
||||
@@ -1214,12 +1214,20 @@ describe('SshConnection', () => {
|
||||
const conn = new SshConnection(createTarget({ configHost: 'fdpass-host' }), createCallbacks())
|
||||
|
||||
try {
|
||||
const connect = expect(conn.connect()).rejects.toThrow('System SSH connection timed out')
|
||||
const connect = expect(conn.connect()).rejects.toThrow(
|
||||
'System SSH connection timed out (sentinel=true, stdoutEnded=true, processExit=0, channelClosed=false)'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
channel.emit('data', Buffer.from('ORCA-SYSTEM-SSH-OK\r\n'))
|
||||
channel.emit('end')
|
||||
channel.emit('exit', 0)
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
|
||||
await connect
|
||||
expect(channel.close).toHaveBeenCalled()
|
||||
expect(channel.listenerCount('data')).toBe(0)
|
||||
expect(channel.listenerCount('end')).toBe(0)
|
||||
expect(channel.listenerCount('exit')).toBe(0)
|
||||
expect(channel.listenerCount('error')).toBe(1)
|
||||
expect(channel.listenerCount('close')).toBe(1)
|
||||
expect(channel.stderr.listenerCount('data')).toBe(0)
|
||||
|
||||
@@ -702,9 +702,14 @@ export class SshConnection {
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let settled = false
|
||||
let sentinelObserved = false
|
||||
let stdoutEnded = false
|
||||
let processExit: number | null | 'not-observed' = 'not-observed'
|
||||
const cleanup = (): void => {
|
||||
clearTimeout(timeout)
|
||||
channel.off('data', onStdoutData)
|
||||
channel.off('end', onStdoutEnd)
|
||||
channel.off('exit', onProcessExit)
|
||||
channel.stderr.off('data', onStderrData)
|
||||
channel.off('error', onError)
|
||||
channel.off('close', onClose)
|
||||
@@ -720,6 +725,13 @@ export class SshConnection {
|
||||
}
|
||||
const onStdoutData = (data: Buffer): void => {
|
||||
stdout += data.toString('utf-8')
|
||||
sentinelObserved = stdout.includes('ORCA-SYSTEM-SSH-OK')
|
||||
}
|
||||
const onStdoutEnd = (): void => {
|
||||
stdoutEnded = true
|
||||
}
|
||||
const onProcessExit = (code: number | null): void => {
|
||||
processExit = code
|
||||
}
|
||||
const onStderrData = (data: Buffer): void => {
|
||||
stderr += data.toString('utf-8')
|
||||
@@ -748,11 +760,19 @@ export class SshConnection {
|
||||
const timeout = setTimeout(() => {
|
||||
settle(() => {
|
||||
channel.close()
|
||||
reject(new Error('System SSH connection timed out'))
|
||||
// Why: native Windows can separate process exit from inherited
|
||||
// stdio closure; these booleans diagnose that boundary safely.
|
||||
reject(
|
||||
new Error(
|
||||
`System SSH connection timed out (sentinel=${sentinelObserved}, stdoutEnded=${stdoutEnded}, processExit=${processExit}, channelClosed=false)`
|
||||
)
|
||||
)
|
||||
})
|
||||
}, CONNECT_TIMEOUT_MS)
|
||||
|
||||
channel.on('data', onStdoutData)
|
||||
channel.on('end', onStdoutEnd)
|
||||
channel.on('exit', onProcessExit)
|
||||
channel.stderr.on('data', onStderrData)
|
||||
channel.on('error', onError)
|
||||
channel.on('close', onClose)
|
||||
|
||||
Reference in New Issue
Block a user