mirror of
https://github.com/stablyai/orca.git
synced 2026-10-03 08:02:12 +00:00
ci: overlap ARM SSH setup and independent observation waits (#24714)
This commit is contained in:
Neil
@@ -84,10 +84,6 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
id: dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
- name: Self-test the provisioning scripts before touching the machine
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -97,6 +93,22 @@ jobs:
|
||||
if($errors.Count){throw "PowerShell parse failed: $($file.FullName)"}
|
||||
}
|
||||
& config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1
|
||||
# ARM inbox servicing can finish while the independent Node artifacts are prepared.
|
||||
- name: Prepare the Windows inbox SSH capability
|
||||
id: inbox-capability
|
||||
background: true
|
||||
shell: pwsh
|
||||
run: |
|
||||
if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){
|
||||
$receipts=Join-Path $pwd '.build/ssh-windows-host-receipts'
|
||||
New-Item -ItemType Directory -Force -Path $receipts | Out-Null
|
||||
. config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1
|
||||
Initialize-WindowsInboxSshCapability -Arch '${{ matrix.arch }}' -Receipt (Join-Path $receipts 'inbox-capability-preparation.json')
|
||||
}
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
id: dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
# The deploy materializes rung A from this template; only this runner's slot exists here.
|
||||
# The process-tree addon carries the launcher that starts the relay outside sshd's job; the
|
||||
# orcad slot and the relay both stage it, and a standard-user host has no other launch route.
|
||||
@@ -122,6 +134,7 @@ jobs:
|
||||
pnpm build:orcad-prebuilds --require-slots "win32-${{ matrix.arch }}"
|
||||
node config/scripts/build-orcad-template.mjs --targets "win32-${{ matrix.arch }}"
|
||||
pnpm run build:relay
|
||||
- wait: inbox-capability
|
||||
- name: Run the Windows host cells against a private ${{ matrix.server }} sshd
|
||||
shell: pwsh
|
||||
timeout-minutes: 50
|
||||
@@ -135,6 +148,8 @@ jobs:
|
||||
$cells=@($env:CELLS -split ',' | ForEach-Object {$_.Trim()} | Where-Object {$_})
|
||||
if(-not $cells.Count){$cells=@('pinned-cmd','pinned-powershell','legacy-opt-out')}
|
||||
$archive=''
|
||||
$preparation=''
|
||||
if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation=Join-Path $receipts 'inbox-capability-preparation.json'}
|
||||
if('${{ matrix.server }}' -eq 'preview'){
|
||||
$archive=Join-Path $env:RUNNER_TEMP 'preview-${{ matrix.archive }}'
|
||||
# The provisioning script refuses the archive unless its sha256 and every binary's match the pin.
|
||||
@@ -144,7 +159,7 @@ jobs:
|
||||
$callback={param($context)
|
||||
& (Join-Path $tools 'invoke-pinned-relay-cells.ps1') -SourceRoot $sourceRoot -Context $context -Target 'win32-${{ matrix.arch }}' -ReceiptRoot $receipts -Cells $cells
|
||||
}.GetNewClosure()
|
||||
& (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -Accounts $cells.Count -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log')
|
||||
& (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -InboxPreparationReceipt $preparation -Accounts $cells.Count -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log')
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
|
||||
@@ -2,46 +2,34 @@
|
||||
# -HiddenTools: the private accounts are denied every machine PATH directory holding one of these
|
||||
# executables, and their own PATH carries logging shims for them, so SSH sessions have no host toolchain.
|
||||
# -HostCellProbe receives a context hashtable (accounts, port, keys, shim log) once provisioning passes.
|
||||
param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,4)][int]$Accounts=1,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe)
|
||||
param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,4)][int]$Accounts=1,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe,[string]$InboxPreparationReceipt)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1')
|
||||
$target=@{arm64=@{os='Arm64';folder='OpenSSH-ARM64';machine='0xAA64';archive='698c6aec31c1dd0fb996206e8741f4531a97355686b5431ef347d531b07fcd42'};x64=@{os='X64';folder='OpenSSH-Win64';machine='0x8664';archive='23f50f3458c4c5d0b12217c6a5ddfde0137210a30fa870e98b29827f7b43aba5'}}[$Arch]
|
||||
$scopeServer=if($Server -eq 'inbox'){'Windows inbox OpenSSH.Server capability binaries'}else{'Microsoft Win32-OpenSSH 10.0.0.0p2-Preview'}
|
||||
$report = @{scope="$scopeServer $Arch private loopback authentication and stock cmd.exe dispatch"; server=$Server; status='running'; imageVersion=$env:ImageVersion; cleanup=@('not-confirmed'); globalBootstrapCleanup='Not qualified: service bootstrap may create ProgramData SSH and OpenSSH registry entries; disposable CI VM destruction is the boundary'; observations=@(); stages=@(); diagnosticCaptureFailures=@()}
|
||||
$script:receiptWritten=$false
|
||||
function Write-Stage([string]$Stage) {
|
||||
$timestamp=[DateTime]::UtcNow.ToString('o')
|
||||
$report.stages += @{stage=$Stage; utc=$timestamp}
|
||||
try {
|
||||
$bytes=[Text.UTF8Encoding]::new($false).GetBytes(($report | ConvertTo-Json -Depth 6))
|
||||
$temporary="$Receipt.pending"
|
||||
$stream=[IO.FileStream]::new($temporary,[IO.FileMode]::Create,[IO.FileAccess]::Write,[IO.FileShare]::Read)
|
||||
try {$stream.Write($bytes,0,$bytes.Length);$stream.Flush($true)} finally {$stream.Dispose()}
|
||||
[IO.File]::Move($temporary,$Receipt,$true)
|
||||
$script:receiptWritten=$true
|
||||
} catch {Write-Warning 'Progress receipt could not be updated; cleanup must still run'}
|
||||
Write-Host "Native SSH stage: $Stage ($timestamp)"
|
||||
if(Write-WindowsSshReceiptStage $report $Receipt $Stage){$script:receiptWritten=$true}
|
||||
}
|
||||
Write-Stage 'preflight-start'
|
||||
if(-not $script:receiptWritten){throw 'Initial progress receipt unavailable; refuse provisioning'}
|
||||
if ($env:GITHUB_ACTIONS -ne 'true' -or $env:ORCA_ISOLATED_SSH_CI -ne '1' -or [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $target.os) { throw "Requires isolated native $Arch GitHub runner" }
|
||||
$admin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
|
||||
if (-not $admin) { throw 'Administrative private service/account setup required' }
|
||||
Assert-IsolatedWindowsSshCi $Arch
|
||||
Write-Stage 'existing-server-query-start'
|
||||
$inboxDir=Join-Path $env:WINDIR 'System32\OpenSSH'
|
||||
function Assert-GlobalServerDormant {
|
||||
$global=Get-CimInstance Win32_Service -Filter "Name='sshd'"
|
||||
if(-not $global){return}
|
||||
# Inbox mode may register the global service; it must stay stopped and never be started here.
|
||||
if($Server -ne 'inbox' -or $global.State -ne 'Stopped' -or $global.PathName.Trim('"') -ne (Join-Path $inboxDir 'sshd.exe')){throw 'Refuse an existing global SSH server'}
|
||||
}
|
||||
Assert-GlobalServerDormant
|
||||
Assert-WindowsSshGlobalServerDormant $Server
|
||||
Write-Stage 'existing-server-query-complete'
|
||||
Write-Stage 'default-shell-query-start'
|
||||
$openSshKey = 'HKLM:\SOFTWARE\OpenSSH'
|
||||
$registry = Get-ItemProperty -LiteralPath $openSshKey -ErrorAction SilentlyContinue
|
||||
# Host-cell probes may set DefaultShell per cell; cleanup restores this stock state.
|
||||
if ($registry.DefaultShell -or $registry.DefaultShellCommandOption) { throw 'Requires stock cmd.exe OpenSSH shell at start' }
|
||||
Assert-WindowsSshStockShell
|
||||
Write-Stage 'default-shell-query-complete'
|
||||
if($InboxPreparationReceipt){
|
||||
if($Server -ne 'inbox'){throw 'Inbox preparation receipt cannot qualify a preview server'}
|
||||
$preparation=Get-Content -LiteralPath $InboxPreparationReceipt -Raw | ConvertFrom-Json
|
||||
if($preparation.status -ne 'passed' -or $preparation.arch -ne $Arch -or $preparation.sourceSha -ne $env:GITHUB_SHA -or $preparation.runId -ne $env:GITHUB_RUN_ID -or $preparation.runAttempt -ne $env:GITHUB_RUN_ATTEMPT -or $preparation.runnerName -ne $env:RUNNER_NAME -or $preparation.imageVersion -ne $env:ImageVersion){throw 'Inbox preparation receipt identity or verdict mismatch'}
|
||||
$report.inboxCapabilityPreparation=$preparation
|
||||
}
|
||||
$id = [Guid]::NewGuid().ToString('N').Substring(0,10)
|
||||
$name = "orca$id"
|
||||
$accountNames = @($name) + @(if($Accounts -gt 1){2..$Accounts | ForEach-Object {"$name$_"}})
|
||||
@@ -180,12 +168,7 @@ try {
|
||||
$report.nativeInputs=$verified
|
||||
Write-Stage 'preview-native-input-verification-complete'
|
||||
} else {
|
||||
Write-Stage 'inbox-capability-start'
|
||||
$capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0'
|
||||
$report.inboxCapabilityInitialState=[string]$capability.State
|
||||
if($capability.State -ne 'Installed'){Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' | Out-Null}
|
||||
Assert-GlobalServerDormant
|
||||
Write-Stage 'inbox-capability-complete'
|
||||
Install-WindowsInboxSshCapability $Arch $report {param($stage) Write-Stage $stage}
|
||||
$verified=@()
|
||||
foreach($binary in @('sshd.exe','ssh.exe','ssh-keygen.exe','sftp.exe','sftp-server.exe')){
|
||||
$path=Join-Path $sshDir $binary
|
||||
@@ -431,21 +414,38 @@ LogLevel DEBUG1
|
||||
foreach($directory in $deniedToolDirs){Invoke-Bounded icacls.exe (@($directory.TrimEnd('\'),'/remove:d')+@($ownedAccounts | ForEach-Object {"*$($_.sid)"})) 120 | Out-Null}
|
||||
Write-Stage 'cleanup-toolchain-acl-complete'
|
||||
Write-Stage 'cleanup-user-profile-start'
|
||||
$profileTargets=@(foreach($account in $ownedAccounts){
|
||||
if($account.sid){@{sid=$account.sid;watch=$null;done=$false;profiles=@();waitMs=0;disposition=$null}}
|
||||
})
|
||||
$report.profileCleanup=@()
|
||||
foreach($account in $ownedAccounts){
|
||||
if(-not $account.sid){continue}
|
||||
$profileWait=[Diagnostics.Stopwatch]::StartNew()
|
||||
do {
|
||||
$profiles=@(Get-CimInstance Win32_UserProfile | Where-Object SID -eq $account.sid)
|
||||
if(-not @($profiles | Where-Object Loaded).Count){break}
|
||||
Start-Sleep -Milliseconds 500
|
||||
} while($profileWait.Elapsed.TotalSeconds -lt 30)
|
||||
$report.profileUnloadWaitMs=$profileWait.ElapsedMilliseconds
|
||||
$report.privateProfile=@($profiles | ForEach-Object {@{loaded=$_.Loaded;status=$_.Status}})
|
||||
Write-Stage 'cleanup-user-profile-observed'
|
||||
$loadedProfiles=@($profiles | Where-Object Loaded)
|
||||
$report.profileCleanup+=if($loadedProfiles.Count){'Loaded profile retained for disposable CI VM destruction'}else{'Unloaded profile removed'}
|
||||
$profiles | Where-Object {-not $_.Loaded} | Remove-CimInstance
|
||||
$report.privateProfiles=@()
|
||||
do {
|
||||
foreach($entry in @($profileTargets | Where-Object {-not $_.done})){
|
||||
if(-not $entry.watch){$entry.watch=[Diagnostics.Stopwatch]::StartNew()}
|
||||
$profiles=@(Get-CimInstance Win32_UserProfile -Filter "SID='$($entry.sid)'")
|
||||
if(@($profiles | Where-Object SID -ne $entry.sid).Count){throw 'Private profile query returned an unrelated SID'}
|
||||
if(@($profiles | Where-Object Loaded).Count -and $entry.watch.ElapsedMilliseconds -lt 30000){continue}
|
||||
# A profile may reload after the polling snapshot.
|
||||
$profiles=@(Get-CimInstance Win32_UserProfile -Filter "SID='$($entry.sid)'")
|
||||
if(@($profiles | Where-Object SID -ne $entry.sid).Count){throw 'Private profile query returned an unrelated SID'}
|
||||
$loadedProfiles=@($profiles | Where-Object Loaded)
|
||||
if($loadedProfiles.Count -and $entry.watch.ElapsedMilliseconds -lt 30000){continue}
|
||||
$profiles | Where-Object {-not $_.Loaded} | Remove-CimInstance
|
||||
$entry.profiles=@($profiles | ForEach-Object {@{loaded=$_.Loaded;status=$_.Status}})
|
||||
$entry.waitMs=$entry.watch.ElapsedMilliseconds
|
||||
$entry.disposition=if($loadedProfiles.Count){'Loaded profile retained for disposable CI VM destruction'}else{'Unloaded profile removed'}
|
||||
$entry.done=$true
|
||||
$report.profileCleanup=@($profileTargets | Where-Object done | ForEach-Object disposition)
|
||||
$report.privateProfiles=@($profileTargets | Where-Object done | ForEach-Object {@{sid=$_.sid;waitMs=$_.waitMs;profiles=$_.profiles;disposition=$_.disposition}})
|
||||
$report.profileUnloadWaitMs=$entry.waitMs
|
||||
$report.privateProfile=$entry.profiles
|
||||
Write-Stage 'cleanup-user-profile-observed'
|
||||
}
|
||||
if(@($profileTargets | Where-Object {-not $_.done}).Count){Start-Sleep -Milliseconds 500}
|
||||
} while(@($profileTargets | Where-Object {-not $_.done}).Count)
|
||||
if($profileTargets.Count){
|
||||
$report.profileUnloadWaitMs=$profileTargets[-1].waitMs
|
||||
$report.privateProfile=$profileTargets[-1].profiles
|
||||
}
|
||||
Write-Stage 'cleanup-user-profile-complete'
|
||||
Write-Stage 'cleanup-user-start'
|
||||
|
||||
@@ -33,5 +33,166 @@ try {
|
||||
if(($result.hidden -join '|') -ne "$nodeDir|$gccDir"){throw 'Toolchain PATH entries not hidden'}
|
||||
if(($result.kept -join '|') -ne "$plainDir|$(Join-Path $pathRoot 'missing')|Q:\no-such-drive"){throw 'Plain PATH entries not kept in order'}
|
||||
} finally {Remove-Item -LiteralPath $pathRoot -Recurse -Force -ErrorAction SilentlyContinue}
|
||||
# Mock only the machine boundary; exercise the shared installer without servicing this host.
|
||||
. (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1')
|
||||
function Assert-IsolatedWindowsSshCi([string]$Arch){if($script:refuseCapabilityHost){throw 'Injected host refusal'}}
|
||||
function Assert-WindowsSshGlobalServerDormant([string]$Server){$script:globalChecks++;if($script:globalChecks -eq $script:refuseGlobalCheck){throw 'Injected global server refusal'}}
|
||||
function Assert-WindowsSshStockShell {$script:shellChecks++;if($script:shellChecks -eq $script:refuseShellCheck){throw 'Injected shell refusal'}}
|
||||
function Get-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityQueries++;return @{State=$script:capabilityState}}
|
||||
function Add-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityAdds++;if($script:failCapabilityInstall){throw 'Injected capability install failure'}}
|
||||
function Reset-CapabilityControl([string]$State){
|
||||
$script:capabilityState=$State;$script:capabilityQueries=0;$script:capabilityAdds=0
|
||||
$script:globalChecks=0;$script:shellChecks=0;$script:refuseGlobalCheck=0;$script:refuseShellCheck=0
|
||||
$script:refuseCapabilityHost=$false;$script:failCapabilityInstall=$false
|
||||
$script:capabilityStages=[Collections.Generic.List[string]]::new()
|
||||
}
|
||||
foreach($state in @('Installed','NotPresent')){
|
||||
Reset-CapabilityControl $state
|
||||
$capabilityReport=@{}
|
||||
Install-WindowsInboxSshCapability 'x64' $capabilityReport {param($name) $script:capabilityStages.Add($name)}
|
||||
$expectedAdds=if($state -eq 'Installed'){0}else{1}
|
||||
if($capabilityReport.inboxCapabilityInitialState -ne $state -or $script:capabilityAdds -ne $expectedAdds -or $script:globalChecks -ne 2 -or $script:shellChecks -ne 2 -or ($script:capabilityStages -join ',') -ne 'inbox-capability-start,inbox-capability-complete'){throw 'Shared capability preparation did not preserve the install and guard boundaries'}
|
||||
}
|
||||
foreach($fault in @('host','global-before','shell-before','global-after','shell-after','install')){
|
||||
Reset-CapabilityControl 'NotPresent'
|
||||
switch($fault){
|
||||
'host' {$script:refuseCapabilityHost=$true}
|
||||
'global-before' {$script:refuseGlobalCheck=1}
|
||||
'shell-before' {$script:refuseShellCheck=1}
|
||||
'global-after' {$script:refuseGlobalCheck=2}
|
||||
'shell-after' {$script:refuseShellCheck=2}
|
||||
'install' {$script:failCapabilityInstall=$true}
|
||||
}
|
||||
$rejected=$false
|
||||
try {Install-WindowsInboxSshCapability 'x64' @{} {param($name) $script:capabilityStages.Add($name)}} catch {$rejected=$true}
|
||||
if(-not $rejected -or $script:capabilityStages.Contains('inbox-capability-complete')){throw "Capability fault did not fail closed: $fault"}
|
||||
if($fault -in @('host','global-before','shell-before') -and $script:capabilityAdds){throw 'Capability mutation preceded its host guards'}
|
||||
}
|
||||
$capabilityRoot=Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString('N'))
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $capabilityRoot | Out-Null
|
||||
$capabilityReceipt=Join-Path $capabilityRoot 'capability.json'
|
||||
Reset-CapabilityControl 'Installed'
|
||||
Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt
|
||||
$completed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if($completed.status -ne 'passed' -or $completed.inboxCapabilityInitialState -ne 'Installed'){throw 'Capability success receipt missing its observed initial state'}
|
||||
Reset-CapabilityControl 'NotPresent';$script:failCapabilityInstall=$true
|
||||
$rejected=$false
|
||||
try {Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt} catch {$rejected=$true}
|
||||
$failed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if(-not $rejected -or $failed.status -ne 'failed' -or $failed.inboxCapabilityInitialState -ne 'NotPresent' -or $failed.error -ne 'Injected capability install failure'){throw 'Failed capability preparation masqueraded as a passing receipt'}
|
||||
} finally {Remove-Item -LiteralPath $capabilityRoot -Recurse -Force -ErrorAction SilentlyContinue}
|
||||
& {
|
||||
param($ProofAst)
|
||||
$cleanup=@($ProofAst.FindAll({param($node) $node -is [Management.Automation.Language.TryStatementAst] -and $node.Body.Extent.Text.Contains("Write-Stage 'cleanup-start'")},$true))
|
||||
if($cleanup.Count -ne 1 -or -not $cleanup[0].Extent.Text.Contains('[Diagnostics.Stopwatch]::StartNew()')){throw 'Cleanup clock boundary missing'}
|
||||
# Run the real cleanup gates with virtual clocks; no Windows machine operation may escape these mocks.
|
||||
$cleanupBlock=[scriptblock]::Create($cleanup[0].Extent.Text.Replace('[Diagnostics.Stopwatch]::StartNew()','(New-ProfileControlStopwatch)').Replace('[DateTime]::UtcNow','(Get-ProfileControlUtcNow)'))
|
||||
$ownedSids=@('S-1-5-21-100-200-300-1001','S-1-5-21-100-200-300-1002','S-1-5-21-100-200-300-1003')
|
||||
$foreignSid='S-1-5-21-100-200-300-9000';$control=@{}
|
||||
function New-ProfileControlStopwatch {
|
||||
$watch=[pscustomobject]@{StartedMilliseconds=$control.clockMs;Control=$control}
|
||||
$watch | Add-Member ScriptProperty ElapsedMilliseconds {$this.Control.clockMs-$this.StartedMilliseconds}
|
||||
return $watch
|
||||
}
|
||||
function Get-ProfileControlUtcNow {[DateTime]::new(2026,10,2,0,0,0,[DateTimeKind]::Utc).AddMilliseconds($control.clockMs)}
|
||||
function Start-Sleep([int]$Milliseconds,[int]$Seconds){$control.clockMs+=$Milliseconds+1000*$Seconds}
|
||||
function Write-Stage([string]$Stage){$control.stages.Add($Stage)}
|
||||
function Record-PrivateServiceDiagnostics([switch]$AfterStop){}
|
||||
function Test-Path([string]$LiteralPath){if($LiteralPath -ne 'HKLM:\SOFTWARE\OpenSSH'){throw 'Unexpected filesystem query'};return $false}
|
||||
function Get-ItemProperty([string]$LiteralPath,[object]$ErrorAction){if($LiteralPath -ne 'HKLM:\SOFTWARE\OpenSSH'){throw 'Unexpected registry query'};return $null}
|
||||
function Remove-ItemProperty {throw 'Unexpected registry mutation'}
|
||||
function Stop-Service([string]$Name,[switch]$Force,[object]$ErrorAction){if($Name -ne 'orca-sshd-control'){throw 'Foreign service stop'};if($control.case.Fault -ne 'server-exit'){$control.serverLive=$false}}
|
||||
function Invoke-Bounded([string]$Program,[string[]]$Arguments){if($Program -ne 'sc.exe' -or ($Arguments -join '|') -ne 'delete|orca-sshd-control'){throw 'Unexpected native command'};if($control.case.Fault -ne 'service-absence'){$control.servicePresent=$false}}
|
||||
function Get-Process([int]$Id,[object]$ErrorAction){if($Id -ne 100){throw 'Foreign process query'};if($control.serverLive){@{Id=100}}}
|
||||
function Get-Service([string]$Name,[object]$ErrorAction){if($Name -ne 'orca-sshd-control'){throw 'Foreign service query'};if($control.servicePresent){@{Name=$Name}}}
|
||||
function Get-ProfileControlLoaded([string]$Sid){
|
||||
$state=$control.profiles[$Sid]
|
||||
if($state.Mode -eq 'late'){return $control.clockMs -lt $state.UnloadAtMs}
|
||||
if($state.Mode -in @('reload','reload-at-deadline')){return $state.CurrentLoaded}
|
||||
return $state.Mode -eq 'loaded'
|
||||
}
|
||||
function Get-CimInstance([Parameter(Position=0)][string]$ClassName,[string]$Filter){
|
||||
if($ClassName -eq 'Win32_Service'){
|
||||
if($Filter -ne "Name='orca-sshd-control'"){throw 'Foreign service query'}
|
||||
if($control.servicePresent){@{PathName=$(if($control.case.Fault -eq 'service-identity'){'C:\foreign\sshd.exe'}else{'C:\fake\ossh-control\sshd.exe'});ProcessId=$(if($control.case.Fault -eq 'service-pid'){200}else{100})}};return
|
||||
}
|
||||
if($ClassName -eq 'Win32_Process'){
|
||||
if($control.case.Fault -eq 'child-exit'){@{ExecutablePath='C:\fake\OpenSSH\session.exe';ProcessId=101;ParentProcessId=100;CreationDate=(Get-ProfileControlUtcNow)}};return
|
||||
}
|
||||
if($ClassName -ne 'Win32_UserProfile' -or $Filter -notmatch "^SID='(S-1-5-21-100-200-300-\d+)'$" -or $Matches[1] -notin $ownedSids){throw 'Profile query must target an owned SID'}
|
||||
$sid=$Matches[1];$control.clockMs+=$control.case.QueryCostMs;$control.profileQueries[$sid]++
|
||||
if($control.case.Fault -eq 'query' -and $sid -eq $ownedSids[1]){throw 'Injected profile query failure'}
|
||||
if($control.case.Fault -eq 'foreign-result' -and $sid -eq $ownedSids[1]){[pscustomobject]@{SID=$foreignSid;Loaded=$false;Status=0};return}
|
||||
$state=$control.profiles[$sid]
|
||||
if($state.Mode -eq 'absent' -or $state.Deleted){return}
|
||||
$loaded=Get-ProfileControlLoaded $sid
|
||||
if($state.Mode -eq 'reload' -and $control.profileQueries[$sid] -eq 1){$loaded=$false;$state.CurrentLoaded=$true}
|
||||
if($state.Mode -eq 'reload-at-deadline' -and $control.clockMs -ge 30000 -and -not $state.Reinjected){$loaded=$false;$state.CurrentLoaded=$true;$state.Reinjected=$true}
|
||||
[pscustomobject]@{SID=$sid;Loaded=$loaded;Status=0}
|
||||
}
|
||||
function Remove-CimInstance {
|
||||
param([Parameter(ValueFromPipeline=$true)][object]$InputObject)
|
||||
process {
|
||||
if($InputObject.SID -notin $ownedSids -or $InputObject.Loaded -or (Get-ProfileControlLoaded $InputObject.SID)){throw 'Unsafe profile deletion attempted'}
|
||||
if($control.case.Fault -eq 'delete'){throw 'Injected profile deletion failure'}
|
||||
$control.profiles[$InputObject.SID].Deleted=$true;$control.removed.Add($InputObject.SID)
|
||||
}
|
||||
}
|
||||
function Get-LocalUser([string]$Name,[object]$ErrorAction){if(-not $control.users.ContainsKey($Name)){throw 'Foreign account query'};if($control.users[$Name]){@{Name=$Name}}}
|
||||
function Remove-LocalUser([string]$Name){if(-not $control.users.ContainsKey($Name)){throw 'Foreign account deletion'};if($control.case.Fault -ne 'account'){$control.users[$Name]=$false}}
|
||||
function Remove-Item([string]$LiteralPath,[switch]$Recurse,[switch]$Force){if($LiteralPath -ne 'C:\fake\ossh-control'){throw 'Foreign filesystem deletion'};if($control.case.Fault -eq 'keys'){throw 'Injected private key deletion failure'};$control.keysRemoved=$true}
|
||||
function Assert-ProfileCleanup([hashtable]$Case){
|
||||
$control.Clear();$control.case=$Case;$control.clockMs=0;$control.serverLive=$true;$control.servicePresent=$true;$control.keysRemoved=$false
|
||||
$control.stages=[Collections.Generic.List[string]]::new();$control.removed=[Collections.Generic.List[string]]::new()
|
||||
$control.profiles=@{};$control.users=@{};$control.profileQueries=@{}
|
||||
$ownedAccounts=@(for($index=0;$index -lt $ownedSids.Count;$index++){
|
||||
$sid=$ownedSids[$index];$control.profileQueries[$sid]=0
|
||||
$control.profiles[$sid]=@{Mode=$Case.Modes[$index];UnloadAtMs=$Case.UnloadAtMs[$index];CurrentLoaded=$true;Deleted=$false}
|
||||
$name="orca-control-$index";$control.users[$name]=$true;@{name=$name;sid=$(if($Case.MissingSid -and $index -eq 2){$null}else{$sid})}
|
||||
})
|
||||
$report=@{status='proof-passed-cleanup-pending'};$openSshKey='HKLM:\SOFTWARE\OpenSSH';$serviceName='orca-sshd-control'
|
||||
$root='C:\fake\ossh-control';$createdService=$true;$ownedServerPid=100;$sshDir='C:\fake\OpenSSH';$preexisting=@();$deniedToolDirs=@()
|
||||
& $cleanupBlock
|
||||
if($Case.Fault){
|
||||
if($report.status -ne 'failed' -or $report.cleanupError -ne $Case.Error -or $control.keysRemoved -or $control.stages.Contains('cleanup-private-files-complete')){throw "Cleanup fault did not fail at its gate: $($Case.Name)"}
|
||||
if($Case.Fault -notin @('account','keys') -and @($control.users.Values | Where-Object {-not $_}).Count){throw "Failure bypassed account cleanup gate: $($Case.Name)"}
|
||||
return
|
||||
}
|
||||
if($report.status -ne 'passed' -or @($control.users.Values | Where-Object {$_}).Count -or -not $control.keysRemoved){throw "Cleanup omitted account/key gates: $($Case.Name)"}
|
||||
if(($control.removed.ToArray() | Sort-Object) -join ',' -ne (($Case.Removed | Sort-Object) -join ',')){throw "Wrong removed SIDs: $($Case.Name)"}
|
||||
if(@($report.profileCleanup | Where-Object {$_ -eq 'Loaded profile retained for disposable CI VM destruction'}).Count -ne $Case.Retained){throw "Wrong retained disposition: $($Case.Name)"}
|
||||
foreach($sid in $Case.FullWindow){
|
||||
$observed=@($report.privateProfiles | Where-Object sid -eq $sid)
|
||||
if($observed.Count -ne 1 -or $observed[0].waitMs -lt 30000){throw "Short profile observation window: $($Case.Name)"}
|
||||
}
|
||||
if($control.clockMs -gt $Case.MaxClockMs){throw "Profile windows were serialized: $($Case.Name)"}
|
||||
if($Case.MissingSid -and $control.profileQueries[$ownedSids[2]]){throw 'Missing SID gained profile deletion authority'}
|
||||
if($report.privateProfiles.Count -and ($report.profileUnloadWaitMs -ne $report.privateProfiles[-1].waitMs -or ($report.privateProfile | ConvertTo-Json -Compress) -ne ($report.privateProfiles[-1].profiles | ConvertTo-Json -Compress))){throw 'Legacy scalar observation lost owned-account order'}
|
||||
}
|
||||
$cases=@(
|
||||
@{Name='three loaded full windows';Modes=@('loaded','loaded','loaded');Retained=3;Removed=@();FullWindow=$ownedSids},
|
||||
@{Name='unload at 29999ms';Modes=@('late','unloaded','absent');UnloadAtMs=@(29999,0,0);Retained=0;Removed=$ownedSids[0..1];FullWindow=@($ownedSids[0])},
|
||||
@{Name='reload before deletion';Modes=@('reload','unloaded','absent');Retained=1;Removed=@($ownedSids[1]);FullWindow=@($ownedSids[0])},
|
||||
@{Name='independent unloads';Modes=@('late','loaded','late');UnloadAtMs=@(5000,0,20000);Retained=1;Removed=@($ownedSids[0],$ownedSids[2]);FullWindow=@($ownedSids[1])},
|
||||
@{Name='slow provider queries';Modes=@('loaded','loaded','loaded');QueryCostMs=200;Retained=3;Removed=@();FullWindow=$ownedSids;MaxClockMs=40000},
|
||||
@{Name='reload at expired window';Modes=@('reload-at-deadline','unloaded','absent');Retained=1;Removed=@($ownedSids[1]);FullWindow=@($ownedSids[0])},
|
||||
@{Name='late unload honestly retained';Modes=@('loaded','loaded','late');UnloadAtMs=@(0,0,45000);Retained=3;Removed=@();FullWindow=$ownedSids},
|
||||
@{Name='missing SID is skipped';Modes=@('unloaded','absent','unloaded');Retained=0;Removed=@($ownedSids[0]);MissingSid=$true}
|
||||
)
|
||||
foreach($case in $cases){
|
||||
if(-not $case.UnloadAtMs){$case.UnloadAtMs=@(0,0,0)}
|
||||
if(-not $case.MaxClockMs){$case.MaxClockMs=33000}
|
||||
Assert-ProfileCleanup $case
|
||||
}
|
||||
$failures=@{
|
||||
query='Injected profile query failure';delete='Injected profile deletion failure';'foreign-result'='Private profile query returned an unrelated SID'
|
||||
'service-identity'='Private service identity changed; refuse stop';'service-pid'='Private service identity changed; refuse stop'
|
||||
'server-exit'='Private sshd process still live; no PID-only kill attempted'
|
||||
'service-absence'='Private service still registered';'child-exit'='Private SSH child processes remain; preserve files and discard ephemeral runner'
|
||||
account='Private account still exists';keys='Injected private key deletion failure'
|
||||
}
|
||||
foreach($failure in $failures.GetEnumerator()){Assert-ProfileCleanup @{Name=$failure.Key;Fault=$failure.Key;Error=$failure.Value;Modes=@('unloaded','unloaded','unloaded');UnloadAtMs=@(0,0,0)}}
|
||||
} $ast
|
||||
# Extract functions through the AST: never provision the fixture while testing diagnostics.
|
||||
'PASS: fixture parse, five numeric-diagnostic cases and the toolchain PATH split'
|
||||
'PASS: fixture parse, diagnostics, PATH split, capability boundaries, profile windows and cleanup failure gates'
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
function Assert-IsolatedWindowsSshCi([ValidateSet('arm64','x64')][string]$Arch) {
|
||||
$os=@{arm64='Arm64';x64='X64'}[$Arch]
|
||||
if($env:GITHUB_ACTIONS -ne 'true' -or $env:ORCA_ISOLATED_SSH_CI -ne '1' -or [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $os){throw "Requires isolated native $Arch GitHub runner"}
|
||||
$admin=([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
|
||||
if(-not $admin){throw 'Administrative private service/account setup required'}
|
||||
}
|
||||
|
||||
function Assert-WindowsSshGlobalServerDormant([ValidateSet('preview','inbox')][string]$Server) {
|
||||
$global=Get-CimInstance Win32_Service -Filter "Name='sshd'"
|
||||
if(-not $global){return}
|
||||
$inboxDir=Join-Path $env:WINDIR 'System32\OpenSSH'
|
||||
# Inbox installation may register the global service; it must never be started here.
|
||||
if($Server -ne 'inbox' -or $global.State -ne 'Stopped' -or $global.PathName.Trim('"') -ne (Join-Path $inboxDir 'sshd.exe')){throw 'Refuse an existing global SSH server'}
|
||||
}
|
||||
|
||||
function Assert-WindowsSshStockShell {
|
||||
$registry=Get-ItemProperty -LiteralPath 'HKLM:\SOFTWARE\OpenSSH' -ErrorAction SilentlyContinue
|
||||
if($registry.DefaultShell -or $registry.DefaultShellCommandOption){throw 'Requires stock cmd.exe OpenSSH shell at start'}
|
||||
}
|
||||
|
||||
function Write-WindowsSshReceiptStage([hashtable]$Report,[string]$Receipt,[string]$Stage) {
|
||||
$timestamp=[DateTime]::UtcNow.ToString('o')
|
||||
$Report.stages+=@{stage=$Stage;utc=$timestamp}
|
||||
try {
|
||||
$bytes=[Text.UTF8Encoding]::new($false).GetBytes(($Report | ConvertTo-Json -Depth 6))
|
||||
$temporary="$Receipt.pending"
|
||||
$stream=[IO.FileStream]::new($temporary,[IO.FileMode]::Create,[IO.FileAccess]::Write,[IO.FileShare]::Read)
|
||||
try {$stream.Write($bytes,0,$bytes.Length);$stream.Flush($true)} finally {$stream.Dispose()}
|
||||
[IO.File]::Move($temporary,$Receipt,$true)
|
||||
$written=$true
|
||||
} catch {$written=$false;Write-Warning 'Progress receipt could not be updated; cleanup must still run'}
|
||||
Write-Host "Native SSH stage: $Stage ($timestamp)"
|
||||
return $written
|
||||
}
|
||||
|
||||
function Install-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[hashtable]$Report,[scriptblock]$Stage) {
|
||||
Assert-IsolatedWindowsSshCi $Arch
|
||||
Assert-WindowsSshGlobalServerDormant 'inbox'
|
||||
Assert-WindowsSshStockShell
|
||||
& $Stage 'inbox-capability-start'
|
||||
$capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0'
|
||||
$Report.inboxCapabilityInitialState=[string]$capability.State
|
||||
if($capability.State -ne 'Installed'){Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' | Out-Null}
|
||||
Assert-WindowsSshGlobalServerDormant 'inbox'
|
||||
Assert-WindowsSshStockShell
|
||||
& $Stage 'inbox-capability-complete'
|
||||
}
|
||||
|
||||
function Initialize-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[string]$Receipt) {
|
||||
$report=@{scope='Windows inbox OpenSSH.Server capability preparation only';arch=$Arch;sourceSha=$env:GITHUB_SHA;runId=$env:GITHUB_RUN_ID;runAttempt=$env:GITHUB_RUN_ATTEMPT;runnerName=$env:RUNNER_NAME;imageVersion=$env:ImageVersion;status='running';stages=@();globalBootstrapCleanup='Disposable CI VM destruction is the boundary; no global sshd is started'}
|
||||
try {
|
||||
if(-not (Write-WindowsSshReceiptStage $report $Receipt 'preparation-start')){throw 'Initial progress receipt unavailable; refuse provisioning'}
|
||||
Install-WindowsInboxSshCapability $Arch $report {param($name)
|
||||
if(-not (Write-WindowsSshReceiptStage $report $Receipt $name)){throw 'Capability preparation progress receipt unavailable'}
|
||||
}
|
||||
$report.status='passed'
|
||||
} catch {$report.status='failed';$report.error=$_.Exception.Message;throw}
|
||||
finally {
|
||||
if(-not (Write-WindowsSshReceiptStage $report $Receipt 'preparation-finished')){throw 'Final capability preparation receipt unavailable'}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,14 @@ const workflow = parse(
|
||||
)
|
||||
const job = workflow.jobs.hosts
|
||||
const runStep = job.steps.find((step) => step.name?.startsWith('Run the Windows host cells'))
|
||||
const provisioning = readFileSync(
|
||||
join(projectDir, 'config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1'),
|
||||
'utf8'
|
||||
)
|
||||
const capability = readFileSync(
|
||||
join(projectDir, 'config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1'),
|
||||
'utf8'
|
||||
)
|
||||
const manifest = (arch) =>
|
||||
JSON.parse(
|
||||
readFileSync(
|
||||
@@ -47,6 +55,99 @@ describe('SSH Windows-host workflow', () => {
|
||||
'x64/windows-2022/preview'
|
||||
])
|
||||
expect(job.env).toMatchObject({ ORCA_BACKGROUND_LAUNCH: '1', ORCA_ISOLATED_SSH_CI: '1' })
|
||||
expect(job.strategy['fail-fast']).toBe(false)
|
||||
expect(job['timeout-minutes']).toBe(75)
|
||||
expect(runStep['timeout-minutes']).toBe(50)
|
||||
})
|
||||
|
||||
it('overlaps only guarded ARM inbox capability preparation with the existing builds', () => {
|
||||
const selfTestIndex = job.steps.findIndex((step) => step.name?.startsWith('Self-test'))
|
||||
const prepareIndex = job.steps.findIndex((step) => step.id === 'inbox-capability')
|
||||
const installIndex = job.steps.findIndex(
|
||||
(step) => step.uses === './.github/actions/install-node-dependencies'
|
||||
)
|
||||
const buildIndex = job.steps.findIndex((step) => step.name?.startsWith('Build this runner'))
|
||||
const prebuildIndex = job.steps.findIndex(
|
||||
(step) => step.uses === './.github/actions/prepare-orcad-prebuilds'
|
||||
)
|
||||
const templateIndex = job.steps.findIndex((step) => step.name?.startsWith('Build the win32'))
|
||||
const waitIndex = job.steps.findIndex((step) => step.wait === 'inbox-capability')
|
||||
const runIndex = job.steps.indexOf(runStep)
|
||||
expect([
|
||||
selfTestIndex,
|
||||
prepareIndex,
|
||||
installIndex,
|
||||
buildIndex,
|
||||
prebuildIndex,
|
||||
templateIndex,
|
||||
waitIndex,
|
||||
runIndex
|
||||
]).toEqual([1, 2, 3, 4, 5, 6, 7, 8])
|
||||
expect(job.steps[prepareIndex]).toMatchObject({
|
||||
background: true,
|
||||
shell: 'pwsh'
|
||||
})
|
||||
expect(job.steps[prepareIndex].if).toBeUndefined()
|
||||
expect(job.steps[waitIndex].if).toBeUndefined()
|
||||
expect(job.steps[prepareIndex].run.trim()).toMatch(
|
||||
/^if\('\$\{\{ matrix\.server }}' -eq 'inbox' -and '\$\{\{ matrix\.arch }}' -eq 'arm64'\)\{[\s\S]+\}$/
|
||||
)
|
||||
expect(job.steps[prepareIndex].run).toContain('Initialize-WindowsInboxSshCapability')
|
||||
expect(job.steps[prepareIndex].run).toContain('inbox-capability-preparation.json')
|
||||
expect(runStep.run).toContain('-InboxPreparationReceipt $preparation')
|
||||
expect(runStep.run).toContain(
|
||||
"$preparation=Join-Path $receipts 'inbox-capability-preparation.json'"
|
||||
)
|
||||
expect(runStep.run).toContain(
|
||||
"if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation="
|
||||
)
|
||||
expect(runStep.background).toBeUndefined()
|
||||
expect(job.steps.at(-1)).toMatchObject({ if: 'always()', uses: 'actions/upload-artifact@v7' })
|
||||
})
|
||||
|
||||
it('shares one capability installer without bypassing native verification or private cleanup', () => {
|
||||
expect(capability.match(/Add-WindowsCapability -Online/g)).toHaveLength(1)
|
||||
expect(provisioning).not.toContain('Add-WindowsCapability')
|
||||
expect(provisioning).toContain(". (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1')")
|
||||
expect(provisioning).toContain('Install-WindowsInboxSshCapability $Arch $report')
|
||||
const install = capability.slice(
|
||||
capability.indexOf('function Install-WindowsInboxSshCapability'),
|
||||
capability.indexOf('function Initialize-WindowsInboxSshCapability')
|
||||
)
|
||||
const mutation = install.indexOf('Add-WindowsCapability')
|
||||
expect(install.indexOf('Assert-IsolatedWindowsSshCi')).toBeLessThan(mutation)
|
||||
expect(install.indexOf('Assert-WindowsSshGlobalServerDormant')).toBeLessThan(mutation)
|
||||
expect(install.lastIndexOf('Assert-WindowsSshGlobalServerDormant')).toBeGreaterThan(mutation)
|
||||
expect(install.indexOf('Assert-WindowsSshStockShell')).toBeLessThan(mutation)
|
||||
expect(install.lastIndexOf('Assert-WindowsSshStockShell')).toBeGreaterThan(mutation)
|
||||
for (const check of [
|
||||
'(Machine $path) -ne $target.machine',
|
||||
'Get-AuthenticodeSignature -LiteralPath $path',
|
||||
'Inbox native input Microsoft signature invalid',
|
||||
"Write-Stage 'host-cell-probe-start'",
|
||||
"Write-Stage 'cleanup-default-shell-start'",
|
||||
"Write-Stage 'cleanup-service-stop-delete-start'"
|
||||
]) {
|
||||
expect(provisioning).toContain(check)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps preparation provenance separate from the oracle current capability state', () => {
|
||||
for (const [field, environment] of [
|
||||
['sourceSha', 'GITHUB_SHA'],
|
||||
['runId', 'GITHUB_RUN_ID'],
|
||||
['runAttempt', 'GITHUB_RUN_ATTEMPT'],
|
||||
['runnerName', 'RUNNER_NAME'],
|
||||
['imageVersion', 'ImageVersion']
|
||||
]) {
|
||||
expect(capability).toContain(`${field}=$env:${environment}`)
|
||||
expect(provisioning).toContain(`$preparation.${field} -ne $env:${environment}`)
|
||||
}
|
||||
expect(provisioning).toContain("$preparation.status -ne 'passed'")
|
||||
expect(provisioning).toContain('$preparation.arch -ne $Arch')
|
||||
expect(provisioning).toContain('$report.inboxCapabilityPreparation=$preparation')
|
||||
expect(capability).toContain('$Report.inboxCapabilityInitialState=[string]$capability.State')
|
||||
expect(capability).toContain("$report.status='failed';$report.error=$_.Exception.Message;throw")
|
||||
})
|
||||
|
||||
it('fetches the preview release its hash manifests pin', () => {
|
||||
|
||||
@@ -1392,3 +1392,94 @@ one CI failure does not establish a failure-rate reduction.
|
||||
The bounded 50-head main sample ending at 8ff6296 contained no root package
|
||||
metadata changes. Removing app-version metadata from the Windows server cache
|
||||
key would not improve reuse in that sample, so the key remains unchanged.
|
||||
|
||||
## Windows ARM SSH: prepare the inbox capability during independent builds
|
||||
|
||||
The ARM inbox lane starts guarded Windows capability preparation after the pure
|
||||
provisioning self-test and waits for it before any private SSH server or host cell
|
||||
runs. Dependency installation and the unchanged native artifacts can run during
|
||||
that preparation. Preview and x64 lanes keep their existing serial provisioning;
|
||||
the registered background step completes without mutation in those lanes.
|
||||
|
||||
The preparation and the foreground provider use the same installer and isolation
|
||||
guards. The receipt must match the source, run, attempt, runner, image and native
|
||||
architecture. The foreground provider still reads the installed capability and
|
||||
verifies every native binary and Microsoft signature. Account ownership, ACLs,
|
||||
DefaultShell, private service identity, host cells and cleanup remain independent
|
||||
checks. A background failure propagates through the unconditional native wait.
|
||||
|
||||
Two full four-lane pairs used frozen source refs and the same dependency and
|
||||
native-install policy. The [first baseline](https://github.com/stablyai/orca/actions/runs/36986929163)
|
||||
ran before the [first candidate](https://github.com/stablyai/orca/actions/runs/36986970976);
|
||||
the [second candidate](https://github.com/stablyai/orca/actions/runs/36991232037)
|
||||
was dispatched before the [second baseline](https://github.com/stablyai/orca/actions/runs/36991234729).
|
||||
Runner image versions matched within each platform in both pairs.
|
||||
|
||||
| Active job, seconds | First baseline | First candidate | Second baseline | Second candidate |
|
||||
| ------------------- | -------------: | --------------: | --------------: | ---------------: |
|
||||
| ARM inbox | 2,403 | 1,644 | 2,353 | 1,667 |
|
||||
| ARM preview | 1,002 | 935 | 886 | 872 |
|
||||
| x64 inbox | 636 | 732 | 616 | 620 |
|
||||
| x64 preview | 562 | 561 | 623 | 566 |
|
||||
|
||||
The ARM inbox observations improved by 759 and 686 seconds. Baseline dependency
|
||||
installation and artifact builds consumed 501 and 498 seconds before capability
|
||||
installation could start. Candidate capability installation ran during that
|
||||
work, but also took about 261 and 232 seconds less than the baseline. Candidate
|
||||
dependency installation was slower, particularly in the second pair. These
|
||||
observations support overlap on ARM; they do not establish a guaranteed 11–13
|
||||
minute saving, a reduction in queue time, or the cause of installer variability.
|
||||
The x64 lane showed no repeatable gain, so it keeps serial preparation.
|
||||
|
||||
All 16 actual Windows providers and 48 host-cell verdicts passed across the two
|
||||
pairs. Receipts verify native machine identity, private service absence, owned
|
||||
process exit, account removal and key removal. Loaded profile disposition remains
|
||||
separate from those required cleanup checks. Hosted execution also verified the
|
||||
native background/wait syntax; older actionlint versions do not recognize it.
|
||||
|
||||
### Overlap the private profile observation budgets
|
||||
|
||||
After service deletion and owned process exit, profile cleanup polls each owned
|
||||
SID with its own full 30-second monotonic budget. Independent budgets now run
|
||||
together. Every deletion follows a fresh targeted read; loaded profiles remain
|
||||
for disposable VM destruction. Service identity, PID ownership, process exit,
|
||||
account removal and key removal still fail the complete provider on error.
|
||||
|
||||
The maintained diagnostics self-test executes the actual cleanup try/catch with
|
||||
scoped Windows API and clock controls. Eight positive cases cover full windows,
|
||||
late unload, reload, query overhead, mixed states and missing SIDs; ten specific
|
||||
failure cases cover foreign profiles and the required cleanup gates. Disposable
|
||||
shortened-deadline and stale-snapshot mutations fail those controls. A separate
|
||||
mocked real-clock observation took 30.179 seconds for three loaded profiles,
|
||||
compared with about 90 seconds for serial full budgets. This measures polling,
|
||||
not an actual Windows provider or the entire job.
|
||||
|
||||
The third profile no longer gains incidental extra time while earlier profiles
|
||||
consume their budgets. A profile unloading at 45 seconds may therefore remain
|
||||
where serial cleanup removed it. This uses the existing disposable-VM fallback;
|
||||
it does not remove a loaded profile or relax mandatory account/key cleanup.
|
||||
Hosted qualification of the combined workflow remains pending.
|
||||
|
||||
## Coordinator mail tests: advance observation windows without removing them
|
||||
|
||||
Six cases advance their original six 1,500 ms and ten 100 ms observation windows
|
||||
with a scoped clock. Real filesystem, SQLite, journal, RPC and runtime work still
|
||||
finishes asynchronously. The original journal-read gate and all counter and
|
||||
operation assertions remain. Cancellation during delayed startup and the
|
||||
Date-only age case retain real timers. Teardown stops the host and closes the
|
||||
database before advancing the known 2,000 ms orphan repair, then asserts no fake
|
||||
timers remain and restores the clock in `finally`.
|
||||
|
||||
Two opposite-order local pairs passed the same 23 cases and unchanged source
|
||||
hashes. Selected-case totals fell from 13.674 to 3.318 seconds and from 13.276 to
|
||||
6.323 seconds. Whole-file test totals fell from 24.845 to 10.829 seconds and from
|
||||
21.500 to 19.410 seconds. Process wall times were 41.488/37.810 seconds and
|
||||
42.140/78.450 seconds; the reverse candidate spent 56.31 seconds importing under
|
||||
unrelated local load. Overall wall-time savings remain inconclusive.
|
||||
|
||||
Injected extra deliveries at 1,499 ms and 99 ms still fail the original assertions
|
||||
in both clock modes. The latter candidate fails the unchanged journal-read gate
|
||||
with the same extra provider start. A separate control confirms the orphan repair
|
||||
actually executes against the closed database and leaves no fake timers. The
|
||||
change retains all 121 original expectation sites and adds one teardown check;
|
||||
it does not shorten the runtime's observation interval or claim a whole-PR gain.
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
ensureStructuredAgentSessionHost,
|
||||
stopStructuredAgentSessionRuntime
|
||||
} from './structured-agent-session-runtime'
|
||||
import { createCoordinatorMailObservationClock } from './structured-chat-coordinator-observation-clock.test-fixture'
|
||||
import {
|
||||
attachParams,
|
||||
fakeCodex,
|
||||
@@ -56,6 +57,7 @@ let db: OrchestrationDb
|
||||
let host: StructuredAgentSessionHost
|
||||
let dispatcher: RpcDispatcher
|
||||
let requests = 0
|
||||
const observationClock = createCoordinatorMailObservationClock(() => host, COORDINATOR)
|
||||
|
||||
function request(
|
||||
method: string,
|
||||
@@ -284,10 +286,15 @@ function startRuntime(): OrcaRuntimeService {
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await stopStructuredAgentSessionRuntime()
|
||||
db.close()
|
||||
vi.restoreAllMocks()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
try {
|
||||
await stopStructuredAgentSessionRuntime()
|
||||
db.close()
|
||||
await observationClock.drainClosedDatabaseRepair()
|
||||
vi.restoreAllMocks()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
} finally {
|
||||
observationClock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
// Pointers are sent on asynchronous edges; the default 1s wait is too tight under a loaded parallel run.
|
||||
@@ -350,15 +357,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
})
|
||||
|
||||
/** Fires both edges and waits until every gate read they started has answered. */
|
||||
async function edgesAnswered(): Promise<void> {
|
||||
const reads = vi.spyOn(host, 'journalSnapshot')
|
||||
runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: null })
|
||||
runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: 'idle' })
|
||||
await vi.waitFor(() => expect(reads).toHaveBeenCalled(), WAIT)
|
||||
await Promise.all(reads.mock.results.map((read) => read.value))
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
reads.mockRestore()
|
||||
}
|
||||
const edgesAnswered = (): Promise<void> => observationClock.edgesAnswered(runtime, WAIT)
|
||||
|
||||
/** The operation ids the coordinator's journal recorded for its pointer turns. */
|
||||
async function pointerSends(): Promise<string[]> {
|
||||
@@ -407,6 +406,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
})
|
||||
|
||||
it('does not restart a provider that dies before every echo, however many edges follow', async () => {
|
||||
observationClock.start()
|
||||
// What this pins: each death's own status edge used to re-point the mail, and that send started
|
||||
// the provider again, about once a second for as long as the mail was unread.
|
||||
await openChat(COORDINATOR)
|
||||
@@ -416,7 +416,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
await finishWorker(taskId)
|
||||
await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(1), WAIT)
|
||||
// A fixed window, not a poll: a respawn loop would restart it several times in it.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500))
|
||||
await observationClock.observe(1_500)
|
||||
await edgesAnswered()
|
||||
expect(codex.connections.length - before).toBe(0)
|
||||
expect(providerFaults.turnStarts).toBe(1)
|
||||
@@ -426,6 +426,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
it.each(['exit-then-throw', 'throw-then-exit'] as const)(
|
||||
'does not restart a provider that crashed while taking the pointer turn (%s)',
|
||||
async (crash) => {
|
||||
observationClock.start()
|
||||
// The crash settles the send `unknown` with the connection's own error, not as a provider
|
||||
// exit; every status edge after it re-pointed the mail and started the provider again.
|
||||
await openChat(COORDINATOR)
|
||||
@@ -434,7 +435,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
const before = providerFaults.starts
|
||||
await finishWorker(taskId)
|
||||
await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(1), WAIT)
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500))
|
||||
await observationClock.observe(1_500)
|
||||
await edgesAnswered()
|
||||
expect(providerFaults.starts - before).toBe(0)
|
||||
expect(providerFaults.turnStarts).toBe(1)
|
||||
@@ -501,7 +502,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
expect(cancelled).toMatchObject({ ok: true })
|
||||
providerFaults.startDelayMs = 0
|
||||
// A fixed window, not a poll: a re-point would start the agent again in it.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500))
|
||||
await observationClock.observe(1_500)
|
||||
expect(providerFaults.starts - before).toBe(1)
|
||||
expect(await pointerSends()).toHaveLength(1)
|
||||
await edgesAnswered()
|
||||
@@ -511,6 +512,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
})
|
||||
|
||||
it('points a held pointer once more after Orca restarts, under a new id', async () => {
|
||||
observationClock.start()
|
||||
await openChat(COORDINATOR)
|
||||
const { runId, taskId } = await coordinatorRunAndTask()
|
||||
providerFaults.dieBeforeEveryEcho = true
|
||||
@@ -525,7 +527,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
const before = providerFaults.starts
|
||||
await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(2), WAIT)
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500))
|
||||
await observationClock.observe(1_500)
|
||||
await edgesAnswered()
|
||||
expect(providerFaults.starts - before).toBe(1)
|
||||
expect(providerFaults.turnStarts).toBe(2)
|
||||
@@ -539,6 +541,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
async function refusedStartsFor(
|
||||
refusal: () => Error
|
||||
): Promise<{ runId: string; starts: number }> {
|
||||
observationClock.start()
|
||||
await openChat(COORDINATOR)
|
||||
const { runId, taskId } = await coordinatorRunAndTask()
|
||||
await host.close(COORDINATOR, 'evict')
|
||||
@@ -551,7 +554,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
WAIT
|
||||
)
|
||||
// A fixed window, not a poll: a retry loop would start it several times in it.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500))
|
||||
await observationClock.observe(1_500)
|
||||
return { runId, starts: providerFaults.starts - before }
|
||||
}
|
||||
|
||||
@@ -576,7 +579,7 @@ describe('a worker result reaches the structured chat that coordinates it', () =
|
||||
const before = providerFaults.starts
|
||||
for (let edge = 0; edge < 5; edge += 1) {
|
||||
runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: 'idle' })
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
await observationClock.observe(100)
|
||||
}
|
||||
await edgesAnswered()
|
||||
expect(providerFaults.starts).toBe(before)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { expect, vi } from 'vitest'
|
||||
import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host'
|
||||
import type { OrcaRuntimeService } from './orca-runtime'
|
||||
import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime'
|
||||
|
||||
export function createCoordinatorMailObservationClock(
|
||||
getHost: () => StructuredAgentSessionHost,
|
||||
sessionId: string
|
||||
) {
|
||||
let active = false
|
||||
return {
|
||||
start(): void {
|
||||
vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] })
|
||||
active = true
|
||||
},
|
||||
async observe(ms: number): Promise<void> {
|
||||
if (!active) {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms))
|
||||
return
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(ms)
|
||||
await waitForStructuredAgentSessionRecovery()
|
||||
await getHost().flushStreamedEvents(sessionId)
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
},
|
||||
async edgesAnswered(
|
||||
runtime: Pick<OrcaRuntimeService, 'onStructuredSessionStatusForMail'>,
|
||||
wait: { timeout: number }
|
||||
): Promise<void> {
|
||||
const reads = vi.spyOn(getHost(), 'journalSnapshot')
|
||||
runtime.onStructuredSessionStatusForMail({ sessionId, status: null })
|
||||
runtime.onStructuredSessionStatusForMail({ sessionId, status: 'idle' })
|
||||
await vi.waitFor(() => expect(reads).toHaveBeenCalled(), wait)
|
||||
await Promise.all(reads.mock.results.map((read) => read.value))
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
reads.mockRestore()
|
||||
},
|
||||
async drainClosedDatabaseRepair(): Promise<void> {
|
||||
if (!active) {
|
||||
return
|
||||
}
|
||||
// The runtime's orphan mailbox repair must still run against the closed database.
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await waitForStructuredAgentSessionRecovery()
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
},
|
||||
restore(): void {
|
||||
if (active) {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
active = false
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user