From cc73c8e1a72b0e9ee9c29e57458ce307f5f019c2 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:00:49 -0700 Subject: [PATCH] ci: overlap ARM SSH setup and independent observation waits (#24714) --- .github/workflows/ssh-windows-hosts.yml | 25 ++- .../preview-ssh/prove-preview-openssh.ps1 | 88 +++++----- .../preview-ssh/test-preview-diagnostics.ps1 | 163 +++++++++++++++++- .../preview-ssh/windows-ssh-capability.ps1 | 61 +++++++ .../ssh-windows-hosts-workflow.test.mjs | 101 +++++++++++ docs/reference/ci-runner-efficiency.md | 91 ++++++++++ .../structured-chat-coordinator-mail.test.ts | 41 +++-- ...rdinator-observation-clock.test-fixture.ts | 55 ++++++ 8 files changed, 556 insertions(+), 69 deletions(-) create mode 100644 config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1 create mode 100644 src/main/runtime/structured-chat-coordinator-observation-clock.test-fixture.ts diff --git a/.github/workflows/ssh-windows-hosts.yml b/.github/workflows/ssh-windows-hosts.yml index 652551fc1d9..cd77ee87b67 100644 --- a/.github/workflows/ssh-windows-hosts.yml +++ b/.github/workflows/ssh-windows-hosts.yml @@ -84,10 +84,6 @@ jobs: - uses: actions/checkout@v6 with: persist-credentials: false - - uses: ./.github/actions/install-node-dependencies - id: dependencies - with: - native-runtime: node - name: Self-test the provisioning scripts before touching the machine shell: pwsh run: | @@ -97,6 +93,22 @@ jobs: if($errors.Count){throw "PowerShell parse failed: $($file.FullName)"} } & config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1 + # ARM inbox servicing can finish while the independent Node artifacts are prepared. + - name: Prepare the Windows inbox SSH capability + id: inbox-capability + background: true + shell: pwsh + run: | + if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){ + $receipts=Join-Path $pwd '.build/ssh-windows-host-receipts' + New-Item -ItemType Directory -Force -Path $receipts | Out-Null + . config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1 + Initialize-WindowsInboxSshCapability -Arch '${{ matrix.arch }}' -Receipt (Join-Path $receipts 'inbox-capability-preparation.json') + } + - uses: ./.github/actions/install-node-dependencies + id: dependencies + with: + native-runtime: node # The deploy materializes rung A from this template; only this runner's slot exists here. # The process-tree addon carries the launcher that starts the relay outside sshd's job; the # orcad slot and the relay both stage it, and a standard-user host has no other launch route. @@ -122,6 +134,7 @@ jobs: pnpm build:orcad-prebuilds --require-slots "win32-${{ matrix.arch }}" node config/scripts/build-orcad-template.mjs --targets "win32-${{ matrix.arch }}" pnpm run build:relay + - wait: inbox-capability - name: Run the Windows host cells against a private ${{ matrix.server }} sshd shell: pwsh timeout-minutes: 50 @@ -135,6 +148,8 @@ jobs: $cells=@($env:CELLS -split ',' | ForEach-Object {$_.Trim()} | Where-Object {$_}) if(-not $cells.Count){$cells=@('pinned-cmd','pinned-powershell','legacy-opt-out')} $archive='' + $preparation='' + if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation=Join-Path $receipts 'inbox-capability-preparation.json'} if('${{ matrix.server }}' -eq 'preview'){ $archive=Join-Path $env:RUNNER_TEMP 'preview-${{ matrix.archive }}' # The provisioning script refuses the archive unless its sha256 and every binary's match the pin. @@ -144,7 +159,7 @@ jobs: $callback={param($context) & (Join-Path $tools 'invoke-pinned-relay-cells.ps1') -SourceRoot $sourceRoot -Context $context -Target 'win32-${{ matrix.arch }}' -ReceiptRoot $receipts -Cells $cells }.GetNewClosure() - & (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -Accounts $cells.Count -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log') + & (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -InboxPreparationReceipt $preparation -Accounts $cells.Count -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log') - uses: actions/upload-artifact@v7 if: always() with: diff --git a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 index 115345cf8a0..10b4d9d67e1 100644 --- a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 +++ b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 @@ -2,46 +2,34 @@ # -HiddenTools: the private accounts are denied every machine PATH directory holding one of these # executables, and their own PATH carries logging shims for them, so SSH sessions have no host toolchain. # -HostCellProbe receives a context hashtable (accounts, port, keys, shim log) once provisioning passes. -param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,4)][int]$Accounts=1,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe) +param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,4)][int]$Accounts=1,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe,[string]$InboxPreparationReceipt) $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1') $target=@{arm64=@{os='Arm64';folder='OpenSSH-ARM64';machine='0xAA64';archive='698c6aec31c1dd0fb996206e8741f4531a97355686b5431ef347d531b07fcd42'};x64=@{os='X64';folder='OpenSSH-Win64';machine='0x8664';archive='23f50f3458c4c5d0b12217c6a5ddfde0137210a30fa870e98b29827f7b43aba5'}}[$Arch] $scopeServer=if($Server -eq 'inbox'){'Windows inbox OpenSSH.Server capability binaries'}else{'Microsoft Win32-OpenSSH 10.0.0.0p2-Preview'} $report = @{scope="$scopeServer $Arch private loopback authentication and stock cmd.exe dispatch"; server=$Server; status='running'; imageVersion=$env:ImageVersion; cleanup=@('not-confirmed'); globalBootstrapCleanup='Not qualified: service bootstrap may create ProgramData SSH and OpenSSH registry entries; disposable CI VM destruction is the boundary'; observations=@(); stages=@(); diagnosticCaptureFailures=@()} $script:receiptWritten=$false function Write-Stage([string]$Stage) { - $timestamp=[DateTime]::UtcNow.ToString('o') - $report.stages += @{stage=$Stage; utc=$timestamp} - try { - $bytes=[Text.UTF8Encoding]::new($false).GetBytes(($report | ConvertTo-Json -Depth 6)) - $temporary="$Receipt.pending" - $stream=[IO.FileStream]::new($temporary,[IO.FileMode]::Create,[IO.FileAccess]::Write,[IO.FileShare]::Read) - try {$stream.Write($bytes,0,$bytes.Length);$stream.Flush($true)} finally {$stream.Dispose()} - [IO.File]::Move($temporary,$Receipt,$true) - $script:receiptWritten=$true - } catch {Write-Warning 'Progress receipt could not be updated; cleanup must still run'} - Write-Host "Native SSH stage: $Stage ($timestamp)" + if(Write-WindowsSshReceiptStage $report $Receipt $Stage){$script:receiptWritten=$true} } Write-Stage 'preflight-start' if(-not $script:receiptWritten){throw 'Initial progress receipt unavailable; refuse provisioning'} -if ($env:GITHUB_ACTIONS -ne 'true' -or $env:ORCA_ISOLATED_SSH_CI -ne '1' -or [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $target.os) { throw "Requires isolated native $Arch GitHub runner" } -$admin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) -if (-not $admin) { throw 'Administrative private service/account setup required' } +Assert-IsolatedWindowsSshCi $Arch Write-Stage 'existing-server-query-start' $inboxDir=Join-Path $env:WINDIR 'System32\OpenSSH' -function Assert-GlobalServerDormant { - $global=Get-CimInstance Win32_Service -Filter "Name='sshd'" - if(-not $global){return} - # Inbox mode may register the global service; it must stay stopped and never be started here. - if($Server -ne 'inbox' -or $global.State -ne 'Stopped' -or $global.PathName.Trim('"') -ne (Join-Path $inboxDir 'sshd.exe')){throw 'Refuse an existing global SSH server'} -} -Assert-GlobalServerDormant +Assert-WindowsSshGlobalServerDormant $Server Write-Stage 'existing-server-query-complete' Write-Stage 'default-shell-query-start' $openSshKey = 'HKLM:\SOFTWARE\OpenSSH' -$registry = Get-ItemProperty -LiteralPath $openSshKey -ErrorAction SilentlyContinue # Host-cell probes may set DefaultShell per cell; cleanup restores this stock state. -if ($registry.DefaultShell -or $registry.DefaultShellCommandOption) { throw 'Requires stock cmd.exe OpenSSH shell at start' } +Assert-WindowsSshStockShell Write-Stage 'default-shell-query-complete' +if($InboxPreparationReceipt){ + if($Server -ne 'inbox'){throw 'Inbox preparation receipt cannot qualify a preview server'} + $preparation=Get-Content -LiteralPath $InboxPreparationReceipt -Raw | ConvertFrom-Json + if($preparation.status -ne 'passed' -or $preparation.arch -ne $Arch -or $preparation.sourceSha -ne $env:GITHUB_SHA -or $preparation.runId -ne $env:GITHUB_RUN_ID -or $preparation.runAttempt -ne $env:GITHUB_RUN_ATTEMPT -or $preparation.runnerName -ne $env:RUNNER_NAME -or $preparation.imageVersion -ne $env:ImageVersion){throw 'Inbox preparation receipt identity or verdict mismatch'} + $report.inboxCapabilityPreparation=$preparation +} $id = [Guid]::NewGuid().ToString('N').Substring(0,10) $name = "orca$id" $accountNames = @($name) + @(if($Accounts -gt 1){2..$Accounts | ForEach-Object {"$name$_"}}) @@ -180,12 +168,7 @@ try { $report.nativeInputs=$verified Write-Stage 'preview-native-input-verification-complete' } else { - Write-Stage 'inbox-capability-start' - $capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' - $report.inboxCapabilityInitialState=[string]$capability.State - if($capability.State -ne 'Installed'){Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' | Out-Null} - Assert-GlobalServerDormant - Write-Stage 'inbox-capability-complete' + Install-WindowsInboxSshCapability $Arch $report {param($stage) Write-Stage $stage} $verified=@() foreach($binary in @('sshd.exe','ssh.exe','ssh-keygen.exe','sftp.exe','sftp-server.exe')){ $path=Join-Path $sshDir $binary @@ -431,21 +414,38 @@ LogLevel DEBUG1 foreach($directory in $deniedToolDirs){Invoke-Bounded icacls.exe (@($directory.TrimEnd('\'),'/remove:d')+@($ownedAccounts | ForEach-Object {"*$($_.sid)"})) 120 | Out-Null} Write-Stage 'cleanup-toolchain-acl-complete' Write-Stage 'cleanup-user-profile-start' + $profileTargets=@(foreach($account in $ownedAccounts){ + if($account.sid){@{sid=$account.sid;watch=$null;done=$false;profiles=@();waitMs=0;disposition=$null}} + }) $report.profileCleanup=@() - foreach($account in $ownedAccounts){ - if(-not $account.sid){continue} - $profileWait=[Diagnostics.Stopwatch]::StartNew() - do { - $profiles=@(Get-CimInstance Win32_UserProfile | Where-Object SID -eq $account.sid) - if(-not @($profiles | Where-Object Loaded).Count){break} - Start-Sleep -Milliseconds 500 - } while($profileWait.Elapsed.TotalSeconds -lt 30) - $report.profileUnloadWaitMs=$profileWait.ElapsedMilliseconds - $report.privateProfile=@($profiles | ForEach-Object {@{loaded=$_.Loaded;status=$_.Status}}) - Write-Stage 'cleanup-user-profile-observed' - $loadedProfiles=@($profiles | Where-Object Loaded) - $report.profileCleanup+=if($loadedProfiles.Count){'Loaded profile retained for disposable CI VM destruction'}else{'Unloaded profile removed'} - $profiles | Where-Object {-not $_.Loaded} | Remove-CimInstance + $report.privateProfiles=@() + do { + foreach($entry in @($profileTargets | Where-Object {-not $_.done})){ + if(-not $entry.watch){$entry.watch=[Diagnostics.Stopwatch]::StartNew()} + $profiles=@(Get-CimInstance Win32_UserProfile -Filter "SID='$($entry.sid)'") + if(@($profiles | Where-Object SID -ne $entry.sid).Count){throw 'Private profile query returned an unrelated SID'} + if(@($profiles | Where-Object Loaded).Count -and $entry.watch.ElapsedMilliseconds -lt 30000){continue} + # A profile may reload after the polling snapshot. + $profiles=@(Get-CimInstance Win32_UserProfile -Filter "SID='$($entry.sid)'") + if(@($profiles | Where-Object SID -ne $entry.sid).Count){throw 'Private profile query returned an unrelated SID'} + $loadedProfiles=@($profiles | Where-Object Loaded) + if($loadedProfiles.Count -and $entry.watch.ElapsedMilliseconds -lt 30000){continue} + $profiles | Where-Object {-not $_.Loaded} | Remove-CimInstance + $entry.profiles=@($profiles | ForEach-Object {@{loaded=$_.Loaded;status=$_.Status}}) + $entry.waitMs=$entry.watch.ElapsedMilliseconds + $entry.disposition=if($loadedProfiles.Count){'Loaded profile retained for disposable CI VM destruction'}else{'Unloaded profile removed'} + $entry.done=$true + $report.profileCleanup=@($profileTargets | Where-Object done | ForEach-Object disposition) + $report.privateProfiles=@($profileTargets | Where-Object done | ForEach-Object {@{sid=$_.sid;waitMs=$_.waitMs;profiles=$_.profiles;disposition=$_.disposition}}) + $report.profileUnloadWaitMs=$entry.waitMs + $report.privateProfile=$entry.profiles + Write-Stage 'cleanup-user-profile-observed' + } + if(@($profileTargets | Where-Object {-not $_.done}).Count){Start-Sleep -Milliseconds 500} + } while(@($profileTargets | Where-Object {-not $_.done}).Count) + if($profileTargets.Count){ + $report.profileUnloadWaitMs=$profileTargets[-1].waitMs + $report.privateProfile=$profileTargets[-1].profiles } Write-Stage 'cleanup-user-profile-complete' Write-Stage 'cleanup-user-start' diff --git a/config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1 b/config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1 index 0304ac7a275..5bd3ccf30cd 100644 --- a/config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1 +++ b/config/ci/windows-ssh-provider/preview-ssh/test-preview-diagnostics.ps1 @@ -33,5 +33,166 @@ try { if(($result.hidden -join '|') -ne "$nodeDir|$gccDir"){throw 'Toolchain PATH entries not hidden'} if(($result.kept -join '|') -ne "$plainDir|$(Join-Path $pathRoot 'missing')|Q:\no-such-drive"){throw 'Plain PATH entries not kept in order'} } finally {Remove-Item -LiteralPath $pathRoot -Recurse -Force -ErrorAction SilentlyContinue} +# Mock only the machine boundary; exercise the shared installer without servicing this host. +. (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1') +function Assert-IsolatedWindowsSshCi([string]$Arch){if($script:refuseCapabilityHost){throw 'Injected host refusal'}} +function Assert-WindowsSshGlobalServerDormant([string]$Server){$script:globalChecks++;if($script:globalChecks -eq $script:refuseGlobalCheck){throw 'Injected global server refusal'}} +function Assert-WindowsSshStockShell {$script:shellChecks++;if($script:shellChecks -eq $script:refuseShellCheck){throw 'Injected shell refusal'}} +function Get-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityQueries++;return @{State=$script:capabilityState}} +function Add-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityAdds++;if($script:failCapabilityInstall){throw 'Injected capability install failure'}} +function Reset-CapabilityControl([string]$State){ + $script:capabilityState=$State;$script:capabilityQueries=0;$script:capabilityAdds=0 + $script:globalChecks=0;$script:shellChecks=0;$script:refuseGlobalCheck=0;$script:refuseShellCheck=0 + $script:refuseCapabilityHost=$false;$script:failCapabilityInstall=$false + $script:capabilityStages=[Collections.Generic.List[string]]::new() +} +foreach($state in @('Installed','NotPresent')){ + Reset-CapabilityControl $state + $capabilityReport=@{} + Install-WindowsInboxSshCapability 'x64' $capabilityReport {param($name) $script:capabilityStages.Add($name)} + $expectedAdds=if($state -eq 'Installed'){0}else{1} + if($capabilityReport.inboxCapabilityInitialState -ne $state -or $script:capabilityAdds -ne $expectedAdds -or $script:globalChecks -ne 2 -or $script:shellChecks -ne 2 -or ($script:capabilityStages -join ',') -ne 'inbox-capability-start,inbox-capability-complete'){throw 'Shared capability preparation did not preserve the install and guard boundaries'} +} +foreach($fault in @('host','global-before','shell-before','global-after','shell-after','install')){ + Reset-CapabilityControl 'NotPresent' + switch($fault){ + 'host' {$script:refuseCapabilityHost=$true} + 'global-before' {$script:refuseGlobalCheck=1} + 'shell-before' {$script:refuseShellCheck=1} + 'global-after' {$script:refuseGlobalCheck=2} + 'shell-after' {$script:refuseShellCheck=2} + 'install' {$script:failCapabilityInstall=$true} + } + $rejected=$false + try {Install-WindowsInboxSshCapability 'x64' @{} {param($name) $script:capabilityStages.Add($name)}} catch {$rejected=$true} + if(-not $rejected -or $script:capabilityStages.Contains('inbox-capability-complete')){throw "Capability fault did not fail closed: $fault"} + if($fault -in @('host','global-before','shell-before') -and $script:capabilityAdds){throw 'Capability mutation preceded its host guards'} +} +$capabilityRoot=Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString('N')) +try { + New-Item -ItemType Directory -Path $capabilityRoot | Out-Null + $capabilityReceipt=Join-Path $capabilityRoot 'capability.json' + Reset-CapabilityControl 'Installed' + Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt + $completed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json + if($completed.status -ne 'passed' -or $completed.inboxCapabilityInitialState -ne 'Installed'){throw 'Capability success receipt missing its observed initial state'} + Reset-CapabilityControl 'NotPresent';$script:failCapabilityInstall=$true + $rejected=$false + try {Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt} catch {$rejected=$true} + $failed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json + if(-not $rejected -or $failed.status -ne 'failed' -or $failed.inboxCapabilityInitialState -ne 'NotPresent' -or $failed.error -ne 'Injected capability install failure'){throw 'Failed capability preparation masqueraded as a passing receipt'} +} finally {Remove-Item -LiteralPath $capabilityRoot -Recurse -Force -ErrorAction SilentlyContinue} +& { + param($ProofAst) + $cleanup=@($ProofAst.FindAll({param($node) $node -is [Management.Automation.Language.TryStatementAst] -and $node.Body.Extent.Text.Contains("Write-Stage 'cleanup-start'")},$true)) + if($cleanup.Count -ne 1 -or -not $cleanup[0].Extent.Text.Contains('[Diagnostics.Stopwatch]::StartNew()')){throw 'Cleanup clock boundary missing'} + # Run the real cleanup gates with virtual clocks; no Windows machine operation may escape these mocks. + $cleanupBlock=[scriptblock]::Create($cleanup[0].Extent.Text.Replace('[Diagnostics.Stopwatch]::StartNew()','(New-ProfileControlStopwatch)').Replace('[DateTime]::UtcNow','(Get-ProfileControlUtcNow)')) + $ownedSids=@('S-1-5-21-100-200-300-1001','S-1-5-21-100-200-300-1002','S-1-5-21-100-200-300-1003') + $foreignSid='S-1-5-21-100-200-300-9000';$control=@{} + function New-ProfileControlStopwatch { + $watch=[pscustomobject]@{StartedMilliseconds=$control.clockMs;Control=$control} + $watch | Add-Member ScriptProperty ElapsedMilliseconds {$this.Control.clockMs-$this.StartedMilliseconds} + return $watch + } + function Get-ProfileControlUtcNow {[DateTime]::new(2026,10,2,0,0,0,[DateTimeKind]::Utc).AddMilliseconds($control.clockMs)} + function Start-Sleep([int]$Milliseconds,[int]$Seconds){$control.clockMs+=$Milliseconds+1000*$Seconds} + function Write-Stage([string]$Stage){$control.stages.Add($Stage)} + function Record-PrivateServiceDiagnostics([switch]$AfterStop){} + function Test-Path([string]$LiteralPath){if($LiteralPath -ne 'HKLM:\SOFTWARE\OpenSSH'){throw 'Unexpected filesystem query'};return $false} + function Get-ItemProperty([string]$LiteralPath,[object]$ErrorAction){if($LiteralPath -ne 'HKLM:\SOFTWARE\OpenSSH'){throw 'Unexpected registry query'};return $null} + function Remove-ItemProperty {throw 'Unexpected registry mutation'} + function Stop-Service([string]$Name,[switch]$Force,[object]$ErrorAction){if($Name -ne 'orca-sshd-control'){throw 'Foreign service stop'};if($control.case.Fault -ne 'server-exit'){$control.serverLive=$false}} + function Invoke-Bounded([string]$Program,[string[]]$Arguments){if($Program -ne 'sc.exe' -or ($Arguments -join '|') -ne 'delete|orca-sshd-control'){throw 'Unexpected native command'};if($control.case.Fault -ne 'service-absence'){$control.servicePresent=$false}} + function Get-Process([int]$Id,[object]$ErrorAction){if($Id -ne 100){throw 'Foreign process query'};if($control.serverLive){@{Id=100}}} + function Get-Service([string]$Name,[object]$ErrorAction){if($Name -ne 'orca-sshd-control'){throw 'Foreign service query'};if($control.servicePresent){@{Name=$Name}}} + function Get-ProfileControlLoaded([string]$Sid){ + $state=$control.profiles[$Sid] + if($state.Mode -eq 'late'){return $control.clockMs -lt $state.UnloadAtMs} + if($state.Mode -in @('reload','reload-at-deadline')){return $state.CurrentLoaded} + return $state.Mode -eq 'loaded' + } + function Get-CimInstance([Parameter(Position=0)][string]$ClassName,[string]$Filter){ + if($ClassName -eq 'Win32_Service'){ + if($Filter -ne "Name='orca-sshd-control'"){throw 'Foreign service query'} + if($control.servicePresent){@{PathName=$(if($control.case.Fault -eq 'service-identity'){'C:\foreign\sshd.exe'}else{'C:\fake\ossh-control\sshd.exe'});ProcessId=$(if($control.case.Fault -eq 'service-pid'){200}else{100})}};return + } + if($ClassName -eq 'Win32_Process'){ + if($control.case.Fault -eq 'child-exit'){@{ExecutablePath='C:\fake\OpenSSH\session.exe';ProcessId=101;ParentProcessId=100;CreationDate=(Get-ProfileControlUtcNow)}};return + } + if($ClassName -ne 'Win32_UserProfile' -or $Filter -notmatch "^SID='(S-1-5-21-100-200-300-\d+)'$" -or $Matches[1] -notin $ownedSids){throw 'Profile query must target an owned SID'} + $sid=$Matches[1];$control.clockMs+=$control.case.QueryCostMs;$control.profileQueries[$sid]++ + if($control.case.Fault -eq 'query' -and $sid -eq $ownedSids[1]){throw 'Injected profile query failure'} + if($control.case.Fault -eq 'foreign-result' -and $sid -eq $ownedSids[1]){[pscustomobject]@{SID=$foreignSid;Loaded=$false;Status=0};return} + $state=$control.profiles[$sid] + if($state.Mode -eq 'absent' -or $state.Deleted){return} + $loaded=Get-ProfileControlLoaded $sid + if($state.Mode -eq 'reload' -and $control.profileQueries[$sid] -eq 1){$loaded=$false;$state.CurrentLoaded=$true} + if($state.Mode -eq 'reload-at-deadline' -and $control.clockMs -ge 30000 -and -not $state.Reinjected){$loaded=$false;$state.CurrentLoaded=$true;$state.Reinjected=$true} + [pscustomobject]@{SID=$sid;Loaded=$loaded;Status=0} + } + function Remove-CimInstance { + param([Parameter(ValueFromPipeline=$true)][object]$InputObject) + process { + if($InputObject.SID -notin $ownedSids -or $InputObject.Loaded -or (Get-ProfileControlLoaded $InputObject.SID)){throw 'Unsafe profile deletion attempted'} + if($control.case.Fault -eq 'delete'){throw 'Injected profile deletion failure'} + $control.profiles[$InputObject.SID].Deleted=$true;$control.removed.Add($InputObject.SID) + } + } + function Get-LocalUser([string]$Name,[object]$ErrorAction){if(-not $control.users.ContainsKey($Name)){throw 'Foreign account query'};if($control.users[$Name]){@{Name=$Name}}} + function Remove-LocalUser([string]$Name){if(-not $control.users.ContainsKey($Name)){throw 'Foreign account deletion'};if($control.case.Fault -ne 'account'){$control.users[$Name]=$false}} + function Remove-Item([string]$LiteralPath,[switch]$Recurse,[switch]$Force){if($LiteralPath -ne 'C:\fake\ossh-control'){throw 'Foreign filesystem deletion'};if($control.case.Fault -eq 'keys'){throw 'Injected private key deletion failure'};$control.keysRemoved=$true} + function Assert-ProfileCleanup([hashtable]$Case){ + $control.Clear();$control.case=$Case;$control.clockMs=0;$control.serverLive=$true;$control.servicePresent=$true;$control.keysRemoved=$false + $control.stages=[Collections.Generic.List[string]]::new();$control.removed=[Collections.Generic.List[string]]::new() + $control.profiles=@{};$control.users=@{};$control.profileQueries=@{} + $ownedAccounts=@(for($index=0;$index -lt $ownedSids.Count;$index++){ + $sid=$ownedSids[$index];$control.profileQueries[$sid]=0 + $control.profiles[$sid]=@{Mode=$Case.Modes[$index];UnloadAtMs=$Case.UnloadAtMs[$index];CurrentLoaded=$true;Deleted=$false} + $name="orca-control-$index";$control.users[$name]=$true;@{name=$name;sid=$(if($Case.MissingSid -and $index -eq 2){$null}else{$sid})} + }) + $report=@{status='proof-passed-cleanup-pending'};$openSshKey='HKLM:\SOFTWARE\OpenSSH';$serviceName='orca-sshd-control' + $root='C:\fake\ossh-control';$createdService=$true;$ownedServerPid=100;$sshDir='C:\fake\OpenSSH';$preexisting=@();$deniedToolDirs=@() + & $cleanupBlock + if($Case.Fault){ + if($report.status -ne 'failed' -or $report.cleanupError -ne $Case.Error -or $control.keysRemoved -or $control.stages.Contains('cleanup-private-files-complete')){throw "Cleanup fault did not fail at its gate: $($Case.Name)"} + if($Case.Fault -notin @('account','keys') -and @($control.users.Values | Where-Object {-not $_}).Count){throw "Failure bypassed account cleanup gate: $($Case.Name)"} + return + } + if($report.status -ne 'passed' -or @($control.users.Values | Where-Object {$_}).Count -or -not $control.keysRemoved){throw "Cleanup omitted account/key gates: $($Case.Name)"} + if(($control.removed.ToArray() | Sort-Object) -join ',' -ne (($Case.Removed | Sort-Object) -join ',')){throw "Wrong removed SIDs: $($Case.Name)"} + if(@($report.profileCleanup | Where-Object {$_ -eq 'Loaded profile retained for disposable CI VM destruction'}).Count -ne $Case.Retained){throw "Wrong retained disposition: $($Case.Name)"} + foreach($sid in $Case.FullWindow){ + $observed=@($report.privateProfiles | Where-Object sid -eq $sid) + if($observed.Count -ne 1 -or $observed[0].waitMs -lt 30000){throw "Short profile observation window: $($Case.Name)"} + } + if($control.clockMs -gt $Case.MaxClockMs){throw "Profile windows were serialized: $($Case.Name)"} + if($Case.MissingSid -and $control.profileQueries[$ownedSids[2]]){throw 'Missing SID gained profile deletion authority'} + if($report.privateProfiles.Count -and ($report.profileUnloadWaitMs -ne $report.privateProfiles[-1].waitMs -or ($report.privateProfile | ConvertTo-Json -Compress) -ne ($report.privateProfiles[-1].profiles | ConvertTo-Json -Compress))){throw 'Legacy scalar observation lost owned-account order'} + } + $cases=@( + @{Name='three loaded full windows';Modes=@('loaded','loaded','loaded');Retained=3;Removed=@();FullWindow=$ownedSids}, + @{Name='unload at 29999ms';Modes=@('late','unloaded','absent');UnloadAtMs=@(29999,0,0);Retained=0;Removed=$ownedSids[0..1];FullWindow=@($ownedSids[0])}, + @{Name='reload before deletion';Modes=@('reload','unloaded','absent');Retained=1;Removed=@($ownedSids[1]);FullWindow=@($ownedSids[0])}, + @{Name='independent unloads';Modes=@('late','loaded','late');UnloadAtMs=@(5000,0,20000);Retained=1;Removed=@($ownedSids[0],$ownedSids[2]);FullWindow=@($ownedSids[1])}, + @{Name='slow provider queries';Modes=@('loaded','loaded','loaded');QueryCostMs=200;Retained=3;Removed=@();FullWindow=$ownedSids;MaxClockMs=40000}, + @{Name='reload at expired window';Modes=@('reload-at-deadline','unloaded','absent');Retained=1;Removed=@($ownedSids[1]);FullWindow=@($ownedSids[0])}, + @{Name='late unload honestly retained';Modes=@('loaded','loaded','late');UnloadAtMs=@(0,0,45000);Retained=3;Removed=@();FullWindow=$ownedSids}, + @{Name='missing SID is skipped';Modes=@('unloaded','absent','unloaded');Retained=0;Removed=@($ownedSids[0]);MissingSid=$true} + ) + foreach($case in $cases){ + if(-not $case.UnloadAtMs){$case.UnloadAtMs=@(0,0,0)} + if(-not $case.MaxClockMs){$case.MaxClockMs=33000} + Assert-ProfileCleanup $case + } + $failures=@{ + query='Injected profile query failure';delete='Injected profile deletion failure';'foreign-result'='Private profile query returned an unrelated SID' + 'service-identity'='Private service identity changed; refuse stop';'service-pid'='Private service identity changed; refuse stop' + 'server-exit'='Private sshd process still live; no PID-only kill attempted' + 'service-absence'='Private service still registered';'child-exit'='Private SSH child processes remain; preserve files and discard ephemeral runner' + account='Private account still exists';keys='Injected private key deletion failure' + } + foreach($failure in $failures.GetEnumerator()){Assert-ProfileCleanup @{Name=$failure.Key;Fault=$failure.Key;Error=$failure.Value;Modes=@('unloaded','unloaded','unloaded');UnloadAtMs=@(0,0,0)}} +} $ast # Extract functions through the AST: never provision the fixture while testing diagnostics. -'PASS: fixture parse, five numeric-diagnostic cases and the toolchain PATH split' +'PASS: fixture parse, diagnostics, PATH split, capability boundaries, profile windows and cleanup failure gates' diff --git a/config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1 b/config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1 new file mode 100644 index 00000000000..d2bdc9096a3 --- /dev/null +++ b/config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1 @@ -0,0 +1,61 @@ +function Assert-IsolatedWindowsSshCi([ValidateSet('arm64','x64')][string]$Arch) { + $os=@{arm64='Arm64';x64='X64'}[$Arch] + if($env:GITHUB_ACTIONS -ne 'true' -or $env:ORCA_ISOLATED_SSH_CI -ne '1' -or [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $os){throw "Requires isolated native $Arch GitHub runner"} + $admin=([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) + if(-not $admin){throw 'Administrative private service/account setup required'} +} + +function Assert-WindowsSshGlobalServerDormant([ValidateSet('preview','inbox')][string]$Server) { + $global=Get-CimInstance Win32_Service -Filter "Name='sshd'" + if(-not $global){return} + $inboxDir=Join-Path $env:WINDIR 'System32\OpenSSH' + # Inbox installation may register the global service; it must never be started here. + if($Server -ne 'inbox' -or $global.State -ne 'Stopped' -or $global.PathName.Trim('"') -ne (Join-Path $inboxDir 'sshd.exe')){throw 'Refuse an existing global SSH server'} +} + +function Assert-WindowsSshStockShell { + $registry=Get-ItemProperty -LiteralPath 'HKLM:\SOFTWARE\OpenSSH' -ErrorAction SilentlyContinue + if($registry.DefaultShell -or $registry.DefaultShellCommandOption){throw 'Requires stock cmd.exe OpenSSH shell at start'} +} + +function Write-WindowsSshReceiptStage([hashtable]$Report,[string]$Receipt,[string]$Stage) { + $timestamp=[DateTime]::UtcNow.ToString('o') + $Report.stages+=@{stage=$Stage;utc=$timestamp} + try { + $bytes=[Text.UTF8Encoding]::new($false).GetBytes(($Report | ConvertTo-Json -Depth 6)) + $temporary="$Receipt.pending" + $stream=[IO.FileStream]::new($temporary,[IO.FileMode]::Create,[IO.FileAccess]::Write,[IO.FileShare]::Read) + try {$stream.Write($bytes,0,$bytes.Length);$stream.Flush($true)} finally {$stream.Dispose()} + [IO.File]::Move($temporary,$Receipt,$true) + $written=$true + } catch {$written=$false;Write-Warning 'Progress receipt could not be updated; cleanup must still run'} + Write-Host "Native SSH stage: $Stage ($timestamp)" + return $written +} + +function Install-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[hashtable]$Report,[scriptblock]$Stage) { + Assert-IsolatedWindowsSshCi $Arch + Assert-WindowsSshGlobalServerDormant 'inbox' + Assert-WindowsSshStockShell + & $Stage 'inbox-capability-start' + $capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' + $Report.inboxCapabilityInitialState=[string]$capability.State + if($capability.State -ne 'Installed'){Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' | Out-Null} + Assert-WindowsSshGlobalServerDormant 'inbox' + Assert-WindowsSshStockShell + & $Stage 'inbox-capability-complete' +} + +function Initialize-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[string]$Receipt) { + $report=@{scope='Windows inbox OpenSSH.Server capability preparation only';arch=$Arch;sourceSha=$env:GITHUB_SHA;runId=$env:GITHUB_RUN_ID;runAttempt=$env:GITHUB_RUN_ATTEMPT;runnerName=$env:RUNNER_NAME;imageVersion=$env:ImageVersion;status='running';stages=@();globalBootstrapCleanup='Disposable CI VM destruction is the boundary; no global sshd is started'} + try { + if(-not (Write-WindowsSshReceiptStage $report $Receipt 'preparation-start')){throw 'Initial progress receipt unavailable; refuse provisioning'} + Install-WindowsInboxSshCapability $Arch $report {param($name) + if(-not (Write-WindowsSshReceiptStage $report $Receipt $name)){throw 'Capability preparation progress receipt unavailable'} + } + $report.status='passed' + } catch {$report.status='failed';$report.error=$_.Exception.Message;throw} + finally { + if(-not (Write-WindowsSshReceiptStage $report $Receipt 'preparation-finished')){throw 'Final capability preparation receipt unavailable'} + } +} diff --git a/config/scripts/ssh-windows-hosts-workflow.test.mjs b/config/scripts/ssh-windows-hosts-workflow.test.mjs index c137b268c0d..3daef7cd4e2 100644 --- a/config/scripts/ssh-windows-hosts-workflow.test.mjs +++ b/config/scripts/ssh-windows-hosts-workflow.test.mjs @@ -13,6 +13,14 @@ const workflow = parse( ) const job = workflow.jobs.hosts const runStep = job.steps.find((step) => step.name?.startsWith('Run the Windows host cells')) +const provisioning = readFileSync( + join(projectDir, 'config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1'), + 'utf8' +) +const capability = readFileSync( + join(projectDir, 'config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1'), + 'utf8' +) const manifest = (arch) => JSON.parse( readFileSync( @@ -47,6 +55,99 @@ describe('SSH Windows-host workflow', () => { 'x64/windows-2022/preview' ]) expect(job.env).toMatchObject({ ORCA_BACKGROUND_LAUNCH: '1', ORCA_ISOLATED_SSH_CI: '1' }) + expect(job.strategy['fail-fast']).toBe(false) + expect(job['timeout-minutes']).toBe(75) + expect(runStep['timeout-minutes']).toBe(50) + }) + + it('overlaps only guarded ARM inbox capability preparation with the existing builds', () => { + const selfTestIndex = job.steps.findIndex((step) => step.name?.startsWith('Self-test')) + const prepareIndex = job.steps.findIndex((step) => step.id === 'inbox-capability') + const installIndex = job.steps.findIndex( + (step) => step.uses === './.github/actions/install-node-dependencies' + ) + const buildIndex = job.steps.findIndex((step) => step.name?.startsWith('Build this runner')) + const prebuildIndex = job.steps.findIndex( + (step) => step.uses === './.github/actions/prepare-orcad-prebuilds' + ) + const templateIndex = job.steps.findIndex((step) => step.name?.startsWith('Build the win32')) + const waitIndex = job.steps.findIndex((step) => step.wait === 'inbox-capability') + const runIndex = job.steps.indexOf(runStep) + expect([ + selfTestIndex, + prepareIndex, + installIndex, + buildIndex, + prebuildIndex, + templateIndex, + waitIndex, + runIndex + ]).toEqual([1, 2, 3, 4, 5, 6, 7, 8]) + expect(job.steps[prepareIndex]).toMatchObject({ + background: true, + shell: 'pwsh' + }) + expect(job.steps[prepareIndex].if).toBeUndefined() + expect(job.steps[waitIndex].if).toBeUndefined() + expect(job.steps[prepareIndex].run.trim()).toMatch( + /^if\('\$\{\{ matrix\.server }}' -eq 'inbox' -and '\$\{\{ matrix\.arch }}' -eq 'arm64'\)\{[\s\S]+\}$/ + ) + expect(job.steps[prepareIndex].run).toContain('Initialize-WindowsInboxSshCapability') + expect(job.steps[prepareIndex].run).toContain('inbox-capability-preparation.json') + expect(runStep.run).toContain('-InboxPreparationReceipt $preparation') + expect(runStep.run).toContain( + "$preparation=Join-Path $receipts 'inbox-capability-preparation.json'" + ) + expect(runStep.run).toContain( + "if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation=" + ) + expect(runStep.background).toBeUndefined() + expect(job.steps.at(-1)).toMatchObject({ if: 'always()', uses: 'actions/upload-artifact@v7' }) + }) + + it('shares one capability installer without bypassing native verification or private cleanup', () => { + expect(capability.match(/Add-WindowsCapability -Online/g)).toHaveLength(1) + expect(provisioning).not.toContain('Add-WindowsCapability') + expect(provisioning).toContain(". (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1')") + expect(provisioning).toContain('Install-WindowsInboxSshCapability $Arch $report') + const install = capability.slice( + capability.indexOf('function Install-WindowsInboxSshCapability'), + capability.indexOf('function Initialize-WindowsInboxSshCapability') + ) + const mutation = install.indexOf('Add-WindowsCapability') + expect(install.indexOf('Assert-IsolatedWindowsSshCi')).toBeLessThan(mutation) + expect(install.indexOf('Assert-WindowsSshGlobalServerDormant')).toBeLessThan(mutation) + expect(install.lastIndexOf('Assert-WindowsSshGlobalServerDormant')).toBeGreaterThan(mutation) + expect(install.indexOf('Assert-WindowsSshStockShell')).toBeLessThan(mutation) + expect(install.lastIndexOf('Assert-WindowsSshStockShell')).toBeGreaterThan(mutation) + for (const check of [ + '(Machine $path) -ne $target.machine', + 'Get-AuthenticodeSignature -LiteralPath $path', + 'Inbox native input Microsoft signature invalid', + "Write-Stage 'host-cell-probe-start'", + "Write-Stage 'cleanup-default-shell-start'", + "Write-Stage 'cleanup-service-stop-delete-start'" + ]) { + expect(provisioning).toContain(check) + } + }) + + it('keeps preparation provenance separate from the oracle current capability state', () => { + for (const [field, environment] of [ + ['sourceSha', 'GITHUB_SHA'], + ['runId', 'GITHUB_RUN_ID'], + ['runAttempt', 'GITHUB_RUN_ATTEMPT'], + ['runnerName', 'RUNNER_NAME'], + ['imageVersion', 'ImageVersion'] + ]) { + expect(capability).toContain(`${field}=$env:${environment}`) + expect(provisioning).toContain(`$preparation.${field} -ne $env:${environment}`) + } + expect(provisioning).toContain("$preparation.status -ne 'passed'") + expect(provisioning).toContain('$preparation.arch -ne $Arch') + expect(provisioning).toContain('$report.inboxCapabilityPreparation=$preparation') + expect(capability).toContain('$Report.inboxCapabilityInitialState=[string]$capability.State') + expect(capability).toContain("$report.status='failed';$report.error=$_.Exception.Message;throw") }) it('fetches the preview release its hash manifests pin', () => { diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md index 6c7f0d059cf..f2c8e649b35 100644 --- a/docs/reference/ci-runner-efficiency.md +++ b/docs/reference/ci-runner-efficiency.md @@ -1392,3 +1392,94 @@ one CI failure does not establish a failure-rate reduction. The bounded 50-head main sample ending at 8ff6296 contained no root package metadata changes. Removing app-version metadata from the Windows server cache key would not improve reuse in that sample, so the key remains unchanged. + +## Windows ARM SSH: prepare the inbox capability during independent builds + +The ARM inbox lane starts guarded Windows capability preparation after the pure +provisioning self-test and waits for it before any private SSH server or host cell +runs. Dependency installation and the unchanged native artifacts can run during +that preparation. Preview and x64 lanes keep their existing serial provisioning; +the registered background step completes without mutation in those lanes. + +The preparation and the foreground provider use the same installer and isolation +guards. The receipt must match the source, run, attempt, runner, image and native +architecture. The foreground provider still reads the installed capability and +verifies every native binary and Microsoft signature. Account ownership, ACLs, +DefaultShell, private service identity, host cells and cleanup remain independent +checks. A background failure propagates through the unconditional native wait. + +Two full four-lane pairs used frozen source refs and the same dependency and +native-install policy. The [first baseline](https://github.com/stablyai/orca/actions/runs/36986929163) +ran before the [first candidate](https://github.com/stablyai/orca/actions/runs/36986970976); +the [second candidate](https://github.com/stablyai/orca/actions/runs/36991232037) +was dispatched before the [second baseline](https://github.com/stablyai/orca/actions/runs/36991234729). +Runner image versions matched within each platform in both pairs. + +| Active job, seconds | First baseline | First candidate | Second baseline | Second candidate | +| ------------------- | -------------: | --------------: | --------------: | ---------------: | +| ARM inbox | 2,403 | 1,644 | 2,353 | 1,667 | +| ARM preview | 1,002 | 935 | 886 | 872 | +| x64 inbox | 636 | 732 | 616 | 620 | +| x64 preview | 562 | 561 | 623 | 566 | + +The ARM inbox observations improved by 759 and 686 seconds. Baseline dependency +installation and artifact builds consumed 501 and 498 seconds before capability +installation could start. Candidate capability installation ran during that +work, but also took about 261 and 232 seconds less than the baseline. Candidate +dependency installation was slower, particularly in the second pair. These +observations support overlap on ARM; they do not establish a guaranteed 11–13 +minute saving, a reduction in queue time, or the cause of installer variability. +The x64 lane showed no repeatable gain, so it keeps serial preparation. + +All 16 actual Windows providers and 48 host-cell verdicts passed across the two +pairs. Receipts verify native machine identity, private service absence, owned +process exit, account removal and key removal. Loaded profile disposition remains +separate from those required cleanup checks. Hosted execution also verified the +native background/wait syntax; older actionlint versions do not recognize it. + +### Overlap the private profile observation budgets + +After service deletion and owned process exit, profile cleanup polls each owned +SID with its own full 30-second monotonic budget. Independent budgets now run +together. Every deletion follows a fresh targeted read; loaded profiles remain +for disposable VM destruction. Service identity, PID ownership, process exit, +account removal and key removal still fail the complete provider on error. + +The maintained diagnostics self-test executes the actual cleanup try/catch with +scoped Windows API and clock controls. Eight positive cases cover full windows, +late unload, reload, query overhead, mixed states and missing SIDs; ten specific +failure cases cover foreign profiles and the required cleanup gates. Disposable +shortened-deadline and stale-snapshot mutations fail those controls. A separate +mocked real-clock observation took 30.179 seconds for three loaded profiles, +compared with about 90 seconds for serial full budgets. This measures polling, +not an actual Windows provider or the entire job. + +The third profile no longer gains incidental extra time while earlier profiles +consume their budgets. A profile unloading at 45 seconds may therefore remain +where serial cleanup removed it. This uses the existing disposable-VM fallback; +it does not remove a loaded profile or relax mandatory account/key cleanup. +Hosted qualification of the combined workflow remains pending. + +## Coordinator mail tests: advance observation windows without removing them + +Six cases advance their original six 1,500 ms and ten 100 ms observation windows +with a scoped clock. Real filesystem, SQLite, journal, RPC and runtime work still +finishes asynchronously. The original journal-read gate and all counter and +operation assertions remain. Cancellation during delayed startup and the +Date-only age case retain real timers. Teardown stops the host and closes the +database before advancing the known 2,000 ms orphan repair, then asserts no fake +timers remain and restores the clock in `finally`. + +Two opposite-order local pairs passed the same 23 cases and unchanged source +hashes. Selected-case totals fell from 13.674 to 3.318 seconds and from 13.276 to +6.323 seconds. Whole-file test totals fell from 24.845 to 10.829 seconds and from +21.500 to 19.410 seconds. Process wall times were 41.488/37.810 seconds and +42.140/78.450 seconds; the reverse candidate spent 56.31 seconds importing under +unrelated local load. Overall wall-time savings remain inconclusive. + +Injected extra deliveries at 1,499 ms and 99 ms still fail the original assertions +in both clock modes. The latter candidate fails the unchanged journal-read gate +with the same extra provider start. A separate control confirms the orphan repair +actually executes against the closed database and leaves no fake timers. The +change retains all 121 original expectation sites and adds one teardown check; +it does not shorten the runtime's observation interval or claim a whole-PR gain. diff --git a/src/main/runtime/structured-chat-coordinator-mail.test.ts b/src/main/runtime/structured-chat-coordinator-mail.test.ts index c9c3b8448ad..49c694b5adb 100644 --- a/src/main/runtime/structured-chat-coordinator-mail.test.ts +++ b/src/main/runtime/structured-chat-coordinator-mail.test.ts @@ -34,6 +34,7 @@ import { ensureStructuredAgentSessionHost, stopStructuredAgentSessionRuntime } from './structured-agent-session-runtime' +import { createCoordinatorMailObservationClock } from './structured-chat-coordinator-observation-clock.test-fixture' import { attachParams, fakeCodex, @@ -56,6 +57,7 @@ let db: OrchestrationDb let host: StructuredAgentSessionHost let dispatcher: RpcDispatcher let requests = 0 +const observationClock = createCoordinatorMailObservationClock(() => host, COORDINATOR) function request( method: string, @@ -284,10 +286,15 @@ function startRuntime(): OrcaRuntimeService { } afterEach(async () => { - await stopStructuredAgentSessionRuntime() - db.close() - vi.restoreAllMocks() - await rm(root, { recursive: true, force: true }) + try { + await stopStructuredAgentSessionRuntime() + db.close() + await observationClock.drainClosedDatabaseRepair() + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) + } finally { + observationClock.restore() + } }) // Pointers are sent on asynchronous edges; the default 1s wait is too tight under a loaded parallel run. @@ -350,15 +357,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = }) /** Fires both edges and waits until every gate read they started has answered. */ - async function edgesAnswered(): Promise { - const reads = vi.spyOn(host, 'journalSnapshot') - runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: null }) - runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: 'idle' }) - await vi.waitFor(() => expect(reads).toHaveBeenCalled(), WAIT) - await Promise.all(reads.mock.results.map((read) => read.value)) - await new Promise((resolve) => setImmediate(resolve)) - reads.mockRestore() - } + const edgesAnswered = (): Promise => observationClock.edgesAnswered(runtime, WAIT) /** The operation ids the coordinator's journal recorded for its pointer turns. */ async function pointerSends(): Promise { @@ -407,6 +406,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = }) it('does not restart a provider that dies before every echo, however many edges follow', async () => { + observationClock.start() // What this pins: each death's own status edge used to re-point the mail, and that send started // the provider again, about once a second for as long as the mail was unread. await openChat(COORDINATOR) @@ -416,7 +416,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = await finishWorker(taskId) await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(1), WAIT) // A fixed window, not a poll: a respawn loop would restart it several times in it. - await new Promise((resolve) => setTimeout(resolve, 1_500)) + await observationClock.observe(1_500) await edgesAnswered() expect(codex.connections.length - before).toBe(0) expect(providerFaults.turnStarts).toBe(1) @@ -426,6 +426,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = it.each(['exit-then-throw', 'throw-then-exit'] as const)( 'does not restart a provider that crashed while taking the pointer turn (%s)', async (crash) => { + observationClock.start() // The crash settles the send `unknown` with the connection's own error, not as a provider // exit; every status edge after it re-pointed the mail and started the provider again. await openChat(COORDINATOR) @@ -434,7 +435,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = const before = providerFaults.starts await finishWorker(taskId) await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(1), WAIT) - await new Promise((resolve) => setTimeout(resolve, 1_500)) + await observationClock.observe(1_500) await edgesAnswered() expect(providerFaults.starts - before).toBe(0) expect(providerFaults.turnStarts).toBe(1) @@ -501,7 +502,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = expect(cancelled).toMatchObject({ ok: true }) providerFaults.startDelayMs = 0 // A fixed window, not a poll: a re-point would start the agent again in it. - await new Promise((resolve) => setTimeout(resolve, 1_500)) + await observationClock.observe(1_500) expect(providerFaults.starts - before).toBe(1) expect(await pointerSends()).toHaveLength(1) await edgesAnswered() @@ -511,6 +512,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = }) it('points a held pointer once more after Orca restarts, under a new id', async () => { + observationClock.start() await openChat(COORDINATOR) const { runId, taskId } = await coordinatorRunAndTask() providerFaults.dieBeforeEveryEcho = true @@ -525,7 +527,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) const before = providerFaults.starts await vi.waitFor(() => expect(providerFaults.turnStarts).toBe(2), WAIT) - await new Promise((resolve) => setTimeout(resolve, 1_500)) + await observationClock.observe(1_500) await edgesAnswered() expect(providerFaults.starts - before).toBe(1) expect(providerFaults.turnStarts).toBe(2) @@ -539,6 +541,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = async function refusedStartsFor( refusal: () => Error ): Promise<{ runId: string; starts: number }> { + observationClock.start() await openChat(COORDINATOR) const { runId, taskId } = await coordinatorRunAndTask() await host.close(COORDINATOR, 'evict') @@ -551,7 +554,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = WAIT ) // A fixed window, not a poll: a retry loop would start it several times in it. - await new Promise((resolve) => setTimeout(resolve, 1_500)) + await observationClock.observe(1_500) return { runId, starts: providerFaults.starts - before } } @@ -576,7 +579,7 @@ describe('a worker result reaches the structured chat that coordinates it', () = const before = providerFaults.starts for (let edge = 0; edge < 5; edge += 1) { runtime.onStructuredSessionStatusForMail({ sessionId: COORDINATOR, status: 'idle' }) - await new Promise((resolve) => setTimeout(resolve, 100)) + await observationClock.observe(100) } await edgesAnswered() expect(providerFaults.starts).toBe(before) diff --git a/src/main/runtime/structured-chat-coordinator-observation-clock.test-fixture.ts b/src/main/runtime/structured-chat-coordinator-observation-clock.test-fixture.ts new file mode 100644 index 00000000000..cced136ce8b --- /dev/null +++ b/src/main/runtime/structured-chat-coordinator-observation-clock.test-fixture.ts @@ -0,0 +1,55 @@ +import { expect, vi } from 'vitest' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import type { OrcaRuntimeService } from './orca-runtime' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' + +export function createCoordinatorMailObservationClock( + getHost: () => StructuredAgentSessionHost, + sessionId: string +) { + let active = false + return { + start(): void { + vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] }) + active = true + }, + async observe(ms: number): Promise { + if (!active) { + await new Promise((resolve) => setTimeout(resolve, ms)) + return + } + await vi.advanceTimersByTimeAsync(ms) + await waitForStructuredAgentSessionRecovery() + await getHost().flushStreamedEvents(sessionId) + await new Promise((resolve) => setImmediate(resolve)) + }, + async edgesAnswered( + runtime: Pick, + wait: { timeout: number } + ): Promise { + const reads = vi.spyOn(getHost(), 'journalSnapshot') + runtime.onStructuredSessionStatusForMail({ sessionId, status: null }) + runtime.onStructuredSessionStatusForMail({ sessionId, status: 'idle' }) + await vi.waitFor(() => expect(reads).toHaveBeenCalled(), wait) + await Promise.all(reads.mock.results.map((read) => read.value)) + await new Promise((resolve) => setImmediate(resolve)) + reads.mockRestore() + }, + async drainClosedDatabaseRepair(): Promise { + if (!active) { + return + } + // The runtime's orphan mailbox repair must still run against the closed database. + await vi.advanceTimersByTimeAsync(2_000) + await waitForStructuredAgentSessionRecovery() + await new Promise((resolve) => setImmediate(resolve)) + expect(vi.getTimerCount()).toBe(0) + }, + restore(): void { + if (active) { + vi.useRealTimers() + } + active = false + } + } +}