mirror of
https://github.com/stablyai/orca.git
synced 2026-11-01 01:27:35 +00:00
fix(wsl): re-read a settled exit only from a probe started after the window (#34374)
* fix(wsl): start no running-distro read inside the stop settle window
settledStartedAfter slept with one setTimeout until the settle anchor. Node truncates timer
delays to whole ms against a ms loop clock, so the wake can land up to ~2ms before the anchor
by performance.now(), and the probe then started inside the window and could still see the
stopping distro as running. That is the hook relay stop-settle flake ("expected true to be
false"). Sleep until performance.now() has passed the anchor.
* test(wsl): spy on setTimeout without a type assertion
* test(wsl): fire the spied timer far early so a single sleep always fails
* fix(wsl): re-read a settled exit only from a probe started after the window
After a watch or holder exit, the liveness supervisor slept EXIT_SETTLE_MS and re-read the
running list with no bound, so it could join a list probe that started inside the settle
window and still listed the stopping distro. The re-read now passes the exit time, and the
wiring reads through the gate's settledStartedAfter with exit + EXIT_SETTLE_MS as the bound.
This commit is contained in:
@@ -14,6 +14,7 @@ import {
|
||||
isWslDistroRunningForBackgroundWork,
|
||||
noteWslDistroMayHaveStopped,
|
||||
noteWslDistrosRunning,
|
||||
readSettledConfirmedRunningWslDistros,
|
||||
WSL_STOP_SETTLE_MS
|
||||
} from './wsl-background-distro-gate'
|
||||
import {
|
||||
@@ -82,6 +83,28 @@ describe('the stop settle window', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('readSettledConfirmedRunningWslDistros', () => {
|
||||
it('never answers from a probe that started inside the window, even one still in flight', async () => {
|
||||
let finishStale: (distros: string[]) => void = () => {}
|
||||
const probe = vi
|
||||
.fn<() => Promise<string[]>>()
|
||||
.mockImplementationOnce(() => new Promise((resolve) => (finishStale = resolve)))
|
||||
.mockResolvedValueOnce([])
|
||||
wslMocks.listRunningWslDistrosAsync.mockImplementation((options) =>
|
||||
resolveRunningWslDistros(probe, options)
|
||||
)
|
||||
const stale = isWslDistroRunning('Ubuntu')
|
||||
await vi.waitFor(() => expect(probe).toHaveBeenCalledOnce())
|
||||
|
||||
const settled = readSettledConfirmedRunningWslDistros('Ubuntu', performance.now())
|
||||
finishStale(['Ubuntu'])
|
||||
|
||||
await expect(stale).resolves.toBe(true)
|
||||
await expect(settled).resolves.toEqual({ status: 'confirmed', distros: [] })
|
||||
expect(probe).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveRunningWslDistros startedAfter', () => {
|
||||
it('answers from a confirmed read that started after the anchor, and reads again for a later one', async () => {
|
||||
const probe = vi.fn(async () => ['Ubuntu'])
|
||||
|
||||
@@ -91,6 +91,16 @@ async function settledStartedAfter(
|
||||
return anchor
|
||||
}
|
||||
|
||||
/** A confirmed list from a probe started after `startedAfter` and after `distro`'s stop settled. */
|
||||
export async function readSettledConfirmedRunningWslDistros(
|
||||
distro: string,
|
||||
startedAfter?: number
|
||||
): Promise<WslRunningDistrosRead> {
|
||||
return readConfirmedRunningWslDistros({
|
||||
startedAfter: await settledStartedAfter(distro, startedAfter)
|
||||
})
|
||||
}
|
||||
|
||||
export type WslDistroRunningState = 'running' | 'stopped' | 'unverifiable'
|
||||
|
||||
/** A failed read is 'unverifiable', never a stop; `retryUnverifiable` re-reads once past the failure backoff. */
|
||||
|
||||
@@ -19,7 +19,11 @@ export type WslOrcadLivenessDependencies = {
|
||||
WslOrcadConnectionManager,
|
||||
'ensure' | 'get' | 'publish' | 'close' | 'retireListener'
|
||||
>
|
||||
readRunning: () => Promise<WslRunningDistrosRead>
|
||||
/** `afterExit`: answer only from a probe started once that exit's settle window has passed. */
|
||||
readRunning: (afterExit?: {
|
||||
distro: string
|
||||
exitedAt: number | null
|
||||
}) => Promise<WslRunningDistrosRead>
|
||||
readIdentity: (distro: string, orcadPid: number | null) => Promise<WslGuestIdentity | null>
|
||||
openWatch: (connection: WslOrcadConnection) => Promise<WslOrcadWatch>
|
||||
collectCensus: (connection: WslOrcadConnection) => Promise<OrcadTerminalCensus>
|
||||
|
||||
@@ -169,6 +169,7 @@ function harness(options: { visible?: boolean } = {}) {
|
||||
jumpClock: (ms: number) => {
|
||||
clock += ms
|
||||
},
|
||||
clock: () => clock,
|
||||
lastWatch: () => watches.at(-1)!
|
||||
}
|
||||
}
|
||||
@@ -403,6 +404,17 @@ describe('WslOrcadLivenessSupervisor', () => {
|
||||
expect(h.statuses.at(-1)).toEqual({ state: 'ready', census: null })
|
||||
})
|
||||
|
||||
it('re-reads after the settle only from a probe started once the exit has settled', async () => {
|
||||
const h = harness()
|
||||
await h.supervisor.connect('Ubuntu')
|
||||
h.jumpClock(1_000)
|
||||
const exitedAt = h.clock()
|
||||
h.lastWatch().exit(1)
|
||||
await h.advance(EXIT_SETTLE_MS)
|
||||
// A list probe begun inside the window may still show the stopping distro; it must not answer.
|
||||
expect(h.readRunning).toHaveBeenNthCalledWith(2, { distro: 'Ubuntu', exitedAt })
|
||||
})
|
||||
|
||||
it('waits for a still-listed distro to settle after a failed exit before probing inside it', async () => {
|
||||
const h = harness()
|
||||
await h.supervisor.connect('Ubuntu')
|
||||
|
||||
@@ -227,7 +227,7 @@ export class WslOrcadLivenessSupervisor {
|
||||
if (isWatchOrHolderExit(trigger) && confirmsRunning(running, entry.key)) {
|
||||
// Stopping distros can still appear running; wait before exec accidentally boots them again.
|
||||
await new Promise<void>((resolve) => this.schedule(EXIT_SETTLE_MS, resolve))
|
||||
running = await this.deps.readRunning()
|
||||
running = await this.deps.readRunning({ distro: entry.distro, exitedAt: entry.lastExitMs })
|
||||
if (entry.generation !== generation) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -9,14 +9,15 @@ import { requestOrcadTerminalCensus } from '../ssh/orcad-terminal-census-client'
|
||||
import { subscribeSystemPowerLifecycle } from '../system-power-lifecycle'
|
||||
import {
|
||||
deferUntilWslDistroRunning,
|
||||
readConfirmedRunningWslDistros
|
||||
readConfirmedRunningWslDistros,
|
||||
readSettledConfirmedRunningWslDistros
|
||||
} from '../wsl-background-distro-gate'
|
||||
import type { WslDistroHolder } from './wsl-distro-holder'
|
||||
import { wslDistroHolderKey } from './wsl-distro-holder-record'
|
||||
import { readWslGuestIdentity } from './wsl-guest-process-identity'
|
||||
import type { WslOrcadConnection, WslOrcadConnectionManager } from './wsl-orcad-connection'
|
||||
import { defaultWslOrcadEnvironmentId } from './wsl-orcad-deploy'
|
||||
import { confirmsRunning } from './wsl-orcad-liveness-evidence'
|
||||
import { confirmsRunning, EXIT_SETTLE_MS } from './wsl-orcad-liveness-evidence'
|
||||
import { WslOrcadLivenessSupervisor } from './wsl-orcad-liveness-supervisor'
|
||||
import { createWslOrcadHostTransport } from './wsl-orcad-host-transport'
|
||||
import { openWslOrcadWatch } from './wsl-orcad-watch'
|
||||
@@ -42,7 +43,13 @@ export function createWslOrcadLivenessSupervisor(options: {
|
||||
}): WslOrcadLivenessSupervisor {
|
||||
return new WslOrcadLivenessSupervisor({
|
||||
manager: options.manager,
|
||||
readRunning: readConfirmedRunningWslDistros,
|
||||
readRunning: (afterExit) =>
|
||||
afterExit
|
||||
? readSettledConfirmedRunningWslDistros(
|
||||
afterExit.distro,
|
||||
afterExit.exitedAt === null ? undefined : afterExit.exitedAt + EXIT_SETTLE_MS
|
||||
)
|
||||
: readConfirmedRunningWslDistros(),
|
||||
readIdentity: (distro, orcadPid) => readWslGuestIdentity(transportFor(distro), orcadPid),
|
||||
openWatch: (connection) =>
|
||||
openWslOrcadWatch(
|
||||
|
||||
Reference in New Issue
Block a user