Files
orca/tests/e2e/helpers/wsl-boot-guard.ts
Kelvin Amoaba 46237aa148 fix(e2e): return from every path of the WSL boot guard's running-list read (#36261)
#36256 made one branch return the retry promise while the others fell
through, which trips noImplicitReturns (TS7030) and turns the e2e
typecheck ratchet red on main.
2026-10-30 06:36:12 +00:00

265 lines
8.9 KiB
TypeScript

/**
* Shared by the E9 real-WSL guards: wsl.exe reads that can never start a distro, a running-list
* watcher with a timestamped timeline, and the spawn-log check that names what started it.
*/
import { execFile } from 'node:child_process'
import type { Page } from '@stablyai/playwright-test'
import { expect } from './orca-app'
import { buildWslExecArgs } from '../../../src/shared/wsl-login-shell-command'
import {
PTY_SPAWN,
readDiagnosticLog,
UNC_TOUCH,
type DiagnosticRecord
} from './wsl-boot-diagnostics-log'
import { orcaWslLaunches, shouldRetryTerminate, WSL_EXE } from './wsl-terminate-retry'
/** Async with a timeout: a hung wsl.exe must fail the test, never block the worker past its timeout. */
export function wsl(args: readonly string[], timeoutMs = 60_000): Promise<string> {
return new Promise((resolve, reject) => {
execFile(
'wsl.exe',
[...args],
// UTF-8 instead of wsl.exe's default UTF-16 for its own listings.
{
encoding: 'utf8',
env: { ...process.env, WSL_UTF8: '1' },
timeout: timeoutMs,
windowsHide: true
},
(error, stdout, stderr) =>
error
? reject(new Error(`wsl ${args.join(' ')} failed: ${String(error)} ${stderr}`))
: resolve(stdout)
)
})
}
/** Timestamped steps in the test output, so a lane failure shows when each read and act happened. */
const startedAt = Date.now()
export function timeline(event: string, at = Date.now()): void {
console.log(`[e9-guard +${Math.round(at - startedAt)}ms] ${event}`)
}
/**
* `--list --running` never starts a distro. Exit 1 is "no running distributions"; any other
* failure (timeout, spawn error, other exit) rejects, so it can never read as "stopped".
*/
export function isDistroRunning(distro: string): Promise<boolean> {
return new Promise((resolve, reject) => {
execFile(
'wsl.exe',
['--list', '--running', '--quiet'],
{
encoding: 'utf8',
env: { ...process.env, WSL_UTF8: '1' },
timeout: 5_000,
windowsHide: true
},
(error, stdout) => {
if (error) {
if (error.code === 1 && !error.killed) {
resolve(false)
} else {
reject(error)
}
return
}
resolve(
stdout
.split(/\r?\n/)
.map((line) => line.replaceAll('\0', '').trim().toLowerCase())
.includes(distro.toLowerCase())
)
}
)
})
}
/** How long a just-terminated instance may still be listed: two reads, each up to 5 s on a loaded runner. */
const TERMINATE_GRACE_MS = 10_000
/**
* Polls the running list once a second, never overlapping, from right after `--terminate`.
* `stopped` resolves on the first not-running read, which must come within the grace; only
* running reads before it are forgiven, so a real boot is never waited out. Any running read
* after it is a boot. Three failed reads in a row fail the guard. With `retryTerminate`, a terminate
* that never took is re-issued once, and only while Orca has launched nothing since `spawnsBefore`.
*/
export function watchDistroRunning(
distro: string,
spawnLog: string,
options: { spawnsBefore?: number; retryTerminate?: () => Promise<void> } = {}
): {
stopped: Promise<void>
stop: () => Promise<{ seenRunning: boolean; reads: number }>
} {
let finished = false
let confirmedStopped = false
let seenRunning = false
let reads = 0
let failures = 0
let failure: unknown = null
let timer: ReturnType<typeof setTimeout> | undefined
let current: Promise<void> = Promise.resolve()
let resolveStopped: () => void = () => {}
let rejectStopped: (error: Error) => void = () => {}
const stopped = new Promise<void>((resolve, reject) => {
resolveStopped = resolve
rejectStopped = reject
})
const watchStartedAt = Date.now()
let retried = false
const armGrace = (): ReturnType<typeof setTimeout> =>
setTimeout(
() =>
rejectStopped(
new Error(
`${distro} was still listed ${TERMINATE_GRACE_MS} ms after --terminate${retried ? ' (retried once)' : ''}`
)
),
TERMINATE_GRACE_MS
)
let grace = armGrace()
// Spawns are interleaved into the timeline so a boot can be matched to the launch before it.
let spawnsSeen = readDiagnosticLog(spawnLog).length
const logNewWslSpawns = (): void => {
const spawns = readDiagnosticLog(spawnLog)
for (const spawn of orcaWslLaunches(spawns.slice(spawnsSeen))) {
const kind = spawn.file === UNC_TOUCH ? 'unc' : spawn.file === PTY_SPAWN ? 'pty' : 'spawn'
// `at` is when it happened; the line prints at this poll, up to a second later.
timeline(`${kind} ${spawn.file} ${spawn.args.join(' ')}`, spawn.at)
if (kind !== 'spawn' || startsDistro(spawn, distro)) {
for (const frame of spawn.stack ?? []) {
timeline(` ${frame}`, spawn.at)
}
}
}
spawnsSeen = spawns.length
}
const tick = (): void => {
logNewWslSpawns()
current = isDistroRunning(distro)
.then((running) => {
reads += 1
failures = 0
timeline(`running list: ${distro} ${running ? 'RUNNING' : 'stopped'}`)
if (confirmedStopped) {
seenRunning ||= running
} else if (
running &&
options.retryTerminate &&
shouldRetryTerminate({
retried,
confirmedStopped,
msSinceTerminate: Date.now() - watchStartedAt,
spawnsSinceTerminate: readDiagnosticLog(spawnLog).slice(options.spawnsBefore ?? 0)
})
) {
retried = true
// Loud on purpose: a pattern of these is a runner problem worth chasing.
timeline(`RETRYING --terminate: ${distro} still listed and Orca launched nothing since`)
console.warn(`[e9-guard] WARNING: --terminate of ${distro} did not take; retrying once`)
clearTimeout(grace)
grace = armGrace()
return options.retryTerminate()
} else if (!running) {
confirmedStopped = true
clearTimeout(grace)
resolveStopped()
}
return undefined
})
.catch((error: unknown) => {
failures += 1
timeline(`running-list read failed: ${String(error)}`)
if (failures >= 3) {
failure = error
rejectStopped(new Error(`wsl --list --running kept failing: ${String(error)}`))
}
})
.finally(() => {
if (!finished && failure === null) {
timer = setTimeout(tick, 1_000)
}
})
}
tick()
return {
stopped,
stop: async () => {
finished = true
clearTimeout(timer)
clearTimeout(grace)
await current
logNewWslSpawns()
if (failure !== null) {
throw new Error(`wsl --list --running kept failing: ${String(failure)}`)
}
return { seenRunning, reads }
}
}
}
export function runInDistro(distro: string, script: string): Promise<string> {
return wsl(['-u', 'root', ...buildWslExecArgs(distro, ['sh', '-c', script])])
}
/** wsl.exe verbs that never start a distro; any other launch counts as starting the default one. */
const NON_BOOTING_WSL_VERBS = new Set([
'-l',
'--list',
'--status',
'--version',
'-v',
'-t',
'--terminate',
'--shutdown',
'--help'
])
/** Fails closed: a wsl.exe launch starts `distro` unless it names another one or only lists. */
export function startsDistro(spawn: DiagnosticRecord, distro: string): boolean {
if (!WSL_EXE.test(spawn.file)) {
return false
}
// main's ChildProcess#spawn args carry argv0; process-host's do not.
const raw = WSL_EXE.test(spawn.args[0] ?? '') ? spawn.args.slice(1) : spawn.args
const args = raw.map((arg) => arg.toLowerCase())
const named = args.findIndex((arg) => arg === '-d' || arg === '--distribution')
if (named !== -1) {
return args[named + 1] === distro.toLowerCase()
}
return !NON_BOOTING_WSL_VERBS.has(args[0] ?? '')
}
/** Waits until no wsl.exe launch has been logged for 3 s, so a user act's tail is over. */
export async function waitForWslLaunchesToSettle(spawnLog: string): Promise<void> {
let last = { count: -1, since: 0 }
await expect
.poll(
() => {
const count = readDiagnosticLog(spawnLog).filter((spawn) => WSL_EXE.test(spawn.file)).length
const now = performance.now()
if (count !== last.count) {
last = { count, since: now }
}
return now - last.since >= 3_000
},
{ timeout: 60_000, intervals: [500] }
)
.toBe(true)
}
export async function cycleWindowFocus(page: Page): Promise<void> {
await page.evaluate(() => {
window.dispatchEvent(new Event('blur'))
document.dispatchEvent(new Event('visibilitychange'))
window.dispatchEvent(new Event('focus'))
})
}
export async function activate(page: Page, worktreeId: string | null): Promise<void> {
await page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId)
}