diff --git a/.github/workflows/wsl-orcad-e2e.yml b/.github/workflows/wsl-orcad-e2e.yml index b7785dc9d1e..678cbdc0830 100644 --- a/.github/workflows/wsl-orcad-e2e.yml +++ b/.github/workflows/wsl-orcad-e2e.yml @@ -34,6 +34,7 @@ on: - '!**/*.test.ts' - '!**/*.test.tsx' - 'src/main/wsl/wsl-orcad-real-distro-lane.test.ts' + - 'src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts' - '.github/workflows/wsl-orcad-e2e.yml' - 'config/ci/wsl-lifecycle/**' - 'config/ci/windows-lifecycle-receipt.ps1' @@ -340,6 +341,16 @@ jobs: run: | node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'survive.log') if ($global:LASTEXITCODE -ne 0) { throw "survive phase failed: $global:LASTEXITCODE" } + # R40: with no terminal left, orcad idle-exits and the holder lets the distro stop on its own. + - name: 'Phase 3: idle orcad releases the distro' + timeout-minutes: 15 + env: + ORCA_RUN_WSL_ORCAD_LANE: '1' + ORCA_WSL_ORCAD_LANE_PHASE: idle-release + ORCA_ORCAD_TEMPLATE_PATH: ${{ runner.temp }}/orcad-template + run: | + node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'idle-release.log') + if ($global:LASTEXITCODE -ne 0) { throw "idle-release phase failed: $global:LASTEXITCODE" } - name: Real desktop agent configuration is unchanged if: always() run: | @@ -379,7 +390,7 @@ jobs: run: | $dir = $env:ORCA_WSL_ORCAD_LANE_RECEIPTS $lines = @("### WSL orcad lane (${{ matrix.family }}) — $env:ImageOS $env:ImageVersion", '') - foreach ($phase in 'launch', 'survive') { + foreach ($phase in 'launch', 'survive', 'idle-release') { $file = Join-Path $dir "$phase.json" if (Test-Path $file) { $lines += "#### $phase", '```json', (Get-Content $file -Raw), '```' } } diff --git a/config/scripts/orcad-e2e-change-scope.test.mjs b/config/scripts/orcad-e2e-change-scope.test.mjs index f786d0bbe27..769e366b741 100644 --- a/config/scripts/orcad-e2e-change-scope.test.mjs +++ b/config/scripts/orcad-e2e-change-scope.test.mjs @@ -161,10 +161,19 @@ it('re-includes the real WSL distro root after excluding unit tests', () => { expect(paths.indexOf('src/main/wsl/wsl-orcad-real-distro-lane.test.ts')).toBeGreaterThan(exclude) const steps = workflows.wsl.jobs.wsl2.steps const phases = steps.filter((step) => step.env?.ORCA_RUN_WSL_ORCAD_LANE === '1') - expect(phases.map((step) => step.env.ORCA_WSL_ORCAD_LANE_PHASE)).toEqual(['launch', 'survive']) + expect(phases.map((step) => step.env.ORCA_WSL_ORCAD_LANE_PHASE)).toEqual([ + 'launch', + 'survive', + 'idle-release' + ]) for (const phase of phases) { - expect(phase.run).toContain('src/main/wsl/wsl-orcad-real-distro-lane.test.ts') + expect(phase.run).toMatch( + /src\/main\/wsl\/wsl-orcad-real-distro-(idle-release-)?lane\.test\.ts/ + ) } + expect( + paths.indexOf('src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts') + ).toBeGreaterThan(exclude) }) // Each remote host lane executes these sources, so a change to one must start the lane. diff --git a/config/scripts/wsl-orcad-workflow-qualification.test.mjs b/config/scripts/wsl-orcad-workflow-qualification.test.mjs index 96344fad4f4..54d8ef16efa 100644 --- a/config/scripts/wsl-orcad-workflow-qualification.test.mjs +++ b/config/scripts/wsl-orcad-workflow-qualification.test.mjs @@ -49,5 +49,9 @@ it('retains both systemd modes on Ubuntu and full scheduled/manual references', const phases = workflow.jobs.wsl2.steps.filter( (step) => step.env?.ORCA_RUN_WSL_ORCAD_LANE === '1' ) - expect(phases.map((step) => step.env.ORCA_WSL_ORCAD_LANE_PHASE)).toEqual(['launch', 'survive']) + expect(phases.map((step) => step.env.ORCA_WSL_ORCAD_LANE_PHASE)).toEqual([ + 'launch', + 'survive', + 'idle-release' + ]) }) diff --git a/src/main/orcad/orcad-managed-idle-exit-host.ts b/src/main/orcad/orcad-managed-idle-exit-host.ts index 81ed34e7e09..5f456099ac4 100644 --- a/src/main/orcad/orcad-managed-idle-exit-host.ts +++ b/src/main/orcad/orcad-managed-idle-exit-host.ts @@ -17,6 +17,11 @@ import { writeOrcadIdleStopRecord } from './orcad-idle-stop-record' import type { OrcadShutdownTrigger } from './orcad-lifecycle' +import { + idleStopMayReleaseWslHolder, + requestOrcadWslHolderRelease, + withdrawOrcadWslHolderRelease +} from './orcad-wsl-holder-release' import type { OrcadIdleStopRecord } from '../../shared/orcad-idle-exit' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' @@ -42,6 +47,7 @@ export function beginOrcadIdleExit(userDataPath: string): { previousIdleStop: OrcadIdleStopRecord | null | undefined start: (ports: OrcadIdleExitRuntimePorts) => Promise } { + withdrawOrcadWslHolderRelease(userDataPath) const config = resolveOrcadManagedIdleExit(process.env) if (!config) { return { previousIdleStop: undefined, start: async () => {} } @@ -133,6 +139,13 @@ async function startOrcadManagedIdleExit( idleRecord.requestShutdown(requestIdleShutdown) return true } + if (idleStopMayReleaseWslHolder(process.env, outcome)) { + try { + requestOrcadWslHolderRelease(input.userDataPath) + } catch (error) { + console.error('[orcad] could not release the WSL distro holder:', error) + } + } idleRecord.requestShutdown(requestIdleShutdown) return true } @@ -149,7 +162,11 @@ export function createIdleStopRecordOwnership(userDataPath: string): { onShutdown(): void } { let idleOwnsStop = false - const discard = (): void => discardOrcadIdleStopRecord(userDataPath) + // The holder release rides on the idle stop: a stop that is not idle must not end the distro. + const discard = (): void => { + discardOrcadIdleStopRecord(userDataPath) + withdrawOrcadWslHolderRelease(userDataPath) + } return { requestShutdown: (request) => { // Cleanup may run before the trigger returns, so establish ownership first. diff --git a/src/main/orcad/orcad-wsl-holder-release.test.ts b/src/main/orcad/orcad-wsl-holder-release.test.ts new file mode 100644 index 00000000000..20efbccf50d --- /dev/null +++ b/src/main/orcad/orcad-wsl-holder-release.test.ts @@ -0,0 +1,146 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { setAppEnvironment } from '../../shared/app-environment' +import { ORCAD_MANAGED_ACTIVATION_ROOT_ENV } from '../../shared/orcad-managed-launch-env' +import { createAdapter } from '../daemon/daemon-pty-router-test-fixture' +import { DaemonPtyRouter } from '../daemon/daemon-pty-router' +import { installDaemonProvider } from '../daemon/daemon-provider-state' +import type { DaemonIdleRetirementResult } from '../daemon/daemon-pty-runtime-state' +import { PROTOCOL_VERSION } from '../daemon/types' +import { + beginOrcadIdleExit, + bindOrcadIdleShutdown, + createIdleStopRecordOwnership +} from './orcad-managed-idle-exit-host' +import { installOrcadManagedIdleExit } from './orcad-managed-idle-exit' +import type * as ManagedIdleExit from './orcad-managed-idle-exit' +import { orcadWslHolderReleasePath } from './orcad-wsl-holder-release' +import { ORCAD_WSL_HOST_ENV } from '../ipc/wsl-guest-windows-drive-path' + +vi.mock('../native-chat/agent-session-wire/structured-agent-session-registry', () => ({ + getStructuredAgentSessionHost: () => ({ hasLoadedProviders: () => false }) +})) +vi.mock('./orcad-managed-idle-exit', async (importOriginal) => ({ + ...(await importOriginal()), + installOrcadManagedIdleExit: vi.fn(() => () => {}) +})) + +let root: string + +beforeEach(() => { + vi.clearAllMocks() + root = mkdtempSync(join(tmpdir(), 'orcad-wsl-holder-release-')) + vi.stubEnv(ORCAD_MANAGED_ACTIVATION_ROOT_ENV, join(root, 'fence')) + vi.stubEnv(ORCAD_WSL_HOST_ENV, '1') + vi.spyOn(console, 'error').mockImplementation(() => {}) + setAppEnvironment({ + getPath: () => root, + getAppPath: () => root, + getVersion: () => '0.0.0-test', + isPackaged: () => false, + onWillQuit: () => {}, + exit: () => {}, + getAppMetrics: () => [] + }) +}) + +afterEach(() => { + vi.unstubAllEnvs() + vi.restoreAllMocks() + rmSync(root, { recursive: true, force: true }) +}) + +async function idleStop( + retirement: DaemonIdleRetirementResult, + options: { clientReturns?: boolean; shutdownAccepted?: boolean } = {} +) { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + vi.mocked(current.requestIdleRetirement).mockResolvedValue(retirement) + installDaemonProvider(null, new DaemonPtyRouter({ current, legacy: [] })) + const activity = { openConnections: 0, requestsInFlight: 0, lastRequestAt: 0 } + const shutdown = vi.fn(() => options.shutdownAccepted ?? true) + bindOrcadIdleShutdown(shutdown) + await beginOrcadIdleExit(root).start({ + rpc: { readClientActivity: () => activity }, + agentStates: () => [], + hasStagedMigration: () => false, + automationsBusy: () => false, + emulatorsBusy: async () => false, + registerCleanup: () => {} + }) + const installed = vi.mocked(installOrcadManagedIdleExit).mock.calls.at(-1)?.[0] + if (!installed) { + throw new Error('idle monitor was not installed') + } + if (options.clientReturns) { + activity.openConnections = 1 + } + const stopped = await installed.stop({ quietSince: 0, stoppedAt: 900_000, timeoutMs: 900_000 }) + return { stopped, shutdown } +} + +it('asks the WSL holder to let go once an idle stop retired the terminal daemon', async () => { + const { stopped } = await idleStop({ state: 'retiring' }) + + expect(stopped).toBe(true) + expect(readFileSync(orcadWslHolderReleasePath(root), 'utf8')).toBe(`${process.pid}\n`) +}) + +it.each([ + ['live terminals keep the daemon', { state: 'busy', liveSessions: 2 } as const], + ['the daemon could not be proven idle', { state: 'unverifiable' } as const] +])('keeps the distro held when %s', async (_name, retirement) => { + await idleStop(retirement) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('keeps the distro held when a client returned during the stop', async () => { + await idleStop({ state: 'retiring' }, { clientReturns: true }) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('never writes the request for an orcad no WSL host launched', async () => { + vi.stubEnv(ORCAD_WSL_HOST_ENV, '') + await idleStop({ state: 'retiring' }) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('withdraws the request when another stop took the shutdown over', async () => { + await idleStop({ state: 'retiring' }, { shutdownAccepted: false }) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('withdraws the request when a stop it asked for ends some other way', () => { + writeFileSync(orcadWslHolderReleasePath(root), '123\n') + createIdleStopRecordOwnership(root).onShutdown() + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('withdraws an earlier request when a new orcad starts', () => { + writeFileSync(orcadWslHolderReleasePath(root), '123\n') + beginOrcadIdleExit(root) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(false) +}) + +it('never touches the request path for an orcad no WSL host launched', () => { + vi.stubEnv(ORCAD_WSL_HOST_ENV, '') + writeFileSync(orcadWslHolderReleasePath(root), '123\n') + beginOrcadIdleExit(root) + + expect(existsSync(orcadWslHolderReleasePath(root))).toBe(true) +}) + +it('never throws at start when the request cannot be removed', () => { + // A directory in its place fails the removal like a locked or unwritable file would. + mkdirSync(join(orcadWslHolderReleasePath(root), 'stuck'), { recursive: true }) + + expect(() => beginOrcadIdleExit(root)).not.toThrow() +}) diff --git a/src/main/orcad/orcad-wsl-holder-release.ts b/src/main/orcad/orcad-wsl-holder-release.ts new file mode 100644 index 00000000000..ba7fbc4371d --- /dev/null +++ b/src/main/orcad/orcad-wsl-holder-release.ts @@ -0,0 +1,47 @@ +/** + * Lets a WSL distro stop once its orcad idle-stops with nothing left to keep (R40). The desktop's + * holder keeps the distro up; it reads this request and ends after orcad's pid exits. + */ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { ORCAD_WSL_HOLDER_RELEASE_FILENAME } from '../../shared/orcad-idle-exit' +import { writeSecureFile } from '../../shared/secure-file' +import { ORCAD_WSL_HOST_ENV } from '../ipc/wsl-guest-windows-drive-path' +import type { OrcadDaemonRetirement } from './orcad-daemon-retirement' + +export function orcadWslHolderReleasePath(userDataPath: string): string { + return join(userDataPath, ORCAD_WSL_HOLDER_RELEASE_FILENAME) +} + +/** + * Only a WSL-host launch has a holder, and only a daemon that proved itself empty and left shows + * no terminal would die with the distro. Why the launch marker: a setsid'd orcad may lack WSL_DISTRO_NAME. + */ +export function idleStopMayReleaseWslHolder( + env: NodeJS.ProcessEnv, + daemon: OrcadDaemonRetirement +): boolean { + return env[ORCAD_WSL_HOST_ENV] === '1' && daemon.retirement === 'retired' +} + +export function requestOrcadWslHolderRelease(userDataPath: string, pid = process.pid): void { + writeSecureFile(orcadWslHolderReleasePath(userDataPath), `${pid}\n`) +} + +/** + * A starting orcad may be given work, so any earlier request no longer stands. WSL-host launches + * only; never throws, so a locked or unwritable data root cannot fail a start or a stop. + */ +export function withdrawOrcadWslHolderRelease( + userDataPath: string, + env: NodeJS.ProcessEnv = process.env +): void { + if (env[ORCAD_WSL_HOST_ENV] !== '1') { + return + } + try { + rmSync(orcadWslHolderReleasePath(userDataPath), { force: true }) + } catch (error) { + console.error('[orcad] could not withdraw the WSL holder release request:', error) + } +} diff --git a/src/main/wsl/wsl-distro-holder-command.ts b/src/main/wsl/wsl-distro-holder-command.ts index 3c0cdcaee80..a3a603ac09a 100644 --- a/src/main/wsl/wsl-distro-holder-command.ts +++ b/src/main/wsl/wsl-distro-holder-command.ts @@ -1,14 +1,65 @@ +import { ORCAD_WSL_HOLDER_RELEASE_FILENAME } from '../../shared/orcad-idle-exit' import { buildWslExecArgs } from '../../shared/wsl-login-shell-command' /** `$0` of the holder's shell; the token that identifies our holder in a process command line. */ export const WSL_DISTRO_HOLDER_MARKER = 'orca-wsl-distro-holder' -// Use bounded sleeps: BusyBox rejects infinity and huge durations vary across implementations. -const HOLDER_SCRIPT = 'while :; do sleep 3600; done' +/** How often the holder looks for orcad's release request; also the floor on how soon it ends. */ +export const WSL_DISTRO_HOLDER_POLL_SECONDS = 15 + +/** + * The user's own processes that never make a distro "in use": session plumbing a login starts. + * Kept tight on purpose; anything else the user runs (tmux, nohup jobs, servers) keeps the hold, + * and so do long-lived per-user agents (gpg-agent, keyboxd, ssh-agent, an editor's server). + */ +export const WSL_DISTRO_HOLDER_IGNORED_PROCESS_NAMES = ['systemd', '(sd-pam)', 'dbus-daemon'] + +// Why short sleeps: busybox `sleep` (Alpine) rejects `infinity`. Why `rm` without -f: it claims the +// request atomically, so an orcad that starts and clears it first keeps the holder (and the distro). +// `~/.orca` is the WSL host's data root (orcad-remote-context); without HOME the path never exists. +export function buildWslDistroHolderScript( + pollSeconds = WSL_DISTRO_HOLDER_POLL_SECONDS, + procRoot = '/proc' +): string { + const ignored = WSL_DISTRO_HOLDER_IGNORED_PROCESS_NAMES.map((name) => `'${name}'`).join('|') + return [ + `f="$HOME/.orca/${ORCAD_WSL_HOLDER_RELEASE_FILENAME}"; me=$(id -u)`, + // A new holder means a client wants the distro: an older orcad's request no longer stands. + 'rm -f "$f"', + // Fails toward holding: any live process of this user outside the holder itself counts, so a + // distro whose default user is root never releases. One exception: with /proc mounted + // hidepid, another user's container shim is invisible and does not hold. + 'user_busy() {', + `for d in ${procRoot}/[0-9]*; do`, + 'n= s= pp= u=', + 'while read -r k v r; do case $k in Name:) n=$v ;; State:) s=$v ;; PPid:) pp=$v ;; Uid:) u=$v; break ;; esac', + 'done 2>/dev/null < "$d/status" || continue', + // A running container is work whoever owns it (docker's shims run as root). + 'case $n in containerd-shim*) return 0 ;; esac', + '[ "$u" = "$me" ] && [ "${d##*/}" != "$$" ] && [ "$pp" != "$$" ] || continue', + 'case $s in Z*) continue ;; esac', + `case $n in ${ignored}) continue ;; esac`, + 'return 0', + 'done; return 1; }', + `while :; do sleep ${pollSeconds}`, + '[ -f "$f" ] || continue', + 'p=; read -r p < "$f"', + "case $p in ''|*[!0-9]*) continue ;; esac", + 'while kill -0 "$p" 2>/dev/null; do sleep 1; done', + 'user_busy && continue', + 'rm "$f" 2>/dev/null && exit 0', + 'done' + ].join('\n') +} /** `wsl.exe` argv for a holder: only a Windows-side WSL client keeps a distro from idling out. */ export function buildWslDistroHolderArgs(distro: string): string[] { - return buildWslExecArgs(distro, ['/bin/sh', '-c', HOLDER_SCRIPT, WSL_DISTRO_HOLDER_MARKER]) + return buildWslExecArgs(distro, [ + '/bin/sh', + '-c', + buildWslDistroHolderScript(), + WSL_DISTRO_HOLDER_MARKER + ]) } function escapeRegExp(value: string): string { diff --git a/src/main/wsl/wsl-distro-holder-script.test.ts b/src/main/wsl/wsl-distro-holder-script.test.ts new file mode 100644 index 00000000000..89a76e834a9 --- /dev/null +++ b/src/main/wsl/wsl-distro-holder-script.test.ts @@ -0,0 +1,171 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir, userInfo } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { ORCAD_WSL_HOLDER_RELEASE_FILENAME } from '../../shared/orcad-idle-exit' +import { + buildWslDistroHolderScript, + WSL_DISTRO_HOLDER_IGNORED_PROCESS_NAMES, + WSL_DISTRO_HOLDER_MARKER +} from './wsl-distro-holder-command' + +// The script runs inside the distro; any POSIX sh here exercises the same text. A fake /proc stands +// in for the distro's, so the test runner's own processes never count as the user's work. +let home: string +let proc: string +let children: ChildProcess[] +const me = userInfo().uid + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'orca-holder-script-')) + mkdirSync(join(home, '.orca')) + proc = join(home, 'proc') + mkdirSync(proc) + children = [] +}) + +afterEach(() => { + for (const child of children) { + child.kill('SIGKILL') + } + rmSync(home, { recursive: true, force: true }) +}) + +function track(child: ChildProcess): ChildProcess { + children.push(child) + return child +} + +function startHolder(env: NodeJS.ProcessEnv = { HOME: home, PATH: process.env.PATH }) { + const child = track( + spawn('/bin/sh', ['-c', buildWslDistroHolderScript(1, proc), WSL_DISTRO_HOLDER_MARKER], { + env, + stdio: 'ignore' + }) + ) + const exited = new Promise((resolve) => child.once('exit', resolve)) + return { child, exited } +} + +function within(ms: number, promise: Promise): Promise { + return Promise.race([ + promise, + // ast-grep-ignore: test-no-real-time-wait -- a real shell must still be holding after this long + new Promise<'running'>((resolve) => setTimeout(() => resolve('running'), ms)) + ]) +} + +const releasePath = (): string => join(home, '.orca', ORCAD_WSL_HOLDER_RELEASE_FILENAME) + +/** One fake /proc entry, as the kernel writes its status file. */ +function fakeProcess( + pid: number, + fields: { name?: string; state?: string; ppid?: number; uid?: number } = {} +): string { + const dir = join(proc, String(pid)) + mkdirSync(dir) + writeFileSync( + join(dir, 'status'), + `Name:\t${fields.name ?? 'tmux: server'}\nState:\t${fields.state ?? 'S (sleeping)'}\n` + + `PPid:\t${fields.ppid ?? 1}\nUid:\t${fields.uid ?? me}\t${fields.uid ?? me}\t0\t0\n` + ) + return dir +} + +/** A holder that already ran its start-up `rm -f`, then a request from an orcad that has exited. */ +async function holderWithRequest() { + const holder = startHolder() + expect(await within(300, holder.exited)).toBe('running') + const orcad = track(spawn('/bin/sh', ['-c', 'exit 0'], { stdio: 'ignore' })) + await new Promise((resolve) => orcad.once('exit', resolve)) + writeFileSync(releasePath(), `${orcad.pid}\n`) + return holder +} + +describe.skipIf(process.platform === 'win32')('WSL distro holder script', () => { + it('holds while nothing asked for a release', async () => { + const { exited } = startHolder() + expect(await within(2_500, exited)).toBe('running') + }) + + it('drops a request left from before it started: a client wants the distro now', async () => { + writeFileSync(releasePath(), '999999\n') + const { exited } = startHolder() + expect(await within(2_500, exited)).toBe('running') + expect(existsSync(releasePath())).toBe(false) + }) + + it('waits for the requesting orcad to exit, then claims the request and ends', async () => { + const holder = startHolder() + expect(await within(300, holder.exited)).toBe('running') + const orcad = track(spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' })) + writeFileSync(releasePath(), `${orcad.pid}\n`) + + expect(await within(2_500, holder.exited)).toBe('running') + orcad.kill('SIGKILL') + expect(await within(4_000, holder.exited)).toBe(0) + expect(existsSync(releasePath())).toBe(false) + }) + + it('keeps holding when a starting orcad withdrew the request first', async () => { + const holder = startHolder() + expect(await within(300, holder.exited)).toBe('running') + const orcad = track(spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' })) + writeFileSync(releasePath(), `${orcad.pid}\n`) + await within(1_500, holder.exited) + rmSync(releasePath()) + orcad.kill('SIGKILL') + expect(await within(3_000, holder.exited)).toBe('running') + }) + + it('ignores a request it cannot read as a pid', async () => { + const holder = startHolder() + expect(await within(300, holder.exited)).toBe('running') + writeFileSync(releasePath(), 'not-a-pid\n') + expect(await within(2_500, holder.exited)).toBe('running') + }) + + it('never releases without a home to find the request in', async () => { + writeFileSync(releasePath(), '999999\n') + const { exited } = startHolder({ PATH: process.env.PATH }) + expect(await within(2_500, exited)).toBe('running') + }) + + it('keeps the distro while the user still runs something, and lets go once it ends', async () => { + // e.g. a detached tmux server or a nohup'd job started from an Orca terminal. + const job = fakeProcess(4242) + const { exited } = await holderWithRequest() + expect(await within(2_500, exited)).toBe('running') + expect(existsSync(releasePath())).toBe(true) + rmSync(job, { recursive: true }) + expect(await within(3_000, exited)).toBe(0) + }) + + it('keeps the distro while a container runs, though its shim belongs to root', async () => { + const shim = fakeProcess(4343, { name: 'containerd-shim-runc-v2', uid: 0 }) + // dockerd itself, idle and root-owned, is a system service, not work. + fakeProcess(4344, { name: 'dockerd', uid: 0 }) + const { exited } = await holderWithRequest() + expect(await within(2_500, exited)).toBe('running') + rmSync(shim, { recursive: true }) + expect(await within(3_000, exited)).toBe(0) + }) + + it('does not count session plumbing, zombies, other users or its own children', async () => { + let pid = 5000 + for (const name of WSL_DISTRO_HOLDER_IGNORED_PROCESS_NAMES) { + fakeProcess(pid++, { name }) + } + fakeProcess(pid++, { state: 'Z (zombie)' }) + fakeProcess(pid++, { uid: me + 1 }) + const holder = startHolder() + fakeProcess(pid++, { name: 'sleep', ppid: holder.child.pid }) + fakeProcess(holder.child.pid ?? pid++, { name: 'sh' }) + expect(await within(300, holder.exited)).toBe('running') + const orcad = track(spawn('/bin/sh', ['-c', 'exit 0'], { stdio: 'ignore' })) + await new Promise((resolve) => orcad.once('exit', resolve)) + writeFileSync(releasePath(), `${orcad.pid}\n`) + expect(await within(3_000, holder.exited)).toBe(0) + }) +}) diff --git a/src/main/wsl/wsl-distro-holder.test.ts b/src/main/wsl/wsl-distro-holder.test.ts index a769407a415..ca06b09c7bb 100644 --- a/src/main/wsl/wsl-distro-holder.test.ts +++ b/src/main/wsl/wsl-distro-holder.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { WindowsProcessEvidence } from '../daemon/daemon-incarnation-evidence-types' import { buildWslDistroHolderArgs, + buildWslDistroHolderScript, isWslDistroHolderCommandLine, WSL_DISTRO_HOLDER_MARKER } from './wsl-distro-holder-command' @@ -112,9 +113,12 @@ describe('holder command line', () => { '--exec', '/bin/sh', '-c', - 'while :; do sleep 3600; done', + buildWslDistroHolderScript(), WSL_DISTRO_HOLDER_MARKER ]) + // A holder that slept an hour between looks would keep an idle distro up that long. + expect(buildWslDistroHolderScript()).toContain('sleep 15\n') + expect(buildWslDistroHolderScript()).not.toContain('infinity') }) it('matches only our holder for the same distro', () => { diff --git a/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts b/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts index 23aaf37c1ae..2720f26a4be 100644 --- a/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts +++ b/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts @@ -257,9 +257,17 @@ describe('WslOrcadLivenessSupervisor', () => { expect(h.readIdentity).toHaveBeenCalledTimes(identityReads) }) + it('re-ensures the holder once a connect is armed', async () => { + const h = harness() + await h.supervisor.connect('Ubuntu') + // An earlier orcad's idle release may have ended the holder the launch adopted. + expect(h.ensureHolder).toHaveBeenCalledExactlyOnceWith('Ubuntu') + }) + it('never boots a stopped distro for background work; the gate resumes it once running', async () => { const h = harness({ visible: false }) await h.supervisor.connect('Ubuntu') + h.ensureHolder.mockClear() h.state.running = { status: 'confirmed', distros: [] } h.lastWatch().exit(1) await h.advance(60_000) diff --git a/src/main/wsl/wsl-orcad-liveness-supervisor.ts b/src/main/wsl/wsl-orcad-liveness-supervisor.ts index 54a4f10371b..cf769a4560f 100644 --- a/src/main/wsl/wsl-orcad-liveness-supervisor.ts +++ b/src/main/wsl/wsl-orcad-liveness-supervisor.ts @@ -156,6 +156,8 @@ export class WslOrcadLivenessSupervisor { } return } + // A holder that took the previous orcad's idle release just before this launch is gone now. + await ensureHolderQuietly(this.deps.ensureHolder, entry.distro) // Do not overwrite the timer owned by a superseding connect. if (entry.generation === generation) { this.scheduleHeartbeat(entry) diff --git a/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts b/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts new file mode 100644 index 00000000000..ab7812636cb --- /dev/null +++ b/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts @@ -0,0 +1,293 @@ +// WSL via orcad, phase 3 of the real-distro lane (R40 / E17): after the survive phase's teardown, +// a fresh orcad with a short idle timeout keeps its distro while a terminal is live, then +// idle-exits once it is closed and the holder lets the distro stop with no release call. +// Records guest and VM memory before and after. .github/workflows/wsl-orcad-e2e.yml runs it. +// +// Run: ORCA_RUN_WSL_ORCAD_LANE=1 pnpm test +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } })) + +import { runProcess } from '@orca/process-host' +import { setAppEnvironment } from '../../shared/app-environment' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/electron-remote-runtime-client-capabilities' +import { ORCAD_E2E_IDLE_TIMEOUT_ENV } from '../../shared/orcad-idle-exit' +import { pairedRuntimeConnectionFromOffer } from '../../shared/paired-runtime-connection' +import { parsePairingCode } from '../../shared/pairing' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime/remote-runtime-client' +import { execCommand } from '../ssh/remote-exec/remote-exec-command' +import { readConfirmedRunningWslDistros } from '../wsl-background-distro-gate' +import { readWindowsProcessTableFresh } from '../windows/windows-process-table' +import { resolveWslInteropSpawnCwd } from '../wsl-interop-spawn-directory' +import { createWslDistroHolder, type WslDistroHolder } from './wsl-distro-holder' +import { WSL_DISTRO_HOLDER_POLL_SECONDS } from './wsl-distro-holder-command' +import { wslDistroHolderKey } from './wsl-distro-holder-record' +import { WslOrcadConnectionManager, type WslOrcadConnection } from './wsl-orcad-connection' +import { defaultWslOrcadEnvironmentId, deployAndLaunchWslOrcad } from './wsl-orcad-deploy' +import { createWslOrcadHostTransport } from './wsl-orcad-host-transport' + +const RUN = process.env.ORCA_RUN_WSL_ORCAD_LANE === '1' +const PLAIN = process.env.ORCA_WSL_ORCAD_LANE_PLAIN_DISTRO ?? 'orca-lane-plain' +const WORK_DIR = process.env.ORCA_WSL_ORCAD_LANE_DIR ?? join(tmpdir(), 'orca-wsl-orcad-lane') +const USER_DATA = join(WORK_DIR, 'userData') +const STATE_FILE = join(WORK_DIR, 'launch-state.json') +const RECEIPT_DIR = process.env.ORCA_WSL_ORCAD_LANE_RECEIPTS ?? join(WORK_DIR, 'receipts') +const IDLE_TIMEOUT_MS = 15_000 +// Idle period, one holder poll, then WSL's own stop of a distro with no client left. +const RELEASE_BUDGET_MS = IDLE_TIMEOUT_MS + WSL_DISTRO_HOLDER_POLL_SECONDS * 1000 + 90_000 +const HELD_WITH_TERMINAL_MS = IDLE_TIMEOUT_MS + WSL_DISTRO_HOLDER_POLL_SECONDS * 1000 + 45_000 + +type Receipt = Record +const receipt: Receipt = { phase: 'idle-release' } +const commandLog: string[] = [] +receipt.commandLog = commandLog +let holder: WslDistroHolder +let manager: WslOrcadConnectionManager + +function installLaneAppEnvironment(): void { + const { version } = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf8')) + setAppEnvironment({ + getPath: (name) => (name === 'userData' ? USER_DATA : tmpdir()), + getAppPath: () => process.cwd(), + getVersion: () => version, + isPackaged: () => false, + onWillQuit: () => {}, + exit: () => {}, + getAppMetrics: () => [] + }) +} + +/** Each distro's template from the launch phase, so this phase does not update it. */ +function launchTemplates(): Map { + const state: unknown = JSON.parse(readFileSync(STATE_FILE, 'utf8')) + const templates = + typeof state === 'object' && state !== null && 'templates' in state ? state.templates : null + return new Map( + Object.entries(typeof templates === 'object' && templates !== null ? templates : {}).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string' + ) + ) +} + +function createManager(): void { + const templates = launchTemplates() + manager = new WslOrcadConnectionManager({ + holder, + // Logs every guest command with its time, to match the zombie's start time to one step. + createTransport: (target) => { + const transport = createWslOrcadHostTransport({ target }) + const log = (path: string, command: string): void => { + commandLog.push(`${Math.round(performance.now())} ${path} ${command.slice(0, 160)}`) + } + return { + ...transport, + exec: (command, options) => { + log('exec', command) + return transport.exec(command, options) + }, + execWithClosedStdin: (command, options) => { + log('loop', command) + return ( + transport.execWithClosedStdin?.(command, options) ?? transport.exec(command, options) + ) + } + } + }, + deployAndLaunch: (distro, options) => + deployAndLaunchWslOrcad(distro, { + ...options, + templateDir: templates.get(wslDistroHolderKey(distro)) + }) + }) +} + +/** Disposed after each command: a kept exec loop is a wsl.exe client that holds the distro up. */ +async function guest(command: string): Promise { + const target = { + kind: 'wsl' as const, + id: defaultWslOrcadEnvironmentId(PLAIN), + distro: PLAIN, + label: PLAIN + } + const transport = createWslOrcadHostTransport({ target }) + try { + return await execCommand(transport, command, { timeoutMs: 60_000 }) + } finally { + transport.dispose() + } +} + +async function rpc(connection: WslOrcadConnection, method: string, params?: unknown) { + const offer = parsePairingCode(connection.pairingOffer) + if (!offer) { + throw new Error(`Unparseable pairing offer for ${connection.distro}`) + } + const response = await sendRemoteRuntimeRequest( + pairedRuntimeConnectionFromOffer(offer), + method, + params, + 30_000, + undefined, + undefined, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + if (!response.ok) { + throw new Error(`${method} failed: ${response.error.code}: ${response.error.message}`) + } + return response.result +} + +async function plainRunning(): Promise { + const running = await readConfirmedRunningWslDistros() + if (running.status !== 'confirmed') { + throw new Error('wsl --list --running was unavailable') + } + return running.distros.some((d) => wslDistroHolderKey(d) === wslDistroHolderKey(PLAIN)) +} + +/** Working set (KiB) of the WSL VM process, or null once the VM has shut down. */ +async function vmWorkingSetKiB(): Promise { + for (const image of ['vmmemWSL', 'vmmem']) { + const result = await runProcess({ + program: join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'tasklist.exe'), + args: ['/fi', `IMAGENAME eq ${image}`, '/fo', 'csv', '/nh'], + cwd: resolveWslInteropSpawnCwd(), + timeoutMs: 30_000 + }) + const row = result.stdout.split(/\r?\n/u).find((line) => line.startsWith(`"${image}"`)) + const memory = row + ?.split('","') + .at(-1) + ?.replace(/[^0-9]/gu, '') + if (memory) { + return Number(memory) + } + } + return null +} + +/** What Orca's processes and the whole guest use while the distro is held. */ +async function guestMemory(): Promise { + const out = await guest( + 'ps -u "$(id -u)" -o rss=,args= | awk \'!/awk/ && /orcad|node|orca-wsl-distro-holder/ {k+=$1} END {print k+0}\'; ' + + "awk '/MemTotal/ {t=$2} /MemAvailable/ {a=$2} END {print t-a}' /proc/meminfo" + ) + const [orcaRssKiB, guestUsedKiB] = out.trim().split(/\s+/u).map(Number) + return { orcaRssKiB, guestUsedKiB } +} + +/** Polls a real distro, whose state no fake clock moves. */ +function waitFor( + what: string, + read: () => Promise, + timeoutMs: number, + intervalMs = 1_000 +): Promise { + return vi.waitFor( + async () => { + const value = await read() + if (value === null) { + throw new Error(`Timed out after ${timeoutMs} ms waiting for ${what}`) + } + return value + }, + { timeout: timeoutMs, interval: intervalMs } + ) +} + +async function idleReleasePhase(): Promise { + process.env[ORCAD_E2E_IDLE_TIMEOUT_ENV] = String(IDLE_TIMEOUT_MS) + receipt.vmBeforeLaunchKiB = await vmWorkingSetKiB() + // Aligns guest uptime with this process's clock for the command log. + receipt.clockAlignment = `${Math.round(performance.now())} ${(await guest('cat /proc/uptime')).trim()}` + let connection = await manager.ensure(PLAIN) + // The release hunt: a zombie `cat` under a session init kept the distro up; find which step leaves it. + const zombies = + 'cat /proc/uptime; for p in $(ps -eo pid=,stat= | awk \'$2 ~ /^Z/ {print $1}\'); do echo "Z $p $(cut -d" " -f4-8,22 /proc/$p/stat)"; done; ps -eo pid,ppid,sid,stat,args' + receipt.processesAfterEnsure = await guest(zombies) + await rpc(connection, 'status.get') + receipt.processesAfterLaunch = await guest(zombies) + const repoPath = (await guest('cd "$HOME/wsl-orcad-lane-repo" && pwd')).trim() + const created = await rpc<{ terminal: { handle: string } }>(connection, 'terminal.create', { + worktree: `path:${repoPath}` + }) + receipt.held = { ...(await guestMemory()), vmKiB: await vmWorkingSetKiB() } + + // No client, but a live terminal: orcad must not idle-exit and the holder must stay. + await manager.dispose() + // ast-grep-ignore: test-no-real-time-wait -- proves the distro stays up across the whole real idle window + await new Promise((resolve) => setTimeout(resolve, HELD_WITH_TERMINAL_MS)) + expect(await plainRunning(), 'a live terminal keeps the distro').toBe(true) + expect(await holder.holderState(PLAIN)).toBe('live') + + createManager() + connection = await manager.ensure(PLAIN) + expect(connection.pid, 'the same orcad served the reconnect').not.toBeNull() + await rpc(connection, 'terminal.close', { terminal: created.terminal.handle }) + await manager.dispose() + + // No terminal left, but a detached job the user started (like tmux or nohup) still runs: + // orcad idle-exits, yet the holder must keep the distro and its request unclaimed. + await guest('setsid -f sleep 600 /dev/null 2>&1') + // ast-grep-ignore: test-no-real-time-wait -- proves the hold across the whole real idle window + await new Promise((resolve) => setTimeout(resolve, HELD_WITH_TERMINAL_MS)) + expect(await holder.holderState(PLAIN), 'a detached user job keeps the holder').toBe('live') + expect( + (await guest('[ -f "$HOME/.orca/wsl-holder-release" ] && echo yes || echo no')).trim() + ).toBe('yes') + // The bracket keeps pkill from matching the shell that runs it. + await guest("pkill -f 'slee[p] 600' || true") + const closedAt = performance.now() + + await waitFor( + `${PLAIN} to stop after orcad idle-exits`, + async () => ((await plainRunning()) ? null : true), + RELEASE_BUDGET_MS + ).catch(async (error: unknown) => { + // The distro is still running, so this boots nothing: record what kept it. + // Windows first: the guest probe below is itself a wsl.exe client. + receipt.windowsWslClients = (await readWindowsProcessTableFresh()) + .filter((row) => /^wsl(host)?\.exe$/iu.test(row.name)) + .map((row) => `${row.pid} ${row.command}`) + receipt.holderState = await holder.holderState(PLAIN) + receipt.stillRunning = await guest( + 'cat /proc/uptime; ls -la "$HOME/.orca/wsl-holder-release" 2>&1; ps -eo pid,ppid,sid,stat,args | grep -v orca-wsl-exec-loop || true; for p in $(ps -eo pid=,ppid=,args= | awk \'$2 == 1 && $3 == "/init" {print $1}\'); do echo "INIT $p $(cut -d" " -f22 /proc/$p/stat)"; done' + ).catch(String) + throw error + }) + receipt.secondsFromLastClientToDistroStop = Math.round((performance.now() - closedAt) / 1000) + // Nothing here called release: the holder ended itself. + expect(await holder.holderState(PLAIN)).toBe('exited') + receipt.vmAfterStopKiB = await waitFor( + 'the WSL VM to shut down', + async () => ((await vmWorkingSetKiB()) === null ? 0 : null), + 150_000, + 5_000 + ).catch(() => vmWorkingSetKiB()) +} + +describe.runIf(RUN && process.platform === 'win32')('WSL distro idle release', () => { + beforeAll(() => { + mkdirSync(USER_DATA, { recursive: true }) + mkdirSync(RECEIPT_DIR, { recursive: true }) + installLaneAppEnvironment() + holder = createWslDistroHolder({ userDataPath: USER_DATA }) + createManager() + }) + + afterAll(async () => { + await manager?.dispose().catch(() => {}) + await holder?.release(PLAIN).catch(() => {}) + writeFileSync(join(RECEIPT_DIR, 'idle-release.json'), `${JSON.stringify(receipt, null, 2)}\n`) + }) + + // Real idle timeout, holder poll, WSL's own stop and VM shutdown, all on a live distro. + it( + 'keeps a distro with a live terminal, then lets it stop once orcad idle-exits', + idleReleasePhase, + RELEASE_BUDGET_MS + 2 * HELD_WITH_TERMINAL_MS + 6 * 60_000 + ) +}) diff --git a/src/shared/orcad-idle-exit.ts b/src/shared/orcad-idle-exit.ts index e9e39211aad..51ec881b3c4 100644 --- a/src/shared/orcad-idle-exit.ts +++ b/src/shared/orcad-idle-exit.ts @@ -28,3 +28,9 @@ export const OrcadIdleStopRecordSchema = z.object({ }) export type OrcadIdleStopRecord = z.infer + +/** + * In a WSL distro's data root (`~/.orca`): orcad's pid, written on an idle stop that left no + * terminal daemon. The distro holder ends once that pid exits, so WSL may stop the idle distro. + */ +export const ORCAD_WSL_HOLDER_RELEASE_FILENAME = 'wsl-holder-release'