diff --git a/src/main/wsl/wsl-host-connector.ts b/src/main/wsl/wsl-host-connector.ts index b187a3acf24..899c3fe9289 100644 --- a/src/main/wsl/wsl-host-connector.ts +++ b/src/main/wsl/wsl-host-connector.ts @@ -1,3 +1,5 @@ +import { WslOrcadBridgeActivity } from './wsl-orcad-bridge-activity' +import { listenWslOrcadStdioBridge } from './wsl-orcad-stdio-bridge' import { createWslOrcadPairingRecovery } from './wsl-orcad-pairing-repair' import { readConfirmedRunningWslDistros } from '../wsl-background-distro-gate' import { parsePairingCode } from '../../shared/pairing' @@ -64,8 +66,15 @@ export function createWslHostConnector(options: { : Promise.reject(refuseUnknown(distro)) } const pairingRecovery = createWslOrcadPairingRecovery(options.userDataPath) + // The desktop's own bridged sockets double as the liveness watch; no bridge is held for it. + const bridgeActivity = new WslOrcadBridgeActivity() const manager = new WslOrcadConnectionManager({ holder, + listen: (transport, listenOptions) => + listenWslOrcadStdioBridge(transport, { + ...listenOptions, + events: bridgeActivity.events(listenOptions.environmentId) + }), pairingRecovery, getPairingOffer: (environmentId) => findWslHostDeployment(options.userDataPath, environmentId) @@ -81,6 +90,7 @@ export function createWslHostConnector(options: { }) const liveness = createWslOrcadLivenessSupervisor({ manager, + bridgeActivity, holder, hasUserVisibleWork: options.hasUserVisibleWork }) diff --git a/src/main/wsl/wsl-orcad-bridge-activity.test.ts b/src/main/wsl/wsl-orcad-bridge-activity.test.ts new file mode 100644 index 00000000000..d7becf9ea75 --- /dev/null +++ b/src/main/wsl/wsl-orcad-bridge-activity.test.ts @@ -0,0 +1,229 @@ +import { connect, createServer, type Server, type Socket } from 'node:net' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { checkOrcadStdioBridge, openOrcadStdioBridgeChannel } from '../ssh/orcad-stdio-bridge' +import type { StartedPortForward } from '../ssh/port-forwarding/ssh-port-forward-provider' +import { + probeThenWatchWslOrcadBridges, + WSL_ORCAD_BRIDGE_EXIT_COOLDOWN_MS, + WslOrcadBridgeActivity +} from './wsl-orcad-bridge-activity' +import { fakeWslBridgeChildren, posixOrcadStdioBridge } from './wsl-orcad-bridge-test-child' +import { createWslOrcadHostTransport } from './wsl-orcad-host-transport' +import { listenWslOrcadStdioBridge } from './wsl-orcad-stdio-bridge' +import type { WslOrcadWatch } from './wsl-orcad-watch' + +const ENV = 'wsl:ubuntu' +const cleanups: (() => unknown)[] = [] + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).toReversed()) { + await cleanup() + } +}) + +/** Stands in for orcad: keeps every accepted socket so a test can end it from the guest side. */ +async function fakeOrcad(): Promise<{ server: Server; port: number; accepted: Socket[] }> { + const accepted: Socket[] = [] + const server = createServer((socket) => { + accepted.push(socket) + socket.on('error', () => {}) + socket.resume() + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + cleanups.push(() => { + for (const socket of accepted) { + socket.destroy() + } + return new Promise((resolve) => server.close(resolve)) + }) + const address = server.address() + return { server, port: typeof address === 'object' && address ? address.port : 0, accepted } +} + +async function bridgedHost( + activity: WslOrcadBridgeActivity, + orcad: { port: number; accepted: Socket[] } +) { + const remotePort = orcad.port + const children = fakeWslBridgeChildren() + const transport = createWslOrcadHostTransport({ + target: { kind: 'wsl', id: ENV, distro: 'Ubuntu', label: 'Ubuntu' }, + spawn: children.spawn + }) + const forward: StartedPortForward = await listenWslOrcadStdioBridge( + transport, + { + environmentId: ENV, + remotePort, + localPort: 0, + label: 'Ubuntu', + events: activity.events(ENV) + }, + { + resolveBridge: async (_conn, port) => posixOrcadStdioBridge(port), + checkBridge: checkOrcadStdioBridge, + openChannel: openOrcadStdioBridgeChannel + } + ) + cleanups.push(() => forward.close()) + // The listener's start-up check dials orcad once; let it land so tests count only their own. + await expect.poll(() => orcad.accepted.length).toBe(1) + return forward +} + +function dial(port: number): Promise { + return new Promise((resolve, reject) => { + const socket = connect(port, '127.0.0.1', () => resolve(socket)) + socket.once('error', reject) + cleanups.push(() => socket.destroy()) + }) +} + +describe('the desktop’s bridged sockets as the liveness watch', () => { + it('fires once when orcad ends a socket, and proves liveness only while one is open', async () => { + const orcad = await fakeOrcad() + const activity = new WslOrcadBridgeActivity() + const forward = await bridgedHost(activity, orcad) + const watch = activity.watch(ENV) + const exits = vi.fn() + watch.onExit(exits) + expect(watch.proving()).toBe(false) + + const before = orcad.accepted.length + await dial(forward.entry.localPort) + await expect.poll(() => orcad.accepted.length).toBe(before + 1) + await expect.poll(() => watch.proving()).toBe(true) + + // orcad exiting closes its end; the guest ends the bridge first. + orcad.accepted.at(-1)?.destroy() + await expect.poll(() => exits.mock.calls.length).toBe(1) + expect(exits).toHaveBeenCalledWith(null) + await expect.poll(() => watch.proving()).toBe(false) + }) + + it('stays quiet when orcad finishes a normal close while another socket is open', async () => { + const orcad = await fakeOrcad() + const activity = new WslOrcadBridgeActivity() + const forward = await bridgedHost(activity, orcad) + const watch = activity.watch(ENV) + const exits = vi.fn() + watch.onExit(exits) + const before = orcad.accepted.length + // The desktop's control socket stays open; a one-shot RPC socket comes and goes. + await dial(forward.entry.localPort) + const oneShot = await dial(forward.entry.localPort) + await expect.poll(() => orcad.accepted.length).toBe(before + 2) + // A WebSocket close: the client sends its close frame, the server ends the stream first. + oneShot.write(Buffer.from([0x88, 0x80, 0, 0, 0, 0])) + orcad.accepted.at(-1)?.end() + await expect.poll(() => oneShot.destroyed || oneShot.readableEnded).toBe(true) + oneShot.end() + await expect.poll(() => orcad.accepted.at(-1)?.closed).toBe(true) + expect(exits).not.toHaveBeenCalled() + expect(activity.openSockets(ENV)).toBe(1) + // orcad exiting then ends the last one: that is the exit signal. + orcad.accepted.at(-2)?.destroy() + await expect.poll(() => exits.mock.calls.length).toBe(1) + }) + + it('stays quiet when the desktop closes its own socket', async () => { + const orcad = await fakeOrcad() + const activity = new WslOrcadBridgeActivity() + const forward = await bridgedHost(activity, orcad) + const watch = activity.watch(ENV) + const exits = vi.fn() + watch.onExit(exits) + const socket = await dial(forward.entry.localPort) + await expect.poll(() => watch.proving()).toBe(true) + + socket.end() + await expect.poll(() => watch.proving()).toBe(false) + expect(exits).not.toHaveBeenCalled() + }) + + it('stops listening once closed, and keys hosts apart', () => { + const activity = new WslOrcadBridgeActivity() + const watch = activity.watch(ENV) + const other = vi.fn() + activity.watch('wsl:debian').onExit(other) + const exits = vi.fn() + watch.onExit(exits) + watch.close() + const events = activity.events(ENV) + events.opened() + events.ended(true) + expect(exits).not.toHaveBeenCalled() + expect(other).not.toHaveBeenCalled() + }) +}) + +describe('bridge activity signals', () => { + it('coalesces a burst of exit signals into one, and reports the rest as idle', () => { + let now = 0 + const activity = new WslOrcadBridgeActivity(() => now) + const events = activity.events(ENV) + const seen: string[] = [] + const watchOnce = (): void => { + const watch = activity.watch(ENV) + watch.onExit(() => seen.push('exit')) + watch.onIdle?.(() => seen.push('idle')) + } + watchOnce() + // One-shot sockets with nothing held: each ends on the guest side first. + events.opened() + events.ended(true) + watchOnce() + now += WSL_ORCAD_BRIDGE_EXIT_COOLDOWN_MS - 1 + events.opened() + events.ended(true) + now += 1 + events.opened() + events.ended(true) + expect(seen).toEqual(['exit', 'idle', 'exit']) + }) + + it('reports the last socket closing from the desktop side as idle, not exit', () => { + const activity = new WslOrcadBridgeActivity(() => 0) + const events = activity.events(ENV) + const watch = activity.watch(ENV) + const exits = vi.fn() + const idles = vi.fn() + watch.onExit(exits) + watch.onIdle?.(idles) + events.opened() + events.opened() + events.ended(false) + expect(idles).not.toHaveBeenCalled() + events.ended(false) + expect(idles).toHaveBeenCalledOnce() + expect(exits).not.toHaveBeenCalled() + }) +}) + +describe('probeThenWatchWslOrcadBridges', () => { + const inert = (signal: WslOrcadWatch['signal']): WslOrcadWatch => ({ + signal, + onExit: () => {}, + close: vi.fn(), + proving: () => false + }) + + it('reports a refused or unverifiable probe as before, holding nothing', async () => { + const activity = new WslOrcadBridgeActivity() + for (const signal of ['refused', 'unverifiable'] as const) { + const probe = inert(signal) + await expect(probeThenWatchWslOrcadBridges(async () => probe, activity, ENV)).resolves.toBe( + probe + ) + } + }) + + it('closes the probe bridge once it proved the port, and watches the desktop’s sockets', async () => { + const activity = new WslOrcadBridgeActivity() + const probe = inert('ready') + const watch = await probeThenWatchWslOrcadBridges(async () => probe, activity, ENV) + expect(probe.close).toHaveBeenCalledOnce() + expect(watch).not.toBe(probe) + expect(watch.signal).toBe('ready') + }) +}) diff --git a/src/main/wsl/wsl-orcad-bridge-activity.ts b/src/main/wsl/wsl-orcad-bridge-activity.ts new file mode 100644 index 00000000000..59e7424349a --- /dev/null +++ b/src/main/wsl/wsl-orcad-bridge-activity.ts @@ -0,0 +1,131 @@ +/** + * The bridged loopback sockets a WSL host's listener carries (the desktop's own control and + * request connections), read as a liveness watch so no extra wsl.exe bridge is held just to see + * orcad exit. The guest ending the last open socket means orcad closed it or exited; anything it + * means for certain is settled by the supervisor's verify. + */ +import type { RemoteCommandChannel } from '../managed-server/install/remote-install-transport' +import type { WslOrcadWatch } from './wsl-orcad-watch' + +export type WslOrcadBridgeChannelEvents = { + opened(): void + /** `byGuest` when the guest side ended before the desktop closed or ended its side. */ + ended(byGuest: boolean): void +} + +/** Reports a bridge channel's lifetime; a close or stdin EOF from the desktop is not the guest's. */ +export function observeWslOrcadBridgeChannel( + channel: RemoteCommandChannel, + events: WslOrcadBridgeChannelEvents +): RemoteCommandChannel { + let closedLocally = false + const close = channel.close.bind(channel) + channel.close = () => { + closedLocally = true + close() + } + // Decided when the guest's output ends: by then the desktop may already be hanging up in turn. + const guestFirst = (): boolean => !closedLocally && !channel.writableEnded + let byGuest: boolean | null = null + channel.once('end', () => { + byGuest ??= guestFirst() + }) + events.opened() + channel.once('exit', () => events.ended(byGuest ?? guestFirst())) + return channel +} + +/** Exit signals this close together are one orcad exit, or one-shot sockets coming and going. */ +export const WSL_ORCAD_BRIDGE_EXIT_COOLDOWN_MS = 2_000 + +type ActivityListener = (kind: 'exit' | 'idle') => void + +export class WslOrcadBridgeActivity { + private readonly live = new Map() + private readonly listeners = new Map>() + private readonly lastExitAt = new Map() + + constructor(private readonly now: () => number = () => performance.now()) {} + + events(environmentId: string): WslOrcadBridgeChannelEvents { + const key = environmentId + return { + opened: () => this.live.set(key, (this.live.get(key) ?? 0) + 1), + ended: (byGuest) => { + const live = Math.max(0, (this.live.get(key) ?? 0) - 1) + this.live.set(key, live) + if (live > 0) { + return + } + // orcad exiting ends every socket at once; ending one while another stays open is a + // normal close, which a WebSocket lets the server finish first, so it reads as guest-ended. + // Leading-edge cooldown: a burst of one-shot sockets with none held is one verify, not many. + const last = this.lastExitAt.get(key) + const exit = + byGuest && (last === undefined || this.now() - last >= WSL_ORCAD_BRIDGE_EXIT_COOLDOWN_MS) + if (exit) { + this.lastExitAt.set(key, this.now()) + } + for (const listener of Array.from(this.listeners.get(key) ?? [])) { + listener(exit ? 'exit' : 'idle') + } + } + } + } + + /** How many of the desktop's sockets to this host are bridged right now. */ + openSockets(environmentId: string): number { + return this.live.get(environmentId) ?? 0 + } + + /** + * A watch that fires once when the guest ends the last open bridged socket, proves only while + * one is open, and reports each time the last one closes any other way (no socket left to prove). + */ + watch(environmentId: string): WslOrcadWatch { + const key = environmentId + const listeners = this.listeners.get(key) ?? new Set() + this.listeners.set(key, listeners) + let onExit: ((code: number | null) => void) | null = null + let onIdle: (() => void) | null = null + const listener: ActivityListener = (kind) => { + if (kind === 'idle') { + onIdle?.() + return + } + const exit = onExit + close() + exit?.(null) + } + const close = (): void => { + listeners.delete(listener) + onExit = onIdle = null + } + listeners.add(listener) + return { + signal: 'ready', + onExit: (exit) => { + onExit = exit + }, + onIdle: (idle) => { + onIdle = idle + }, + close, + proving: () => (this.live.get(key) ?? 0) > 0 + } + } +} + +/** Probes the port with a short-lived bridge, then watches the desktop's own bridged sockets. */ +export async function probeThenWatchWslOrcadBridges( + probe: () => Promise, + activity: WslOrcadBridgeActivity, + environmentId: string +): Promise { + const probed = await probe() + if (probed.signal !== 'ready') { + return probed + } + probed.close() + return activity.watch(environmentId) +} diff --git a/src/main/wsl/wsl-orcad-liveness-building-blocks.test.ts b/src/main/wsl/wsl-orcad-liveness-building-blocks.test.ts index 70cb64902b0..6076f6a8cec 100644 --- a/src/main/wsl/wsl-orcad-liveness-building-blocks.test.ts +++ b/src/main/wsl/wsl-orcad-liveness-building-blocks.test.ts @@ -25,6 +25,7 @@ function fakeWatch(signal: WslOrcadWatch['signal']) { exit = listener }), close: vi.fn(), + proving: () => signal === 'ready', exit: (code: number | null) => exit?.(code) } return watch diff --git a/src/main/wsl/wsl-orcad-liveness-monitor.ts b/src/main/wsl/wsl-orcad-liveness-monitor.ts index fa21306f412..09bd9de685f 100644 --- a/src/main/wsl/wsl-orcad-liveness-monitor.ts +++ b/src/main/wsl/wsl-orcad-liveness-monitor.ts @@ -1,5 +1,7 @@ +import type { WslOrcadConnection } from './wsl-orcad-connection' import type { Supervised, + Trigger, WslOrcadLivenessDependencies, WslOrcadTimer } from './wsl-orcad-liveness-state' @@ -55,11 +57,48 @@ export async function ensureHolderQuietly( } } +/** A held watch that proves the port answers for it; a fresh one probes; a retired listener proves nothing. */ +export async function probeOrcadPort( + entry: Supervised, + connection: WslOrcadConnection | null, + attach: (connection: WslOrcadConnection) => Promise +): Promise { + if (!connection) { + return 'refused' + } + return entry.watch?.proving() ? 'ready' : attach(connection) +} + +/** While no watch proves orcad (no desktop socket open), a full verify runs this often instead. */ +export const WSL_ORCAD_IDLE_VERIFY_MS = 30_000 + +/** The next liveness tick: the heartbeat while a watch proves orcad, a sooner full verify while none does. */ +export function scheduleSupervisedHeartbeat( + entry: Supervised, + schedule: (delayMs: number, task: () => void) => WslOrcadTimer, + now: () => number, + heartbeatMs: number, + trigger: (trigger: Trigger) => void +): void { + entry.timer?.cancel() + entry.lastBeatMs = now() + const delayMs = entry.watch?.proving() + ? heartbeatMs + : Math.min(heartbeatMs, WSL_ORCAD_IDLE_VERIFY_MS) + entry.timer = schedule(delayMs, () => { + entry.timer = null + // Long-overdue ticks imply process/VM suspension; reverify all evidence. + const late = now() - entry.lastBeatMs > delayMs * 2 + trigger(late ? { source: 'verify' } : 'heartbeat') + }) +} + /** Holds a new watch unless a later connect superseded this one or already holds one. */ export async function attachSupervisedWatch( entry: Supervised, openWatch: () => Promise, - onExit: (code: number | null) => void + onExit: (code: number | null) => void, + onIdle: () => void = () => {} ): Promise { const generation = entry.generation const watch = await openWatch() @@ -77,5 +116,10 @@ export async function attachSupervisedWatch( onExit(code) } }) + watch.onIdle?.(() => { + if (entry.watch === watch) { + onIdle() + } + }) return 'ready' } diff --git a/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts b/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts index 5f6667fcfa9..b4e9828ab43 100644 --- a/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts +++ b/src/main/wsl/wsl-orcad-liveness-supervisor.test.ts @@ -7,6 +7,7 @@ import type { WslOrcadConnection } from './wsl-orcad-connection' import type { WslOrcadConnectionStatus } from './wsl-orcad-connection-status' import { EXIT_SETTLE_MS } from './wsl-orcad-liveness-evidence' import { WSL_ORCAD_HEARTBEAT_MS } from './wsl-orcad-liveness-state' +import { WSL_ORCAD_IDLE_VERIFY_MS } from './wsl-orcad-liveness-monitor' import { WslDistroNotRunningError } from './wsl-orcad-connection-errors' import { WslOrcadLivenessSupervisor } from './wsl-orcad-liveness-supervisor' import { WSL_ORCAD_RECONNECT_DELAYS_MS } from './wsl-orcad-reconnect-backoff' @@ -24,12 +25,13 @@ function harness(options: { visible?: boolean } = {}) { let pid = 100 let connection: WslOrcadConnection | null = null const statuses: WslOrcadConnectionStatus[] = [] - const watches: { exit: (code: number | null) => void; closed: boolean }[] = [] + const watches: { exit: (code: number | null) => void; idle: () => void; closed: boolean }[] = [] const state: { visible: boolean running: WslRunningDistrosRead identity: WslGuestIdentity | null watchSignal: WslOrcadWatch['signal'] + watchProves: boolean ensureFails: number ensureError?: Error } = { @@ -37,6 +39,7 @@ function harness(options: { visible?: boolean } = {}) { running: { status: 'confirmed', distros: ['Ubuntu'] }, identity: { distroInstance: 'boot:1', orcad: 'orcad:100' }, watchSignal: 'ready', + watchProves: true, ensureFails: 0 } const deferred: (() => unknown)[] = [] @@ -90,18 +93,27 @@ function harness(options: { visible?: boolean } = {}) { readIdentity, openWatch: async () => { if (state.watchSignal !== 'ready') { - return { signal: state.watchSignal, onExit: () => {}, close: () => {} } + return { + signal: state.watchSignal, + onExit: () => {}, + close: () => {}, + proving: () => false + } } - const record = { exit: (_code: number | null) => {}, closed: false } + const record = { exit: (_code: number | null) => {}, idle: () => {}, closed: false } watches.push(record) return { signal: 'ready', onExit: (listener) => { record.exit = listener }, + onIdle: (listener) => { + record.idle = listener + }, close: () => { record.closed = true - } + }, + proving: () => !record.closed && state.watchProves } }, collectCensus, @@ -461,6 +473,60 @@ describe('WslOrcadLivenessSupervisor', () => { expect(h.ensureHolder).toHaveBeenCalledWith('Ubuntu') }) + it('verifies fully on a heartbeat while its watch holds no open socket to prove orcad', async () => { + const h = harness() + await h.supervisor.connect('Ubuntu') + // The desktop's sockets are all closed: the watch proves nothing, so ask the guest. + h.state.watchProves = false + await h.advance(WSL_ORCAD_HEARTBEAT_MS) + expect(h.readIdentity).toHaveBeenCalledTimes(2) + // Same orcad: still ready, no verdict change. + expect(h.statuses.filter((status) => status.state !== 'ready')).toEqual([]) + }) + + it('bounds detection to the idle verify period while no desktop socket is open', async () => { + const h = harness() + h.state.watchProves = false + await h.supervisor.connect('Ubuntu') + const reads = h.readIdentity.mock.calls.length + await h.advance(WSL_ORCAD_IDLE_VERIFY_MS - 1) + expect(h.readIdentity).toHaveBeenCalledTimes(reads) + await h.advance(1) + expect(h.readIdentity).toHaveBeenCalledTimes(reads + 1) + await h.advance(WSL_ORCAD_IDLE_VERIFY_MS) + expect(h.readIdentity).toHaveBeenCalledTimes(reads + 2) + // With a socket open again, ticks go back to the heartbeat's cheap running-list read. + h.state.watchProves = true + await h.advance(WSL_ORCAD_IDLE_VERIFY_MS + WSL_ORCAD_HEARTBEAT_MS) + expect(h.readIdentity).toHaveBeenCalledTimes(reads + 2) + }) + + it('brings the next full check forward when the last desktop socket closes normally', async () => { + const h = harness() + await h.supervisor.connect('Ubuntu') + const reads = h.readIdentity.mock.calls.length + await h.advance(WSL_ORCAD_HEARTBEAT_MS / 2) + // The desktop closed its last socket: nothing proves orcad until the next check. + h.state.watchProves = false + h.lastWatch().idle() + await h.advance(WSL_ORCAD_IDLE_VERIFY_MS - 1) + expect(h.readIdentity).toHaveBeenCalledTimes(reads) + await h.advance(1) + expect(h.readIdentity).toHaveBeenCalledTimes(reads + 1) + }) + + it('treats a socket orcad closed for another reason as a verify, not a verdict', async () => { + const h = harness() + await h.supervisor.connect('Ubuntu') + // A watch riding the desktop's sockets reports no exit code. + h.lastWatch().exit(null) + await h.advance(5_000) + expect(h.readIdentity).toHaveBeenCalledTimes(2) + expect(h.statuses.filter((status) => status.state !== 'ready')).toEqual([]) + expect(h.ensure).toHaveBeenCalledTimes(1) + expect(h.watches).toHaveLength(2) + }) + it('treats a heartbeat far past due as a pause and verifies fully', async () => { const h = harness() await h.supervisor.connect('Ubuntu') diff --git a/src/main/wsl/wsl-orcad-liveness-supervisor.ts b/src/main/wsl/wsl-orcad-liveness-supervisor.ts index f99e0ffa675..b98a06cf1ab 100644 --- a/src/main/wsl/wsl-orcad-liveness-supervisor.ts +++ b/src/main/wsl/wsl-orcad-liveness-supervisor.ts @@ -14,6 +14,8 @@ import { cancellableWait, defaultSchedule, ensureHolderQuietly, + probeOrcadPort, + scheduleSupervisedHeartbeat, subscribeWake } from './wsl-orcad-liveness-monitor' import { @@ -168,19 +170,16 @@ export class WslOrcadLivenessSupervisor { return attachSupervisedWatch( entry, () => this.deps.openWatch(connection), - (code) => this.trigger(entry, { source: 'bridge', code }) + (code) => this.trigger(entry, { source: 'bridge', code }), + // Nothing proves orcad now: bring the next full check forward to the idle period. + () => this.scheduleHeartbeat(entry) ) } private scheduleHeartbeat(entry: Supervised): void { - entry.timer?.cancel() - entry.lastBeatMs = this.now() - entry.timer = this.schedule(this.heartbeatMs, () => { - entry.timer = null - // Long-overdue ticks imply process/VM suspension; reverify all evidence. - const late = this.now() - entry.lastBeatMs > this.heartbeatMs * 2 - this.trigger(entry, late ? { source: 'verify' } : 'heartbeat') - }) + scheduleSupervisedHeartbeat(entry, this.schedule, this.now, this.heartbeatMs, (trigger) => + this.trigger(entry, trigger) + ) } private scheduleVerify(entry: Supervised): void { @@ -232,7 +231,7 @@ export class WslOrcadLivenessSupervisor { return } } - if (trigger === 'heartbeat' && entry.watch && confirmsRunning(running, entry.key)) { + if (trigger === 'heartbeat' && entry.watch?.proving() && confirmsRunning(running, entry.key)) { await ensureHolderQuietly(this.deps.ensureHolder, entry.distro) this.scheduleHeartbeat(entry) return @@ -248,9 +247,7 @@ export class WslOrcadLivenessSupervisor { const connection = this.deps.manager.get(entry.distro) await gatherWslLivenessEvidence(evidence, { readIdentity: () => this.deps.readIdentity(entry.distro, connection?.pid ?? null), - // A held watch proves the port; a fresh one probes it; a retired listener proves nothing. - probePort: async () => - !connection ? 'refused' : entry.watch ? 'ready' : this.attachWatch(entry, connection) + probePort: () => probeOrcadPort(entry, connection, (held) => this.attachWatch(entry, held)) }) if (entry.generation === generation) { await this.act(entry, classifyWslOrcadLiveness(evidence)) diff --git a/src/main/wsl/wsl-orcad-liveness-wiring.ts b/src/main/wsl/wsl-orcad-liveness-wiring.ts index 4a60617dd20..1aecf19a686 100644 --- a/src/main/wsl/wsl-orcad-liveness-wiring.ts +++ b/src/main/wsl/wsl-orcad-liveness-wiring.ts @@ -1,4 +1,8 @@ /** C7 supplies visible demand; these production probes serve the WSL liveness supervisor. */ +import { + probeThenWatchWslOrcadBridges, + type WslOrcadBridgeActivity +} from './wsl-orcad-bridge-activity' import { pairedRuntimeConnectionFromOffer } from '../../shared/paired-runtime-connection' import { parsePairingCode } from '../../shared/pairing' import { @@ -38,6 +42,7 @@ function collectWslOrcadCensus(connection: WslOrcadConnection): Promise boolean }): WslOrcadLivenessSupervisor { @@ -52,9 +57,14 @@ export function createWslOrcadLivenessSupervisor(options: { : readConfirmedRunningWslDistros(), readIdentity: (distro, orcadPid) => readWslGuestIdentity(transportFor(distro), orcadPid), openWatch: (connection) => - openWslOrcadWatch( - transportFor(connection.distro, connection.environmentId), - connection.remotePort + probeThenWatchWslOrcadBridges( + () => + openWslOrcadWatch( + transportFor(connection.distro, connection.environmentId), + connection.remotePort + ), + options.bridgeActivity, + connection.environmentId ), collectCensus: collectWslOrcadCensus, hasUserVisibleWork: options.hasUserVisibleWork, diff --git a/src/main/wsl/wsl-orcad-real-distro-lane.test.ts b/src/main/wsl/wsl-orcad-real-distro-lane.test.ts index 007236c91a6..4aedfef174b 100644 --- a/src/main/wsl/wsl-orcad-real-distro-lane.test.ts +++ b/src/main/wsl/wsl-orcad-real-distro-lane.test.ts @@ -24,6 +24,9 @@ import { execCommand } from '../ssh/remote-exec/remote-exec-command' import { readConfirmedRunningWslDistros } from '../wsl-background-distro-gate' import { readWindowsProcessTableFresh } from '../windows/windows-process-table' import { resolveWslInteropSpawnCwd } from '../wsl-interop-spawn-directory' +import { RemoteRuntimeSharedControlConnection } from '../../shared/remote-runtime/control/remote-runtime-shared-control-connection' +import { WslOrcadBridgeActivity } from './wsl-orcad-bridge-activity' +import { listenWslOrcadStdioBridge } from './wsl-orcad-stdio-bridge' import { createWslDistroHolder, type WslDistroHolder } from './wsl-distro-holder' import { proveWslStructuredChat } from './wsl-orcad-structured-chat-contract' import { WSL_CHAT_GUEST_BIN } from './wsl-orcad-chat-provider-fixture' @@ -79,6 +82,7 @@ receipt.wslInterop = interopChecks const statuses = new Map() let visibleWork = true let holder: WslDistroHolder +const bridgeActivity = new WslOrcadBridgeActivity() let manager: WslOrcadConnectionManager let supervisor: WslOrcadLivenessSupervisor | null = null // Per distro key; unset deploys the packaged template (ORCA_ORCAD_TEMPLATE_PATH). @@ -254,6 +258,11 @@ function createLaneManager(): void { }) manager = new WslOrcadConnectionManager({ holder, + listen: (transport, listenOptions) => + listenWslOrcadStdioBridge(transport, { + ...listenOptions, + events: bridgeActivity.events(listenOptions.environmentId) + }), deployAndLaunch: (distro, options) => deployAndLaunchWslOrcad(distro, { ...options, @@ -266,6 +275,29 @@ function createLaneManager(): void { }) } +/** One-shot RPCs open and close their own socket (the server ends it first); none is an exit. */ +async function countWatchFiresAcrossRpcs(connection: WslOrcadConnection, count: number) { + let fires = 0 + let watch = bridgeActivity.watch(connection.environmentId) + const rearm = (): void => { + fires += 1 + watch = bridgeActivity.watch(connection.environmentId) + watch.onExit(rearm) + } + watch.onExit(rearm) + for (let i = 0; i < count; i++) { + await rpc(connection, 'status.get') + } + // Count only once every one-shot bridge has ended; the control socket stays. + await waitFor( + 'one-shot bridges to close', + async () => (bridgeActivity.openSockets(connection.environmentId) === 1 ? true : null), + 10_000 + ) + watch.close() + return fires +} + /** Runs `command` in a terminal and waits for a printf-split token the echoed input never matches. */ async function runInTerminal( connection: WslOrcadConnection, @@ -486,7 +518,16 @@ async function launchPhase(distro: string, systemd: boolean): Promise { expect(await runningDistros()).toContain(wslDistroHolderKey(distro)) result.rpc = await proveRuntimeRpc(connection, distro) - // Watch exit must classify orcad-exited and trigger recovery. + // Watch exit must classify orcad-exited and trigger recovery. The desktop always holds a + // control socket to a connected host; that socket is now what the liveness watch rides. + const control = new RemoteRuntimeSharedControlConnection(pairedConnection(connection), { + clientCapabilities: ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + }) + await control.request('status.get', undefined, 30_000) + result.watchFiresOnNormalCloses = await countWatchFiresAcrossRpcs(connection, 10) + expect(result.watchFiresOnNormalCloses, 'normal socket closes must not read as orcad exit').toBe( + 0 + ) let since = statusLog(distro).length started = performance.now() await guest(distro, `kill -9 ${connection.pid}; true`) @@ -495,6 +536,7 @@ async function launchPhase(distro: string, systemd: boolean): Promise { result.killDetectMs = Math.round(performance.now() - started) const ready = await waitForStatus(distro, since, 'ready', 180_000) result.killRecoverMs = Math.round(performance.now() - started) + control.close() expect(ready.census).not.toBeNull() result.census = ready.census const relaunched = manager.get(distro) @@ -640,6 +682,7 @@ describe.runIf(RUN && process.platform === 'win32')('WSL orcad on real WSL2 dist async () => { supervisor = createWslOrcadLivenessSupervisor({ manager, + bridgeActivity, holder, hasUserVisibleWork: () => visibleWork }) diff --git a/src/main/wsl/wsl-orcad-stdio-bridge.ts b/src/main/wsl/wsl-orcad-stdio-bridge.ts index 9f2d0a813e1..7ee00319e67 100644 --- a/src/main/wsl/wsl-orcad-stdio-bridge.ts +++ b/src/main/wsl/wsl-orcad-stdio-bridge.ts @@ -1,4 +1,8 @@ /** D2: each accepted loopback socket owns a pinned-Node wsl.exe stdio bridge. Avoid WSL localhost forwarding, whose startup refusal window and network-mode dependence make it unreliable. */ +import { + observeWslOrcadBridgeChannel, + type WslOrcadBridgeChannelEvents +} from './wsl-orcad-bridge-activity' import type { OrcadHostTransport } from '../ssh/orcad-host-transport' import { ORCAD_STDIO_BRIDGE_DEPENDENCIES, @@ -14,6 +18,8 @@ export type WslOrcadStdioBridgeOptions = { /** 0 picks a free one. */ localPort: number label: string + /** Each bridged socket's lifetime, for the liveness watch that rides the desktop's sockets. */ + events?: WslOrcadBridgeChannelEvents } export function listenWslOrcadStdioBridge( @@ -21,6 +27,7 @@ export function listenWslOrcadStdioBridge( options: WslOrcadStdioBridgeOptions, dependencies: OrcadStdioBridgeDependencies = ORCAD_STDIO_BRIDGE_DEPENDENCIES ): Promise { + const { events } = options return startOrcadStdioBridge( transport, { @@ -32,7 +39,13 @@ export function listenWslOrcadStdioBridge( remotePort: options.remotePort, label: options.label }, - dependencies + events + ? { + ...dependencies, + openChannel: async (conn, bridge) => + observeWslOrcadBridgeChannel(await dependencies.openChannel(conn, bridge), events) + } + : dependencies ) } diff --git a/src/main/wsl/wsl-orcad-watch.ts b/src/main/wsl/wsl-orcad-watch.ts index b817b3d1cd3..351c14dcae3 100644 --- a/src/main/wsl/wsl-orcad-watch.ts +++ b/src/main/wsl/wsl-orcad-watch.ts @@ -13,12 +13,16 @@ export type WslOrcadWatch = { /** Called once, with the bridge's exit code, when a `ready` watch ends on its own. */ onExit(listener: (code: number | null) => void): void close(): void + /** Whether it currently proves orcad answers on its port (false while it holds no open path). */ + proving(): boolean + /** Called each time it stops proving without an exit, so the next check can come sooner. */ + onIdle?(listener: () => void): void } const WATCH_OPEN_TIMEOUT_MS = 10_000 function inertWatch(signal: 'refused' | 'unverifiable'): WslOrcadWatch { - return { signal, onExit: () => {}, close: () => {} } + return { signal, onExit: () => {}, close: () => {}, proving: () => false } } export async function openWslOrcadWatch( @@ -86,7 +90,8 @@ export function watchChannel( listener(exitCode) } }, - close + close, + proving: () => !ended && !closedByUs }) } decoder.on('signal', (signal: OrcadStdioBridgeSignal) => settle(signal))