Keep managed servers unverifiable until they answer (#26637)

* Require a server reply before treating a live managed process as serving

* Log the unanswered live managed server like the other unverifiable verdicts

---------

Co-authored-by: m4air <m4air@Mac.localdomain>
Co-authored-by: OrcaWin <alpha-eng@stably.ai>

config/scripts/orcad-e2e-routing.test.mjs

@@ -63,6 +63,7 @@ it('routes the idle-exit spec from its idle sources and the shared convert harne
     [
       'src/shared/orcad-idle-exit.ts',
       'src/main/ssh/orcad-remote-runtime-control.ts',
+      'src/main/ssh/orcad-managed-serving.ts',
       'tests/e2e/helpers/orcad-convert-flow.ts',
       'tests/e2e/helpers/orcad-convert-host.ts'
     ],

config/scripts/pr-e2e-source-routing.mjs

@@ -127,7 +127,7 @@ export const PR_E2E_SOURCE_ROUTES = [
     matches: (file) =>
       /^tests\/e2e\/helpers\/orcad-convert-(?:flow|host)\.ts$/.test(file) ||
       (isProductSource(file) &&
-        /^src\/(?:main\/(?:orcad\/orcad-(?:idle-|managed-idle-)|ssh\/orcad-(?:managed-wake|managed-tunnel|recovery-slot|remote-launch|remote-runtime-control))|shared\/orcad-idle-exit)/.test(
+        /^src\/(?:main\/(?:orcad\/orcad-(?:idle-|managed-idle-)|ssh\/orcad-(?:managed-serving|managed-wake|managed-tunnel|recovery-slot|remote-launch|remote-runtime-control))|shared\/orcad-idle-exit)/.test(
           file
         ))
   },

src/main/ssh/orcad-managed-serving.test.ts

@@ -84,7 +84,29 @@ describe('ensureManagedOrcadServing', () => {
 
   it('leaves a live process that did not answer alone', async () => {
     vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' })
-    expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ state: 'serving' })
+    const serving = await ensureManagedOrcadServing(input(async () => false))
+    expect(serving).toMatchObject({ state: 'unverifiable' })
+    expect(listener.starting).not.toHaveBeenCalled()
+  })
+
+  it('accepts a live process once a fresh probe answers after the host check', async () => {
+    vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' })
+    const probe = vi.fn(async () => true).mockResolvedValueOnce(false)
+    expect(await ensureManagedOrcadServing(input(probe))).toEqual({ state: 'serving' })
+    expect(probe).toHaveBeenCalledTimes(2)
+    expect(listener.starting).not.toHaveBeenCalled()
+  })
+
+  it('keeps a failed fresh probe unverifiable without starting another process', async () => {
+    vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' })
+    const probe = vi
+      .fn(async () => false)
+      .mockResolvedValueOnce(false)
+      .mockRejectedValueOnce(new Error('SSH link was replaced'))
+    expect(await ensureManagedOrcadServing(input(probe))).toEqual({
+      state: 'unverifiable',
+      detail: 'SSH link was replaced'
+    })
     expect(listener.starting).not.toHaveBeenCalled()
   })
 

src/main/ssh/orcad-managed-serving.ts

@@ -137,7 +137,13 @@ async function wakeIfStopped(
     }
     // A live process that did not answer may still be starting; it is not restarted.
     if (wake.outcome === 'serving') {
-      return { state: 'serving' }
+      // Why: a live PID proves the process exists, not that it answers.
+      if (await input.probe(input.environment, PROBE_TIMEOUT_MS)) {
+        return { state: 'serving' }
+      }
+      const detail = 'The managed Orca server process is live but is not answering.'
+      console.warn(`[ssh] The managed Orca server on ${label} is not answering: ${detail}`)
+      return { state: 'unverifiable', detail }
     }
     const detail = wakeRefusal(wake.outcome)
     console.warn(`[ssh] The managed Orca server on ${label} is not answering: ${detail}`)

tests/e2e/ssh-orcad-idle-exit.spec.ts

@@ -174,6 +174,66 @@ function processAlive(target: DockerSshRelayTarget, marker: string): boolean {
   return found.trim() === 'yes'
 }
 
+test('a live managed process that does not answer remains unverifiable', async (// oxlint-disable-next-line no-empty-pattern -- This test owns its isolated app and host.
+{}, testInfo) => {
+  test.skip(
+    HOST !== 'docker' || !TEMPLATE_SOURCE,
+    'Requires the owned Docker SSH host and template'
+  )
+  test.setTimeout(5 * 60_000)
+  const target = startDockerSshRelayTarget(testInfo)
+  const session = createRestartSession(testInfo, { ORCA_ORCAD_TEMPLATE_PATH: TEMPLATE_SOURCE! })
+  let app: ElectronApplication | null = null
+  let pausedPid: string | null = null
+  try {
+    const launched = await session.launch()
+    app = launched.app
+    const page = launched.page
+    await waitForSessionReady(page)
+    const { targetId, environmentId } = await connectManagedHost(page, target)
+    await page.evaluate((targetId) => window.api.ssh.disconnect({ targetId }), targetId)
+    const pids = runningOrcadPids(target)
+    expect(pids).toHaveLength(1)
+    const pid = pids[0]
+    expect(pid).toMatch(/^\d+$/)
+    execDockerSshRelayTargetCommand(target, `kill -STOP ${pid}`)
+    pausedPid = pid
+    const held = await page.evaluate((targetId) => window.api.ssh.connect({ targetId }), targetId)
+    console.log(
+      '[managed-live-process] held',
+      JSON.stringify({ held, pids: runningOrcadPids(target) })
+    )
+    expect(runningOrcadPids(target)).toEqual([pid])
+    expect(held?.managedServer).toMatchObject({
+      kind: 'managed',
+      environmentId,
+      serving: { state: 'unverifiable' }
+    })
+    execDockerSshRelayTargetCommand(target, `kill -CONT ${pid}`)
+    pausedPid = null
+    const recovered = JSON.parse(await reconnect(page, targetId))
+    console.log(
+      '[managed-live-process] recovered',
+      JSON.stringify({ recovered, pids: runningOrcadPids(target) })
+    )
+    expect(recovered).toMatchObject({ kind: 'managed', environmentId })
+    expect(recovered).not.toHaveProperty('serving')
+    expect(await callEnvironment(page, environmentId, 'terminal.list', {})).toMatchObject({
+      ok: true
+    })
+    expect(runningOrcadPids(target)).toEqual([pid])
+  } finally {
+    if (pausedPid) {
+      execDockerSshRelayTargetCommand(target, `kill -CONT ${pausedPid}`)
+    }
+    if (app) {
+      await session.close(app)
+    }
+    await session.dispose()
+    cleanupDockerSshRelayTarget(target)
+  }
+})
+
 test('a killed managed orcad comes back with its terminal, and starts fresh after a reboot', async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set.
 {}, testInfo) => {
   test.skip(