orca/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts

import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import type { ElectronApplication, Page } from '@stablyai/playwright-test'
import type {
  RuntimeMobileSessionTabsResult,
  RuntimeTerminalListResult,
  RuntimeTerminalRead,
  RuntimeTerminalShow
} from '../../src/shared/runtime-types'
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
import { expect, test } from './helpers/orca-app'
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
import {
  createRuntimeDesktopPairingOffer,
  launchPairedElectronClient
} from './helpers/paired-electron-client'
import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal'

const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-materialize-'))
const fixturePath = path.join(scratch, 'materialize-terminal.mjs')
const processedInputPath = path.join(scratch, 'processed-input.txt')

writeFileSync(
  fixturePath,
  [
    "import { appendFileSync } from 'node:fs'",
    'const processedInputPath = process.argv[2]',
    "process.stdout.write('MATERIALIZE_READY\\r\\n')",
    "process.stdin.setEncoding('utf8')",
    "let pending = ''",
    "process.stdin.on('data', (data) => {",
    '  pending += data',
    '  const commands = pending.split(/\\r\\n|\\r|\\n/)',
    '  pending = commands.pop() ?? ""',
    '  for (const input of commands) {',
    '    appendFileSync(processedInputPath, `${input}\\n`)',
    '    process.stdout.write(`LIVE:${input}\\r\\n`)',
    '  }',
    '})',
    'process.stdin.resume()'
  ].join('\n')
)

test.describe.configure({ mode: 'serial' })

test.afterAll(() => {
  rmSync(scratch, { recursive: true, force: true })
})

function shellQuote(value: string): string {
  return `'${value.replaceAll("'", `'\\''`)}'`
}

function fixtureCommand(): string {
  const command = [process.execPath, fixturePath, processedInputPath]
  return process.platform === 'win32'
    ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
    : command.map(shellQuote).join(' ')
}

async function callRuntime<TResult>(
  page: Page,
  selector: string,
  method: string,
  params: unknown
): Promise<TResult> {
  return page.evaluate(
    async ({ method, params, selector }) => {
      const response = await window.api.runtimeEnvironments.call({ selector, method, params })
      if (!response.ok) {
        throw new Error(`${response.error.code}: ${response.error.message}`)
      }
      return response.result
    },
    { method, params, selector }
  ) as Promise<TResult>
}

async function showClient(app: ElectronApplication, page: Page): Promise<void> {
  const clientWindow = await app.browserWindow(page)
  await clientWindow.evaluate((window) => {
    window.show()
    window.focus()
  })
  await expect.poll(() => clientWindow.evaluate((window) => window.isVisible())).toBe(true)
}

async function waitForClientWorktree(page: Page, expectedId?: string): Promise<string> {
  await expect
    .poll(
      () =>
        page.evaluate(
          (id) =>
            window.__store
              ?.getState()
              .allWorktrees()
              .find((worktree) => !id || worktree.id === id)?.id ?? null,
          expectedId
        ),
      { timeout: 30_000 }
    )
    .not.toBeNull()
  const worktreeId = await page.evaluate(
    (id) =>
      window.__store
        ?.getState()
        .allWorktrees()
        .find((worktree) => !id || worktree.id === id)?.id ?? null,
    expectedId
  )
  if (!worktreeId) {
    throw new Error('Paired client did not receive the host workspace')
  }
  return worktreeId
}

async function hostSurfaceStatus(
  page: Page,
  environmentId: string,
  worktreeId: string,
  parentTabId: string
): Promise<string | null> {
  const snapshot = await callRuntime<RuntimeMobileSessionTabsResult>(
    page,
    environmentId,
    'session.tabs.list',
    { worktree: `id:${worktreeId}` }
  )
  const surface = snapshot.tabs.find(
    (candidate) => candidate.type === 'terminal' && candidate.parentTabId === parentTabId
  )
  return surface?.type === 'terminal' ? surface.status : null
}

/** Park the fixture PTY so the host republishes its pane as a pending handle.
 *  Why: exact stop only confirms when it observes the fixture exit inside its verification
 *  window, and a loaded headless host can miss that window even though the PTY is going away.
 *  The precondition this journey needs is the parked surface, so retry until the host shows it. */
async function parkHostTerminal(
  page: Page,
  environmentId: string,
  worktreeId: string,
  parentTabId: string,
  options: { expectedPtyId: string }
): Promise<void> {
  let lastError = 'terminal.stopExact was never attempted'
  for (let attempt = 0; attempt < 10; attempt += 1) {
    const stop = await callRuntime<{
      stopped: number
      stoppedPtyIds: string[]
      postStopVerified: boolean
    }>(page, environmentId, 'terminal.stopExact', {
      worktree: `id:${worktreeId}`,
      expectedPtyIds: [options.expectedPtyId],
      keepHistory: true,
      targetOnly: true
    }).catch((error: unknown) => {
      lastError = error instanceof Error ? error.message : String(error)
      return null
    })
    if (stop) {
      expect(stop.postStopVerified).toBe(true)
      expect(stop.stopped).toBe(1)
      expect(stop.stoppedPtyIds).toEqual([options.expectedPtyId])
      return
    }
    // Why: the host reports a set mismatch once the target PTY is no longer live, which is the
    // parked state this journey needs even when the stop call itself missed the exit.
    if (
      lastError.includes('terminal_stop_pty_set_mismatch') ||
      (await hostSurfaceStatus(page, environmentId, worktreeId, parentTabId)) !== 'ready'
    ) {
      return
    }
    await page.waitForTimeout(1_000)
  }
  throw new Error(`Host never parked the fixture terminal: ${lastError}`)
}

async function runMaterializationJourney(
  page: Page,
  environmentId: string,
  worktreeId: string
): Promise<void> {
  writeFileSync(processedInputPath, '')
  const created = await callRuntime<{
    tab: { parentTabId: string; terminal: string | null }
  }>(page, environmentId, 'session.tabs.createTerminal', {
    worktree: `id:${worktreeId}`,
    command: fixtureCommand(),
    activate: false,
    select: false,
    navigation: 'caller'
  })
  const originalHandle = created.tab.terminal
  if (!originalHandle) {
    throw new Error('Host did not publish the fixture terminal')
  }

  const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
  await page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId)
  const tab = page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`)
  await expect(tab).toBeVisible({ timeout: 30_000 })
  await tab.click()
  await expect(tab).toHaveAttribute('data-active', 'true')
  const originalClientPtyId = await waitForActivePanePtyId(page, 30_000)
  await expect
    .poll(() => getTerminalContent(page), { timeout: 30_000 })
    .toContain('MATERIALIZE_READY')

  const originalTerminal = await callRuntime<{ terminal: RuntimeTerminalShow }>(
    page,
    environmentId,
    'terminal.show',
    { terminal: originalHandle }
  )
  if (!originalTerminal.terminal.ptyId) {
    throw new Error('Host fixture terminal has no authoritative PTY')
  }
  await page.evaluate((terminal) => {
    const gate = (
      window as typeof window & {
        __remoteTerminalMultiplexAckGate?: { holdEnd: (terminals: string[]) => void }
      }
    ).__remoteTerminalMultiplexAckGate
    if (!gate) {
      throw new Error('Remote terminal fault gate is unavailable')
    }
    gate.holdEnd([terminal])
  }, originalHandle)
  await parkHostTerminal(page, environmentId, worktreeId, created.tab.parentTabId, {
    expectedPtyId: originalTerminal.terminal.ptyId
  })
  // Why: the stale error must land on an already-parked surface, or the journey proves nothing
  // about materializing a pending handle.
  await expect
    .poll(() => hostSurfaceStatus(page, environmentId, worktreeId, created.tab.parentTabId), {
      timeout: 15_000,
      message: 'Host never published the stopped pane as pending-handle'
    })
    .toBe('pending-handle')
  const dispatched = await page.evaluate((terminal) => {
    const gate = (
      window as typeof window & {
        __remoteTerminalMultiplexAckGate?: {
          forceError: (terminals: string[], message: string) => number
          release: () => void
        }
      }
    ).__remoteTerminalMultiplexAckGate
    if (!gate) {
      throw new Error('Remote terminal fault gate is unavailable')
    }
    const dispatched = gate.forceError([terminal], 'terminal_handle_stale')
    gate.release()
    return dispatched
  }, originalHandle)
  expect(dispatched).toBe(1)

  let replacementHandle: string | null = null
  await expect
    .poll(
      async () => {
        const snapshot = await callRuntime<RuntimeMobileSessionTabsResult>(
          page,
          environmentId,
          'session.tabs.list',
          { worktree: `id:${worktreeId}` }
        )
        const surface = snapshot.tabs.find(
          (candidate) =>
            candidate.type === 'terminal' && candidate.parentTabId === created.tab.parentTabId
        )
        replacementHandle = surface?.type === 'terminal' ? surface.terminal : null
        return replacementHandle !== null && replacementHandle !== originalHandle
      },
      { timeout: 20_000, message: 'Reconnect never materialized the sleeping host surface' }
    )
    .toBe(true)
  expect(replacementHandle).not.toBeNull()

  // Why: parking the host PTY can clear this client's active-worktree selection, so reselect the
  // pane before reading the PTY it rebound to — the rebind itself is what this journey asserts.
  await page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId)
  await expect(tab).toBeVisible({ timeout: 10_000 })
  await tab.click()
  await expect(tab).toHaveAttribute('data-active', 'true')
  const replacementClientPtyId = await waitForActivePanePtyId(page, 20_000)
  expect(replacementClientPtyId).not.toBe(originalClientPtyId)

  const marker = `MATERIALIZED_${Date.now()}`
  await callRuntime(page, environmentId, 'terminal.send', {
    terminal: replacementHandle,
    text: `echo ${marker}\r`,
    client: { id: 'paired-materialization-e2e', type: 'desktop' }
  })
  await expect
    .poll(
      async () => {
        const read = await callRuntime<{ terminal: RuntimeTerminalRead }>(
          page,
          environmentId,
          'terminal.read',
          { terminal: replacementHandle }
        )
        return read.terminal.tail.join('\n')
      },
      { timeout: 10_000 }
    )
    .toContain(marker)
  await page.evaluate(async (id) => {
    await window.__store?.getState().setActiveWorktree(id)
  }, worktreeId)
  await expect(tab).toBeVisible({ timeout: 10_000 })
  await tab.click()
  await expect.poll(() => getTerminalContent(page), { timeout: 10_000 }).toContain(marker)

  const listed = await callRuntime<RuntimeTerminalListResult>(
    page,
    environmentId,
    'terminal.list',
    {
      worktree: `id:${worktreeId}`,
      requireFreshPtyLiveness: true
    }
  )
  expect(
    listed.terminals.filter((terminal) => terminal.tabId === created.tab.parentTabId)
  ).toHaveLength(1)
  await callRuntime(page, environmentId, 'terminal.closeTab', { terminal: replacementHandle })
}

test('materializes a stopped terminal on reconnect from a headed paired host', async ({
  orcaPage
}, testInfo) => {
  test.setTimeout(120_000)
  const worktreeId = await orcaPage.evaluate(() => window.__store?.getState().activeWorktreeId)
  if (!worktreeId) {
    throw new Error('Headed host has no active seeded workspace')
  }
  const offer = await createRuntimeDesktopPairingOffer(orcaPage)
  const client = await launchPairedElectronClient(offer, testInfo, 'headed-materialization-client')
  try {
    await showClient(client.app, client.page)
    await runMaterializationJourney(
      client.page,
      client.environmentId,
      await waitForClientWorktree(client.page, worktreeId)
    )
    expect(await client.getDirectSshAttemptTargetIds()).toEqual([])
  } finally {
    await client.dispose()
  }
})

// Why fixme: this journey's fault injection cannot be set up on a headless `orca serve` host.
// `terminal.stopExact` keeps returning terminal_exact_stop_failed because stopAndWait's
// keep-history verification window expires before the parked PTY is observed gone, so the pane
// never reaches pending-handle and the reconnect behavior is never exercised. That precondition
// fails identically on this PR's base, so it is a pre-existing exact-stop defect rather than a
// reconnect-activation one. The recovery behavior itself was confirmed by hand in this topology
// (the host materializes the pending surface and the client rebinds to the replacement PTY);
// re-enable once exact stop settles deterministically against a serve host.
test.fixme('materializes a stopped terminal on reconnect from a headless folder host', async ({
  testRepoPath
}, testInfo) => {
  test.setTimeout(150_000)
  const host = await launchHeadlessPairedRuntimeHost()
  await host.client.call('repo.add', { path: testRepoPath, kind: 'folder' })
  const client = await launchPairedElectronClient(
    host.offer,
    testInfo,
    'headless-folder-materialization-client'
  ).catch(async (error) => {
    await host.dispose()
    throw error
  })
  try {
    await showClient(client.app, client.page)
    await runMaterializationJourney(
      client.page,
      client.environmentId,
      await waitForClientWorktree(client.page)
    )
    expect(await client.getDirectSshAttemptTargetIds()).toEqual([])
  } finally {
    await client.dispose()
    await host.dispose()
  }
})