test: add docker ssh relay perf guard (#4778)

config/scripts/run-ssh-docker-perf-e2e.mjs

@@ -0,0 +1,39 @@
+import { spawnSync } from 'node:child_process'
+
+const extraArgs = process.argv.slice(2)
+const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'
+const env = {
+  ...process.env,
+  ORCA_E2E_SSH_DOCKER: '1'
+}
+
+const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], {
+  stdio: 'inherit',
+  env
+})
+
+if (runtime.status !== 0) {
+  process.exit(runtime.status ?? 1)
+}
+
+const result = spawnSync(
+  pnpm,
+  [
+    'exec',
+    'playwright',
+    'test',
+    'tests/e2e/ssh-docker-relay-perf.spec.ts',
+    '--config',
+    'tests/playwright.config.ts',
+    '--project',
+    'electron-headless',
+    '--workers=1',
+    ...extraArgs
+  ],
+  {
+    stdio: 'inherit',
+    env
+  }
+)
+
+process.exit(result.status ?? 1)

package.json

@@ -61,6 +61,7 @@
     "build:linux": "pnpm run build:desktop && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --linux",
     "test:e2e": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headless",
     "test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
+    "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs",
     "test:e2e:headful": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headful",
     "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts"
   },

tests/e2e/global-setup.ts

@@ -41,11 +41,11 @@ export default function globalSetup(): void {
     })
     console.log('[e2e] Build complete.')
   }
-  if (process.env.ORCA_E2E_SSH_LOCALHOST === '1') {
-    // Why: the localhost SSH spec deploys Orca's relay from out/relay. The
+  if (process.env.ORCA_E2E_SSH_LOCALHOST === '1' || process.env.ORCA_E2E_SSH_DOCKER === '1') {
+    // Why: the SSH specs deploy Orca's relay from out/relay. The
     // normal Electron E2E build does not produce that bundle, so build it only
-    // for the explicit local-machine SSH run.
-    console.log('[e2e] Building SSH relay bundle for localhost SSH E2E...')
+    // for explicit SSH runs.
+    console.log('[e2e] Building SSH relay bundle for SSH E2E...')
     execSync('pnpm run build:relay', {
       cwd: root,
       stdio: 'inherit',

tests/e2e/helpers/docker-ssh-relay-target.ts

@@ -0,0 +1,152 @@
+import { execFileSync, spawnSync } from 'node:child_process'
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import type { TestInfo } from '@stablyai/playwright-test'
+
+export const DOCKER_SSH_RELAY_REMOTE_REPO_PATH = '/tmp/orca-docker-relay-perf-repo'
+
+export type DockerSshRelayTarget = {
+  containerName: string
+  identityFile: string
+  port: number
+  tempDir: string
+}
+
+const CONTAINER_IMAGE = process.env.ORCA_E2E_SSH_DOCKER_IMAGE ?? 'node:22-bookworm'
+
+function run(command: string, args: string[], opts: { timeoutMs?: number } = {}): string {
+  return execFileSync(command, args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    timeout: opts.timeoutMs ?? 30_000
+  }).trim()
+}
+
+function tryRun(command: string, args: string[], opts: { timeoutMs?: number } = {}): void {
+  spawnSync(command, args, { stdio: 'ignore', timeout: opts.timeoutMs ?? 10_000 })
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replaceAll("'", "'\\''")}'`
+}
+
+function dockerExec(target: DockerSshRelayTarget, command: string): string {
+  return run('docker', ['exec', target.containerName, 'bash', '-lc', command], {
+    timeoutMs: 60_000
+  })
+}
+
+function sshArgs(target: DockerSshRelayTarget, command: string): string[] {
+  return [
+    '-i',
+    target.identityFile,
+    '-p',
+    String(target.port),
+    '-o',
+    'StrictHostKeyChecking=no',
+    '-o',
+    'UserKnownHostsFile=/dev/null',
+    '-o',
+    'BatchMode=yes',
+    'root@127.0.0.1',
+    command
+  ]
+}
+
+function waitForSsh(target: DockerSshRelayTarget): void {
+  const deadline = Date.now() + 90_000
+  let lastError = ''
+  while (Date.now() < deadline) {
+    const result = spawnSync('ssh', sshArgs(target, 'true'), {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      timeout: 5_000
+    })
+    if (result.status === 0) {
+      return
+    }
+    lastError = result.stderr || result.stdout || `exit ${result.status}`
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000)
+  }
+  throw new Error(`Timed out waiting for Docker SSH target: ${lastError}`)
+}
+
+function seedRemoteRepo(target: DockerSshRelayTarget): void {
+  dockerExec(
+    target,
+    [
+      `rm -rf ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`,
+      `mkdir -p ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`,
+      `cd ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`,
+      'git init',
+      'git config user.email e2e@test.local',
+      'git config user.name "Orca Docker SSH E2E"',
+      'printf "remote relay perf\\n" > README.md',
+      'git add README.md',
+      'git commit -m initial'
+    ].join(' && ')
+  )
+}
+
+export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTarget {
+  const tempDir = mkdtempSync(path.join(os.tmpdir(), 'orca-ssh-docker-'))
+  const identityFile = path.join(tempDir, 'id_ed25519')
+  run('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', identityFile, '-q'])
+  const publicKey = readFileSync(`${identityFile}.pub`, 'utf8').trim()
+  const containerName = `orca-ssh-e2e-${testInfo.workerIndex}-${Date.now()}`
+  let target: DockerSshRelayTarget | null = null
+
+  try {
+    tryRun('docker', ['rm', '-f', containerName])
+    run(
+      'docker',
+      [
+        'run',
+        '-d',
+        '--name',
+        containerName,
+        '-p',
+        '127.0.0.1::22',
+        '-e',
+        `AUTHORIZED_KEY=${publicKey}`,
+        CONTAINER_IMAGE,
+        'bash',
+        '-lc',
+        [
+          'apt-get update >/tmp/apt-update.log',
+          'DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server git >/tmp/apt-install.log',
+          'mkdir -p /run/sshd /root/.ssh',
+          'chmod 700 /root/.ssh',
+          'printf "%s\\n" "$AUTHORIZED_KEY" > /root/.ssh/authorized_keys',
+          'chmod 600 /root/.ssh/authorized_keys',
+          'git config --global user.email e2e@test.local',
+          'git config --global user.name "Orca Docker SSH E2E"',
+          'exec /usr/sbin/sshd -D -e'
+        ].join(' && ')
+      ],
+      { timeoutMs: 120_000 }
+    )
+
+    const port = Number(run('docker', ['port', containerName, '22/tcp']).split(':').at(-1))
+    if (!Number.isInteger(port) || port <= 0) {
+      throw new Error(`Unable to read mapped SSH port for ${containerName}`)
+    }
+    target = { containerName, identityFile, port, tempDir }
+    waitForSsh(target)
+    seedRemoteRepo(target)
+    return target
+  } catch (error) {
+    cleanupDockerSshRelayTarget(target ?? { containerName, identityFile, port: 0, tempDir })
+    throw error
+  }
+}
+
+export function cleanupDockerSshRelayTarget(target: DockerSshRelayTarget | null): void {
+  if (!target) {
+    return
+  }
+  tryRun('docker', ['rm', '-f', target.containerName], { timeoutMs: 20_000 })
+  rmSync(target.tempDir, { recursive: true, force: true })
+}

tests/e2e/helpers/orca-app.ts

@@ -244,7 +244,9 @@ export const test = base.extend<OrcaTestFixtures, OrcaWorkerFixtures>({
         ...cleanEnv,
         NODE_ENV: 'development',
         ORCA_E2E_USER_DATA_DIR: userDataDir,
-        ...(process.env.ORCA_E2E_SSH_LOCALHOST === '1' && !cleanEnv.ORCA_RELAY_PATH
+        ...((process.env.ORCA_E2E_SSH_LOCALHOST === '1' ||
+          process.env.ORCA_E2E_SSH_DOCKER === '1') &&
+        !cleanEnv.ORCA_RELAY_PATH
           ? { ORCA_RELAY_PATH: path.join(process.cwd(), 'out', 'relay') }
           : {}),
         ...(headful ? { ORCA_E2E_HEADFUL: '1' } : { ORCA_E2E_HEADLESS: '1' })

tests/e2e/helpers/terminal.ts

@@ -532,10 +532,11 @@ export async function countVisibleTerminalPanes(page: Page): Promise<number> {
 export async function waitForTerminalOutput(
   page: Page,
   expected: string,
-  timeoutMs = 10_000
+  timeoutMs = 10_000,
+  charLimit = 4000
 ): Promise<void> {
   await expect
-    .poll(async () => (await getTerminalContent(page)).includes(expected), {
+    .poll(async () => (await getTerminalContent(page, charLimit)).includes(expected), {
       timeout: timeoutMs,
       message: `Terminal did not contain "${expected}"`
     })

tests/e2e/ssh-docker-relay-perf.spec.ts

@@ -0,0 +1,180 @@
+import type { Page } from '@stablyai/playwright-test'
+import { test, expect } from './helpers/orca-app'
+import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
+import {
+  execInTerminal,
+  waitForActivePanePtyId,
+  waitForActiveTerminalManager,
+  waitForTerminalOutput
+} from './helpers/terminal'
+import {
+  cleanupDockerSshRelayTarget,
+  DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
+  startDockerSshRelayTarget,
+  type DockerSshRelayTarget
+} from './helpers/docker-ssh-relay-target'
+
+const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
+const KEY_LATENCY_SAMPLES = 'abcdefghij'
+const MAX_MEDIAN_KEY_LATENCY_MS = 500
+const MAX_WORST_KEY_LATENCY_MS = 2_000
+
+type TypingMeasurement = {
+  latencies: number[]
+  medianLatencyMs: number
+  worstLatencyMs: number
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replaceAll("'", "'\\''")}'`
+}
+
+function remoteTypingLoadScript(runId: string): string {
+  return [
+    "process.stdin.setEncoding('utf8')",
+    'if (process.stdin.isTTY) process.stdin.setRawMode(true)',
+    'process.stdin.resume()',
+    'let seq = 0',
+    'let frame = 0',
+    'let bg = null',
+    `process.stdout.write('REMOTE_TUI_READY_${runId}\\n')`,
+    "setTimeout(() => { bg = setInterval(() => { frame += 1; process.stdout.write('BG_' + frame + '_' + 'x'.repeat(4096) + '\\n') }, 8) }, 500)",
+    "process.stdin.on('data', (chunk) => {",
+    '  if (chunk.includes(String.fromCharCode(3))) { if (bg) clearInterval(bg); process.exit(0) }',
+    '  for (const char of chunk) {',
+    "    if (char === '\\r' || char === '\\n') continue",
+    '    seq += 1',
+    `    process.stdout.write('\\x1b[20;2HREMOTE_KEY_${runId}_' + seq + '_' + char + '\\n')`,
+    '  }',
+    '})'
+  ].join(';')
+}
+
+async function connectDockerRemote(page: Page, target: DockerSshRelayTarget): Promise<void> {
+  await page.evaluate(
+    async ({ target, remotePath }) => {
+      const store = window.__store
+      if (!store) {
+        throw new Error('Store unavailable')
+      }
+      const credentialUnsub = window.api.ssh.onCredentialRequest((request) => {
+        void window.api.ssh.submitCredential({ requestId: request.requestId, value: null })
+      })
+      try {
+        const createdTarget = await window.api.ssh.addTarget({
+          target: {
+            label: `Docker SSH Relay Perf ${Date.now()}`,
+            host: '127.0.0.1',
+            port: target.port,
+            username: 'root',
+            identityFile: target.identityFile,
+            identitiesOnly: true,
+            relayGracePeriodSeconds: 1
+          }
+        })
+        const state = await window.api.ssh.connect({ targetId: createdTarget.id })
+        if (!state || state.status !== 'connected') {
+          throw new Error(`SSH target did not connect: ${JSON.stringify(state)}`)
+        }
+        store.getState().setSshConnectionState(createdTarget.id, state)
+        const labels = new Map(store.getState().sshTargetLabels)
+        labels.set(createdTarget.id, createdTarget.label)
+        store.getState().setSshTargetLabels(labels)
+
+        const result = await window.api.repos.addRemote({
+          connectionId: createdTarget.id,
+          remotePath,
+          displayName: 'Docker SSH Relay Perf'
+        })
+        if ('error' in result) {
+          throw new Error(result.error)
+        }
+        await store.getState().fetchRepos()
+        await store.getState().fetchWorktrees(result.repo.id)
+        const worktree = (store.getState().worktreesByRepo[result.repo.id] ?? [])[0]
+        if (!worktree) {
+          throw new Error(`No remote worktree found for ${result.repo.path}`)
+        }
+        store.getState().setActiveWorktree(worktree.id)
+        if ((store.getState().tabsByWorktree[worktree.id] ?? []).length === 0) {
+          store.getState().createTab(worktree.id)
+        }
+        store.getState().setActiveTabType('terminal')
+      } finally {
+        credentialUnsub()
+      }
+    },
+    { target, remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH }
+  )
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b)
+  return sorted[Math.floor(sorted.length / 2)] ?? 0
+}
+
+async function measureRemoteTyping(
+  page: Page,
+  ptyId: string,
+  runId: string
+): Promise<TypingMeasurement> {
+  const latencies: number[] = []
+  for (let index = 0; index < KEY_LATENCY_SAMPLES.length; index += 1) {
+    const char = KEY_LATENCY_SAMPLES[index]
+    const marker = `REMOTE_KEY_${runId}_${index + 1}_${char}`
+    const started = performance.now()
+    await page.evaluate(({ ptyId, char }) => window.api.pty.write(ptyId, char), { ptyId, char })
+    await waitForTerminalOutput(page, marker, 10_000, 80_000)
+    latencies.push(performance.now() - started)
+  }
+  return {
+    latencies,
+    medianLatencyMs: median(latencies),
+    worstLatencyMs: Math.max(...latencies)
+  }
+}
+
+async function stopRemoteLoad(page: Page, ptyId: string): Promise<void> {
+  await page.evaluate((targetPtyId) => window.api.pty.write(targetPtyId, '\x03'), ptyId)
+}
+
+test.describe('Docker SSH relay perf', () => {
+  test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH relay perf.')
+  test.skip(process.platform === 'win32', 'Docker SSH relay perf uses POSIX ssh tooling.')
+
+  test('keeps remote typing responsive while the Linux relay streams TUI output', async ({
+    orcaPage
+  }, testInfo) => {
+    test.slow()
+    let target: DockerSshRelayTarget | null = null
+    try {
+      target = startDockerSshRelayTarget(testInfo)
+      await waitForSessionReady(orcaPage)
+      await waitForActiveWorktree(orcaPage)
+      await connectDockerRemote(orcaPage, target)
+      await ensureTerminalVisible(orcaPage, 45_000)
+      await waitForActiveTerminalManager(orcaPage, 60_000)
+      const ptyId = await waitForActivePanePtyId(orcaPage, 60_000)
+
+      const runId = String(Date.now())
+      await execInTerminal(orcaPage, ptyId, `node -e ${shellQuote(remoteTypingLoadScript(runId))}`)
+      await waitForTerminalOutput(orcaPage, `REMOTE_TUI_READY_${runId}`, 30_000, 80_000)
+      const measurement = await measureRemoteTyping(orcaPage, ptyId, runId)
+      const summary = `median=${measurement.medianLatencyMs.toFixed(
+        1
+      )}ms worst=${measurement.worstLatencyMs.toFixed(1)}ms samples=${measurement.latencies
+        .map((value) => value.toFixed(1))
+        .join(',')}`
+      console.log(`[docker-ssh-relay-perf] ${summary}`)
+      testInfo.annotations.push({
+        type: 'docker-ssh-relay-typing',
+        description: summary
+      })
+      expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS)
+      expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS)
+      await stopRemoteLoad(orcaPage, ptyId)
+    } finally {
+      cleanupDockerSshRelayTarget(target)
+    }
+  })
+})