diff --git a/config/scripts/run-ssh-docker-perf-e2e.mjs b/config/scripts/run-ssh-docker-perf-e2e.mjs new file mode 100644 index 00000000000..afa880d8cec --- /dev/null +++ b/config/scripts/run-ssh-docker-perf-e2e.mjs @@ -0,0 +1,39 @@ +import { spawnSync } from 'node:child_process' + +const extraArgs = process.argv.slice(2) +const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], { + stdio: 'inherit', + env +}) + +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + pnpm, + [ + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-docker-relay-perf.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + { + stdio: 'inherit', + env + } +) + +process.exit(result.status ?? 1) diff --git a/package.json b/package.json index 785c40af2eb..090526b34aa 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "build:linux": "pnpm run build:desktop && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --linux", "test:e2e": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headless", "test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2", + "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs", "test:e2e:headful": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headful", "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts" }, diff --git a/tests/e2e/global-setup.ts b/tests/e2e/global-setup.ts index d37927bf988..ef029fa79aa 100644 --- a/tests/e2e/global-setup.ts +++ b/tests/e2e/global-setup.ts @@ -41,11 +41,11 @@ export default function globalSetup(): void { }) console.log('[e2e] Build complete.') } - if (process.env.ORCA_E2E_SSH_LOCALHOST === '1') { - // Why: the localhost SSH spec deploys Orca's relay from out/relay. The + if (process.env.ORCA_E2E_SSH_LOCALHOST === '1' || process.env.ORCA_E2E_SSH_DOCKER === '1') { + // Why: the SSH specs deploy Orca's relay from out/relay. The // normal Electron E2E build does not produce that bundle, so build it only - // for the explicit local-machine SSH run. - console.log('[e2e] Building SSH relay bundle for localhost SSH E2E...') + // for explicit SSH runs. + console.log('[e2e] Building SSH relay bundle for SSH E2E...') execSync('pnpm run build:relay', { cwd: root, stdio: 'inherit', diff --git a/tests/e2e/helpers/docker-ssh-relay-target.ts b/tests/e2e/helpers/docker-ssh-relay-target.ts new file mode 100644 index 00000000000..403588d1850 --- /dev/null +++ b/tests/e2e/helpers/docker-ssh-relay-target.ts @@ -0,0 +1,152 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import type { TestInfo } from '@stablyai/playwright-test' + +export const DOCKER_SSH_RELAY_REMOTE_REPO_PATH = '/tmp/orca-docker-relay-perf-repo' + +export type DockerSshRelayTarget = { + containerName: string + identityFile: string + port: number + tempDir: string +} + +const CONTAINER_IMAGE = process.env.ORCA_E2E_SSH_DOCKER_IMAGE ?? 'node:22-bookworm' + +function run(command: string, args: string[], opts: { timeoutMs?: number } = {}): string { + return execFileSync(command, args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: opts.timeoutMs ?? 30_000 + }).trim() +} + +function tryRun(command: string, args: string[], opts: { timeoutMs?: number } = {}): void { + spawnSync(command, args, { stdio: 'ignore', timeout: opts.timeoutMs ?? 10_000 }) +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} + +function dockerExec(target: DockerSshRelayTarget, command: string): string { + return run('docker', ['exec', target.containerName, 'bash', '-lc', command], { + timeoutMs: 60_000 + }) +} + +function sshArgs(target: DockerSshRelayTarget, command: string): string[] { + return [ + '-i', + target.identityFile, + '-p', + String(target.port), + '-o', + 'StrictHostKeyChecking=no', + '-o', + 'UserKnownHostsFile=/dev/null', + '-o', + 'BatchMode=yes', + 'root@127.0.0.1', + command + ] +} + +function waitForSsh(target: DockerSshRelayTarget): void { + const deadline = Date.now() + 90_000 + let lastError = '' + while (Date.now() < deadline) { + const result = spawnSync('ssh', sshArgs(target, 'true'), { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5_000 + }) + if (result.status === 0) { + return + } + lastError = result.stderr || result.stdout || `exit ${result.status}` + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000) + } + throw new Error(`Timed out waiting for Docker SSH target: ${lastError}`) +} + +function seedRemoteRepo(target: DockerSshRelayTarget): void { + dockerExec( + target, + [ + `rm -rf ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`, + `mkdir -p ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`, + `cd ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`, + 'git init', + 'git config user.email e2e@test.local', + 'git config user.name "Orca Docker SSH E2E"', + 'printf "remote relay perf\\n" > README.md', + 'git add README.md', + 'git commit -m initial' + ].join(' && ') + ) +} + +export function startDockerSshRelayTarget(testInfo: TestInfo): DockerSshRelayTarget { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'orca-ssh-docker-')) + const identityFile = path.join(tempDir, 'id_ed25519') + run('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', identityFile, '-q']) + const publicKey = readFileSync(`${identityFile}.pub`, 'utf8').trim() + const containerName = `orca-ssh-e2e-${testInfo.workerIndex}-${Date.now()}` + let target: DockerSshRelayTarget | null = null + + try { + tryRun('docker', ['rm', '-f', containerName]) + run( + 'docker', + [ + 'run', + '-d', + '--name', + containerName, + '-p', + '127.0.0.1::22', + '-e', + `AUTHORIZED_KEY=${publicKey}`, + CONTAINER_IMAGE, + 'bash', + '-lc', + [ + 'apt-get update >/tmp/apt-update.log', + 'DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server git >/tmp/apt-install.log', + 'mkdir -p /run/sshd /root/.ssh', + 'chmod 700 /root/.ssh', + 'printf "%s\\n" "$AUTHORIZED_KEY" > /root/.ssh/authorized_keys', + 'chmod 600 /root/.ssh/authorized_keys', + 'git config --global user.email e2e@test.local', + 'git config --global user.name "Orca Docker SSH E2E"', + 'exec /usr/sbin/sshd -D -e' + ].join(' && ') + ], + { timeoutMs: 120_000 } + ) + + const port = Number(run('docker', ['port', containerName, '22/tcp']).split(':').at(-1)) + if (!Number.isInteger(port) || port <= 0) { + throw new Error(`Unable to read mapped SSH port for ${containerName}`) + } + target = { containerName, identityFile, port, tempDir } + waitForSsh(target) + seedRemoteRepo(target) + return target + } catch (error) { + cleanupDockerSshRelayTarget(target ?? { containerName, identityFile, port: 0, tempDir }) + throw error + } +} + +export function cleanupDockerSshRelayTarget(target: DockerSshRelayTarget | null): void { + if (!target) { + return + } + tryRun('docker', ['rm', '-f', target.containerName], { timeoutMs: 20_000 }) + rmSync(target.tempDir, { recursive: true, force: true }) +} diff --git a/tests/e2e/helpers/orca-app.ts b/tests/e2e/helpers/orca-app.ts index 9de98d8deae..b68fc00d82d 100644 --- a/tests/e2e/helpers/orca-app.ts +++ b/tests/e2e/helpers/orca-app.ts @@ -244,7 +244,9 @@ export const test = base.extend({ ...cleanEnv, NODE_ENV: 'development', ORCA_E2E_USER_DATA_DIR: userDataDir, - ...(process.env.ORCA_E2E_SSH_LOCALHOST === '1' && !cleanEnv.ORCA_RELAY_PATH + ...((process.env.ORCA_E2E_SSH_LOCALHOST === '1' || + process.env.ORCA_E2E_SSH_DOCKER === '1') && + !cleanEnv.ORCA_RELAY_PATH ? { ORCA_RELAY_PATH: path.join(process.cwd(), 'out', 'relay') } : {}), ...(headful ? { ORCA_E2E_HEADFUL: '1' } : { ORCA_E2E_HEADLESS: '1' }) diff --git a/tests/e2e/helpers/terminal.ts b/tests/e2e/helpers/terminal.ts index c0444e64c34..794aefb2ef4 100644 --- a/tests/e2e/helpers/terminal.ts +++ b/tests/e2e/helpers/terminal.ts @@ -532,10 +532,11 @@ export async function countVisibleTerminalPanes(page: Page): Promise { export async function waitForTerminalOutput( page: Page, expected: string, - timeoutMs = 10_000 + timeoutMs = 10_000, + charLimit = 4000 ): Promise { await expect - .poll(async () => (await getTerminalContent(page)).includes(expected), { + .poll(async () => (await getTerminalContent(page, charLimit)).includes(expected), { timeout: timeoutMs, message: `Terminal did not contain "${expected}"` }) diff --git a/tests/e2e/ssh-docker-relay-perf.spec.ts b/tests/e2e/ssh-docker-relay-perf.spec.ts new file mode 100644 index 00000000000..e3dd37395e8 --- /dev/null +++ b/tests/e2e/ssh-docker-relay-perf.spec.ts @@ -0,0 +1,180 @@ +import type { Page } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + execInTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { + cleanupDockerSshRelayTarget, + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +const KEY_LATENCY_SAMPLES = 'abcdefghij' +const MAX_MEDIAN_KEY_LATENCY_MS = 500 +const MAX_WORST_KEY_LATENCY_MS = 2_000 + +type TypingMeasurement = { + latencies: number[] + medianLatencyMs: number + worstLatencyMs: number +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} + +function remoteTypingLoadScript(runId: string): string { + return [ + "process.stdin.setEncoding('utf8')", + 'if (process.stdin.isTTY) process.stdin.setRawMode(true)', + 'process.stdin.resume()', + 'let seq = 0', + 'let frame = 0', + 'let bg = null', + `process.stdout.write('REMOTE_TUI_READY_${runId}\\n')`, + "setTimeout(() => { bg = setInterval(() => { frame += 1; process.stdout.write('BG_' + frame + '_' + 'x'.repeat(4096) + '\\n') }, 8) }, 500)", + "process.stdin.on('data', (chunk) => {", + ' if (chunk.includes(String.fromCharCode(3))) { if (bg) clearInterval(bg); process.exit(0) }', + ' for (const char of chunk) {', + " if (char === '\\r' || char === '\\n') continue", + ' seq += 1', + ` process.stdout.write('\\x1b[20;2HREMOTE_KEY_${runId}_' + seq + '_' + char + '\\n')`, + ' }', + '})' + ].join(';') +} + +async function connectDockerRemote(page: Page, target: DockerSshRelayTarget): Promise { + await page.evaluate( + async ({ target, remotePath }) => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable') + } + const credentialUnsub = window.api.ssh.onCredentialRequest((request) => { + void window.api.ssh.submitCredential({ requestId: request.requestId, value: null }) + }) + try { + const createdTarget = await window.api.ssh.addTarget({ + target: { + label: `Docker SSH Relay Perf ${Date.now()}`, + host: '127.0.0.1', + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + }) + const state = await window.api.ssh.connect({ targetId: createdTarget.id }) + if (!state || state.status !== 'connected') { + throw new Error(`SSH target did not connect: ${JSON.stringify(state)}`) + } + store.getState().setSshConnectionState(createdTarget.id, state) + const labels = new Map(store.getState().sshTargetLabels) + labels.set(createdTarget.id, createdTarget.label) + store.getState().setSshTargetLabels(labels) + + const result = await window.api.repos.addRemote({ + connectionId: createdTarget.id, + remotePath, + displayName: 'Docker SSH Relay Perf' + }) + if ('error' in result) { + throw new Error(result.error) + } + await store.getState().fetchRepos() + await store.getState().fetchWorktrees(result.repo.id) + const worktree = (store.getState().worktreesByRepo[result.repo.id] ?? [])[0] + if (!worktree) { + throw new Error(`No remote worktree found for ${result.repo.path}`) + } + store.getState().setActiveWorktree(worktree.id) + if ((store.getState().tabsByWorktree[worktree.id] ?? []).length === 0) { + store.getState().createTab(worktree.id) + } + store.getState().setActiveTabType('terminal') + } finally { + credentialUnsub() + } + }, + { target, remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH } + ) +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b) + return sorted[Math.floor(sorted.length / 2)] ?? 0 +} + +async function measureRemoteTyping( + page: Page, + ptyId: string, + runId: string +): Promise { + const latencies: number[] = [] + for (let index = 0; index < KEY_LATENCY_SAMPLES.length; index += 1) { + const char = KEY_LATENCY_SAMPLES[index] + const marker = `REMOTE_KEY_${runId}_${index + 1}_${char}` + const started = performance.now() + await page.evaluate(({ ptyId, char }) => window.api.pty.write(ptyId, char), { ptyId, char }) + await waitForTerminalOutput(page, marker, 10_000, 80_000) + latencies.push(performance.now() - started) + } + return { + latencies, + medianLatencyMs: median(latencies), + worstLatencyMs: Math.max(...latencies) + } +} + +async function stopRemoteLoad(page: Page, ptyId: string): Promise { + await page.evaluate((targetPtyId) => window.api.pty.write(targetPtyId, '\x03'), ptyId) +} + +test.describe('Docker SSH relay perf', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH relay perf.') + test.skip(process.platform === 'win32', 'Docker SSH relay perf uses POSIX ssh tooling.') + + test('keeps remote typing responsive while the Linux relay streams TUI output', async ({ + orcaPage + }, testInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + await connectDockerRemote(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + + const runId = String(Date.now()) + await execInTerminal(orcaPage, ptyId, `node -e ${shellQuote(remoteTypingLoadScript(runId))}`) + await waitForTerminalOutput(orcaPage, `REMOTE_TUI_READY_${runId}`, 30_000, 80_000) + const measurement = await measureRemoteTyping(orcaPage, ptyId, runId) + const summary = `median=${measurement.medianLatencyMs.toFixed( + 1 + )}ms worst=${measurement.worstLatencyMs.toFixed(1)}ms samples=${measurement.latencies + .map((value) => value.toFixed(1)) + .join(',')}` + console.log(`[docker-ssh-relay-perf] ${summary}`) + testInfo.annotations.push({ + type: 'docker-ssh-relay-typing', + description: summary + }) + expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS) + expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS) + await stopRemoteLoad(orcaPage, ptyId) + } finally { + cleanupDockerSshRelayTarget(target) + } + }) +})