Files
orca/tests/e2e/ssh-codex-reconnect-replay-driver.ts
T
JinjingandOrca d7fe9d6bcc fix(ai-vault): support session scanning in SSH worktrees (#11004)
* fix(ai-vault): support session scanning in SSH worktrees

Add relay-native aiVault.listSessions scanning that discovers agent
sessions on SSH hosts. Includes fallback to filesystem crawl for
legacy relays, full cancellation support, result validation, and
scan coalescing to reduce redundant work.

* fix(ai-vault): scan sessions in SSH worktrees with coordinated cancellat

- Extract batching logic to `mapRemoteScanBatches` for reuse and proper cancellation checkpoints
- Move `AiVaultScanCoordinator` from relay to main to handle concurrent same-key requests with individual cancellation signals
- Report scope path truncation consistently across relay and SSH fallback paths
- Gracefully degrade relay handler on unsupported platforms instead of aborting startup
- Refactor issue display to separate blocking errors, scope notices, and skipped transcript counts

* fix(ai-vault): stabilize SSH session scan CI

Swallow async WSL relay stdin EPIPE so the live hook-relay shard no longer
fails after all tests pass. Merge main, resolve scan/relay conflicts, and
align cancellation/host-issue reporting with IPC expectations.

* fix(ai-vault): harden session scan cancellation, relay timeouts, and preemption

Thread the abort signal through every scan and parse path so superseded or
cancelled scans stop promptly instead of parsing every remaining transcript
for a caller that already left.  Replace the fragile message-text relay
timeout check with a typed error code so unrelated errors carrying the
phrase "timed out after" no longer suppress the filesystem fallback.  Fix
scan coordinator preemption so a forced Refresh in one window no longer
re-enters as a spurious cancellation in another.  Add a host-leg cache for
the all-hosts view and cap filesystem concurrency so a single slow remote
home cannot stall the whole merge.

Co-authored-by: Orca <help@stably.ai>

* fix(ai-vault): use stable React keys for scan issue banners

Drop array-index keys so react-doctor/no-array-index-as-key passes.
Uniqueness comes from host, kind, agent, path, and message.

* fix(ai-vault): SSH session scanning with configurable depth limits

Implement depth-aware caching and proper scan boundaries to make SSH session
scanning reliable in worktrees. Users can now select between faster (250
sessions) and comprehensive (unlimited) history scans. The scanner:
- Deduplicates scans across relay, host leg, runtime, and renderer layers
- Reuses larger scans to serve smaller depth requests
- Properly bounds in-scope discovery per-limit
- Fixes timeout enforcement when SSH providers ignore abort signals

* Move sessionLimit ref update to useLayoutEffect

Keep render pure for React Doctor by deferring ref updates to
a layout effect, which still executes before render-dependent
effects that consume the ref.

* fix(adhoc): stamp version prefix from main, not the feature branch

Adhoc builds check out arbitrary refs whose package.json often lags
version bumps (e.g. 1.4.165-rc.0 while main is 1.4.168-rc.1). Hourly
always builds main so it already tracks the product line; adhoc now
resolves the base version from origin/main (or ORCA_ADHOC_BASE_VERSION)
so branch builds share that prefix.

* Revert "fix(adhoc): stamp version prefix from main, not the feature branch"

This reverts commit a26a18eb3fd83f7e7d2db9a6a7c3e02e0f79089a.

* fix(ai-vault): fix scoped backfill and coordinator race conditions

Resolve race where the last waiter leaving could abort an already-settled scan (add `settled` flag). Redesign scoped session backfill to keep searching through newer files until the scope reaches its requested session quota instead of stopping at the candidate limit; out-of-scope files no longer consume the scope budget. Centralize scan limit normalization and fix error classification for cancelled scans using the proper helper instead of checking Error.name. Disambiguate cache keys using JSON and add cancellation check after scope discovery phase.

---------

Co-authored-by: Orca <help@stably.ai>
2026-08-03 16:17:00 -07:00

258 lines
8.2 KiB
TypeScript

import { execFileSync } from 'node:child_process'
import type { Page } from '@stablyai/playwright-test'
import { expect } from './helpers/orca-app'
import {
DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
type DockerSshRelayTarget
} from './helpers/docker-ssh-relay-target'
export type ConnectedDockerRemote = {
targetId: string
worktreeId: string
}
export function dropDockerSshClientSessions(target: DockerSshRelayTarget): void {
execFileSync(
'docker',
[
'exec',
target.containerName,
'bash',
'-lc',
`ps -eo pid=,comm=,args= | awk '$2 == "sshd" && index($0, "sshd: root") { print $1 }' | xargs -r kill -9`
],
{ stdio: ['ignore', 'pipe', 'pipe'], timeout: 60_000 }
)
}
export async function connectDockerRemote(
page: Page,
target: DockerSshRelayTarget
): Promise<ConnectedDockerRemote> {
const remote = await page.evaluate(
async ({ target, remotePath }) => {
const store = window.__store
if (!store) {
throw new Error('Store unavailable')
}
const credentialUnsub = window.api.ssh.onCredentialRequest((request) => {
void window.api.ssh.submitCredential({ requestId: request.requestId, value: null })
})
try {
const { target: createdTarget, repoReadoptions } = await window.api.ssh.addTarget({
target: {
label: `Docker SSH Codex Artifact Repro ${Date.now()}`,
host: '127.0.0.1',
port: target.port,
username: 'root',
identityFile: target.identityFile,
identitiesOnly: true,
relayGracePeriodSeconds: 1
}
})
store.getState().recordSshRepoReadoptions(repoReadoptions)
const state = await window.api.ssh.connect({ targetId: createdTarget.id })
if (!state || state.status !== 'connected') {
throw new Error(`SSH target did not connect: ${JSON.stringify(state)}`)
}
store.getState().setSshConnectionState(createdTarget.id, state)
const labels = new Map(store.getState().sshTargetLabels)
labels.set(createdTarget.id, createdTarget.label)
store.getState().setSshTargetLabels(labels)
const result = await window.api.repos.addRemote({
connectionId: createdTarget.id,
remotePath,
displayName: 'Docker SSH Codex Artifact Repro'
})
if ('error' in result) {
throw new Error(result.error)
}
await store.getState().fetchRepos()
await store.getState().fetchWorktrees(result.repo.id)
return { targetId: createdTarget.id, repoId: result.repo.id, repoPath: result.repo.path }
} finally {
credentialUnsub()
}
},
{ target, remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH }
)
await expect
.poll(
() =>
page.evaluate(async (repoId) => {
const store = window.__store
if (!store) {
return 0
}
await store.getState().fetchWorktrees(repoId)
return store.getState().worktreesByRepo[repoId]?.length ?? 0
}, remote.repoId),
{ timeout: 30_000, message: `No remote worktree found for ${remote.repoPath}` }
)
.toBeGreaterThan(0)
const worktreeId = await page.evaluate((repoId) => {
const store = window.__store
const worktree = store?.getState().worktreesByRepo[repoId]?.[0]
if (!store || !worktree) {
throw new Error(`Remote worktree disappeared for repo ${repoId}`)
}
store.getState().setActiveWorktree(worktree.id)
if ((store.getState().tabsByWorktree[worktree.id] ?? []).length === 0) {
store.getState().createTab(worktree.id)
}
store.getState().setActiveTabType('terminal')
return worktree.id
}, remote.repoId)
return { targetId: remote.targetId, worktreeId }
}
export async function switchToNonRemoteWorktree(
page: Page,
remoteWorktreeId: string
): Promise<string> {
const otherWorktreeId = await page.evaluate((remoteWorktreeId) => {
const store = window.__store
if (!store) {
return null
}
const state = store.getState()
const other = Object.values(state.worktreesByRepo)
.flat()
.find((worktree) => worktree.id !== remoteWorktreeId)
if (!other) {
return null
}
state.setActiveWorktree(other.id)
return other.id
}, remoteWorktreeId)
if (!otherWorktreeId) {
throw new Error('No non-remote worktree available to hide the SSH terminal')
}
return otherWorktreeId
}
export async function installPtyReplayProbe(page: Page): Promise<void> {
await page.evaluate(() => {
const api = window.api?.pty
if (!api || typeof api.onReplay !== 'function') {
throw new Error('PTY replay API unavailable')
}
const holder = window as unknown as {
__orcaSshCodexReplayProbe?: {
payloads: { id: string; length: number; preview: string }[]
dispose: () => void
}
}
holder.__orcaSshCodexReplayProbe?.dispose()
const payloads: { id: string; length: number; preview: string }[] = []
const dispose = api.onReplay(({ id, data }) => {
payloads.push({
id,
length: data.length,
preview: data.slice(-400)
})
})
holder.__orcaSshCodexReplayProbe = { payloads, dispose }
})
}
export async function waitForDockerRemoteReconnected(page: Page, targetId: string): Promise<void> {
let observedNonConnected = false
await expect
.poll(
async () => {
const status = await page.evaluate((targetId) => {
const state = window.__store?.getState().sshConnectionStates.get(targetId)
return state?.status ?? null
}, targetId)
if (status !== 'connected') {
observedNonConnected = true
}
return observedNonConnected && status === 'connected'
},
{
timeout: 90_000,
message: 'Docker SSH target did not auto-reconnect after transport drop'
}
)
.toBe(true)
}
export async function readReplayProbeSnapshot(page: Page): Promise<Record<string, unknown>> {
return page.evaluate(() => {
const probe = (
window as unknown as {
__orcaSshCodexReplayProbe?: {
payloads: { id: string; length: number; preview: string }[]
}
}
).__orcaSshCodexReplayProbe
return {
replayCount: probe?.payloads.length ?? 0,
replayPayloads: probe?.payloads.slice(-8) ?? []
}
})
}
export async function readDuplicateStatusRows(page: Page): Promise<string[]> {
return page.evaluate(() => {
const state = window.__store?.getState()
const worktreeId = state?.activeWorktreeId
const tabId =
state?.activeTabType === 'terminal'
? state.activeTabId
: worktreeId
? (state?.activeTabIdByWorktree?.[worktreeId] ?? null)
: null
const manager = tabId ? window.__paneManagers?.get(tabId) : null
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
const text = pane?.serializeAddon?.serialize?.() ?? ''
const counts = new Map<string, number>()
const escapeSequencePattern = new RegExp(
`${String.fromCharCode(27)}\\[[0-9;?]*[ -/]*[@-~]`,
'g'
)
for (const line of text.split(/\r?\n/)) {
const normalized = line.replace(escapeSequencePattern, '').trim()
if (!/gpt-5\.5|background terminal|\/ps to view|\/stop to close/i.test(normalized)) {
continue
}
counts.set(normalized, (counts.get(normalized) ?? 0) + 1)
}
return Array.from(counts)
.filter(([, count]) => count > 1)
.map(([line, count]) => `${count}x ${line}`)
.slice(0, 12)
})
}
export async function enableRiskyTerminalRendererPath(page: Page): Promise<void> {
await page.evaluate(() => {
const store = window.__store
if (!store) {
throw new Error('window.__store unavailable')
}
const state = store.getState()
store.setState({
settings: {
...state.settings!,
terminalGpuAcceleration: 'on',
theme: 'dark'
}
})
const worktreeId = state.activeWorktreeId
const tabId =
state.activeTabType === 'terminal'
? state.activeTabId
: worktreeId
? (state.activeTabIdByWorktree?.[worktreeId] ?? null)
: null
const manager = tabId ? window.__paneManagers?.get(tabId) : null
manager?.setTerminalGpuAcceleration('on')
})
}