test(e2e): cover SSH user journeys on an orcad-managed host (#27323)

* test(e2e): cover SSH terminal and workspace journeys on a managed orcad host

* test(e2e): port the provisioned-root recipe journey; read ports from the host scan

* test(e2e): fix the provisioned-root spec's catalog refresh call

* test(e2e): wait for the shell prompt, open file links directly, and mark forward restore fixme

* test(e2e): record launches for teardown, match the prompt robustly, park the link test

* test(ci): list the journeys job in the workflow permission matrix

* test(e2e): run the terminal image-link journey again

* test(e2e): park the external-path link journey on its product gap

---------

Co-authored-by: m4air <m4air@Mac.localdomain>
This commit is contained in:
OrcaWin
2026-10-10 18:07:50 -07:00
committed by GitHub
co-authored by m4air
parent 43fbd2389f
commit db309a5777
11 changed files with 977 additions and 7 deletions
+52 -1
View File
@@ -363,7 +363,10 @@ jobs:
. != "tests/e2e/ssh-orcad-explorer-watch-recovery.spec.ts" and
. != "tests/e2e/ssh-orcad-explorer-selected-host.spec.ts" and
. != "tests/e2e/ssh-orcad-browser-drop-owner.spec.ts" and
. != "tests/e2e/ssh-orcad-terminal-root-owner.spec.ts"
. != "tests/e2e/ssh-orcad-terminal-root-owner.spec.ts" and
. != "tests/e2e/ssh-orcad-terminal-journeys.spec.ts" and
. != "tests/e2e/ssh-orcad-workspace-tools.spec.ts" and
. != "tests/e2e/ssh-orcad-vm-provisioned-root.spec.ts"
)' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs"
fi
mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs"
@@ -642,6 +645,54 @@ jobs:
retention-days: 7
if-no-files-found: ignore
# SSH user journeys (terminal reconnect/restore/faults, Quick Open, ports, links, agent status) on a
# Docker host served by managed orcad. Its own runner so the convert lane keeps its time budget.
orcad-journeys-docker:
name: ssh journeys on managed orcad (Docker)
needs: [build, prepare-native-cache]
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-terminal-journeys.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-workspace-tools.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-vm-provisioned-root.spec.ts')
runs-on: ubuntu-latest
timeout-minutes: 60
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential ripgrep xvfb openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- uses: actions/download-artifact@v8
with:
name: e2e-build-out
path: out/
- name: Build the linux-x64-glibc orcad template
run: |
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots linux-x64-glibc
pnpm build:orcad
node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc
mv out/orcad-template "$RUNNER_TEMP/orcad-convert-template"
- name: Run SSH journeys on a managed Docker host
env:
SKIP_BUILD: '1'
ORCA_E2E_SSH_DOCKER: '1'
ORCA_E2E_ORCAD_CONVERT_HOST: docker
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: |
export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template"
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-terminal-journeys.spec.ts tests/e2e/ssh-orcad-workspace-tools.spec.ts tests/e2e/ssh-orcad-vm-provisioned-root.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
with:
name: orcad-journeys-docker-traces
path: test-results/
retention-days: 7
if-no-files-found: ignore
# D7 on Windows: both hosts share <userData>\daemon and so one daemon pipe. Built here rather than
# from the Linux e2e build because the slot, template and addons are Windows-native. Also runs the
# missing-AppData startup check, which needs the same Windows e2e build.
+10 -2
View File
@@ -71,6 +71,12 @@ export const ORCAD_EXPLORER_WATCH_RECOVERY_E2E_SPEC =
export const ORCAD_EXPLORER_SELECTED_HOST_E2E_SPEC =
'tests/e2e/ssh-orcad-explorer-selected-host.spec.ts'
export const ORCAD_TERMINAL_ROOT_OWNER_E2E_SPEC = 'tests/e2e/ssh-orcad-terminal-root-owner.spec.ts'
// Run in their own job, which builds the template they need.
export const ORCAD_JOURNEY_E2E_SPECS = [
'tests/e2e/ssh-orcad-terminal-journeys.spec.ts',
'tests/e2e/ssh-orcad-workspace-tools.spec.ts',
'tests/e2e/ssh-orcad-vm-provisioned-root.spec.ts'
]
export const DEDICATED_E2E_SPECS = [
...DOCKER_SSH_E2E_SPECS,
NODE_NETWORK_E2E_SPEC,
@@ -93,7 +99,8 @@ export const DEDICATED_E2E_SPECS = [
ORCAD_EXPLORER_WATCH_RECOVERY_E2E_SPEC,
ORCAD_EXPLORER_SELECTED_HOST_E2E_SPEC,
ORCAD_BROWSER_DROP_OWNER_E2E_SPEC,
ORCAD_TERMINAL_ROOT_OWNER_E2E_SPEC
ORCAD_TERMINAL_ROOT_OWNER_E2E_SPEC,
...ORCAD_JOURNEY_E2E_SPECS
]
const dedicatedSpecs = new Set(DEDICATED_E2E_SPECS)
const dockerSpecs = new Set(DOCKER_SSH_E2E_SPECS)
@@ -148,7 +155,8 @@ export function classifyE2eJobs(input, sshSourceChanged = 'false') {
spec === ORCAD_EXPLORER_WATCH_RECOVERY_E2E_SPEC ||
spec === ORCAD_EXPLORER_SELECTED_HOST_E2E_SPEC ||
spec === ORCAD_BROWSER_DROP_OWNER_E2E_SPEC ||
spec === ORCAD_TERMINAL_ROOT_OWNER_E2E_SPEC
spec === ORCAD_TERMINAL_ROOT_OWNER_E2E_SPEC ||
ORCAD_JOURNEY_E2E_SPECS.includes(spec)
)
}
}
+18
View File
@@ -6,6 +6,7 @@ import {
classifyE2eJobs,
ORCAD_AUTO_CONVERT_E2E_SPEC,
ORCAD_IDLE_EXIT_E2E_SPEC,
ORCAD_JOURNEY_E2E_SPECS,
ORCAD_OPEN_IN_OWNER_E2E_SPEC,
ORCAD_SERVE_MODE_SWITCH_E2E_SPEC,
WINDOWS_MISSING_APPDATA_E2E_SPEC
@@ -137,3 +138,20 @@ it('routes managed target-owner guards and menus to the real launch regression',
)
).toBe(true)
})
it('runs the managed-host journey specs in their own job when routed', () => {
const job = jobs['orcad-journeys-docker']
expect(job.needs).toEqual(['build', 'prepare-native-cache'])
for (const spec of ORCAD_JOURNEY_E2E_SPECS) {
expectRouted(['tests/e2e/helpers/orcad-managed-workspace.ts'], spec)
expect(job.if, spec).toContain(`contains(inputs.test_files, '${spec}')`)
expect(
job.steps.some((step) => step.run?.includes(spec)),
spec
).toBe(true)
expect(classifyE2eJobs(JSON.stringify([spec])), spec).toEqual({
e2e_run_changed: false,
e2e_needs_build: true
})
}
})
+17
View File
@@ -186,6 +186,23 @@ export const PR_E2E_SOURCE_ROUTES = [
file
))
},
{
// Why: managed SSH hosts are now the only SSH path, so these specs carry its user journeys.
id: 'ssh.orcad-journeys',
specs: [
'tests/e2e/ssh-orcad-terminal-journeys.spec.ts',
'tests/e2e/ssh-orcad-workspace-tools.spec.ts',
'tests/e2e/ssh-orcad-vm-provisioned-root.spec.ts'
],
matches: (file) =>
/^tests\/e2e\/helpers\/(?:orcad-managed-workspace|orcad-convert-(?:flow|host)|orcad-upgrade-profile|paired-host-terminal|terminal-workspace-root-link)\.ts$/.test(
file
) ||
(isProductSource(file) &&
/^src\/(?:main\/ssh\/orcad-managed-|main\/ipc\/ssh-port-forward|renderer\/src\/startup\/(?:ssh-startup-reconnect|startup-ssh-connection-restore)\.ts$|renderer\/src\/runtime\/web-runtime-session|renderer\/src\/components\/terminal-pane\/remote-runtime-)/.test(
file
))
},
{
id: 'ssh.localhost-agent-hooks',
specs: ['tests/e2e/ssh-localhost.spec.ts'],
@@ -12,6 +12,7 @@ const EXPECTED_MATRIX = {
'.github/workflows/e2e.yml#changed-e2e': { contents: 'read' },
'.github/workflows/e2e.yml#e2e': { contents: 'read' },
'.github/workflows/e2e.yml#orcad-auto-convert-docker': { contents: 'read' },
'.github/workflows/e2e.yml#orcad-journeys-docker': { contents: 'read' },
'.github/workflows/e2e.yml#orcad-serve-mode-switch': { contents: 'read' },
'.github/workflows/e2e.yml#orcad-serve-mode-switch-windows': { contents: 'read' },
'.github/workflows/e2e.yml#prepare-native-cache': { contents: 'read' },
+27 -1
View File
@@ -2,6 +2,7 @@
* The real SSH host a conversion cell runs against: the Linux Docker fixture, or a Windows
* OpenSSH host the ssh-windows-hosts lane provisioned and described in a descriptor file.
*/
import { execFileSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import type { TestInfo } from '@stablyai/playwright-test'
import type { SshTargetCreateInput } from '../../../src/shared/ssh-types'
@@ -11,6 +12,7 @@ import {
cleanupDockerSshRelayTarget,
DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
execDockerSshRelayTargetCommand,
execDockerSshRelayTargetControlCommand,
startDockerSshRelayTarget
} from './docker-ssh-relay-target'
@@ -27,8 +29,23 @@ export type OrcadConvertHost = {
blockTcpForwarding?: () => void
/** Docker only: runs a shell command on the host as the SSH user and returns its stdout. */
exec?: (command: string) => string
/** Docker only: kills every live SSH connection, leaving sshd listening; returns how many. */
dropSshConnections?: () => number
/** Docker only: freezes or thaws the whole host; TCP stays up but nothing answers. */
setFrozen?: (frozen: boolean) => void
}
// The listener is the oldest sshd; every other sshd/sshd-session is a live connection.
const DROP_SSH_CONNECTIONS = `
daemon="$(pgrep -x sshd | sort -n | head -1)"
killed=0
for pid in $(pgrep -x sshd; pgrep -x sshd-session); do
[ "$pid" = "$daemon" ] && continue
kill -9 "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "$killed"
`
/** `docker`, or the path of a Windows host-cell descriptor. */
export function startOrcadConvertHost(source: string, testInfo: TestInfo): OrcadConvertHost {
const label = `orcad convert E2E ${Date.now()}`
@@ -48,7 +65,16 @@ export function startOrcadConvertHost(source: string, testInfo: TestInfo): Orcad
remoteFolderPath: '/tmp',
cleanup: () => cleanupDockerSshRelayTarget(target),
blockTcpForwarding: () => blockDockerSshRelayTargetTcpForwarding(target),
exec: (command) => execDockerSshRelayTargetCommand(target, command)
exec: (command) => execDockerSshRelayTargetCommand(target, command),
dropSshConnections: () =>
Number(
execDockerSshRelayTargetControlCommand(target, DROP_SSH_CONNECTIONS).split('\n').at(-1)
),
setFrozen: (frozen) =>
execFileSync('docker', [frozen ? 'pause' : 'unpause', target.containerName], {
stdio: 'ignore',
timeout: 30_000
})
}
}
const descriptor = parseWindowsHostCellDescriptor(readFileSync(source, 'utf8'))
@@ -0,0 +1,208 @@
/**
* A repository workspace on an orcad-managed Docker SSH host, reached the way users reach one:
* a saved SSH profile whose first connect deploys the managed server. Plus the terminal moves the
* managed-host journey specs share, driven through the tab bar and the keyboard.
*/
import type { ElectronApplication, Page, TestInfo } from '@stablyai/playwright-test'
import { expect } from './orca-app'
import { createRestartSession } from './orca-restart'
import { startOrcadConvertHost, type OrcadConvertHost } from './orcad-convert-host'
import { seedRelayEraProfile, type RelayEraProfile } from './orcad-upgrade-profile'
import { convertAndRetain } from './orcad-convert-flow'
import { readPairedPaneContent } from './paired-host-terminal'
import { waitForSessionReady } from './store'
import { dismissTransientAnnouncement } from './ssh-config-host-picker'
import { focusActiveTerminalInput } from './terminal'
import { toRuntimeExecutionHostId } from '../../../src/shared/execution-host'
export const ORCAD_TEMPLATE = process.env.ORCA_E2E_ORCAD_CONVERT_TEMPLATE
export type ManagedHostFixture = {
host: OrcadConvertHost
session: ReturnType<typeof createRestartSession>
/** Set by the body to whichever launch is live, so teardown closes it. */
app: ElectronApplication | null
}
/** A fresh Docker SSH host and profile for `body`, torn down whatever it throws. */
export async function withManagedHost(
testInfo: TestInfo,
body: (fixture: ManagedHostFixture) => Promise<void>
): Promise<void> {
const host = startOrcadConvertHost('docker', testInfo)
const session = createRestartSession(testInfo, { ORCA_ORCAD_TEMPLATE_PATH: ORCAD_TEMPLATE! })
const fixture: ManagedHostFixture = { host, session, app: null }
try {
await body(fixture)
} finally {
try {
if (fixture.app) {
await session.close(fixture.app)
}
await session.dispose()
} finally {
host.cleanup()
}
}
}
export type ManagedWorkspace = {
app: ElectronApplication
page: Page
seeded: RelayEraProfile
environmentId: string
}
/** Leaves the managed repository active; each launch is recorded on `fixture` for teardown. */
export async function launchManagedWorkspace(
fixture: ManagedHostFixture
): Promise<ManagedWorkspace> {
const { host, session } = fixture
// The first launch only creates the profile the saved SSH host is written into.
const first = await session.launch()
fixture.app = first.app
await waitForSessionReady(first.page)
await session.close(first.app)
fixture.app = null
const seeded = seedRelayEraProfile(session.userDataDir, host.input, {
repoPath: host.remoteRepoPath,
folderPath: host.remoteFolderPath
})
const { app, page } = await session.launch()
fixture.app = app
await waitForSessionReady(page)
await convertAndRetain(page, session.userDataDir, seeded)
const environmentId = await managedEnvironmentId(page, seeded.targetId)
await activateManagedWorkspace(page, seeded.worktreeId, environmentId)
return { app, page, seeded, environmentId }
}
export async function managedEnvironmentId(page: Page, targetId: string): Promise<string> {
const environment = (await page.evaluate(() => window.api.runtimeEnvironments.list())).find(
(entry) => entry.orcadDeployment?.sshTargetId === targetId
)
if (!environment) {
throw new Error(`No managed server registered for ${targetId}`)
}
return environment.id
}
export async function activateManagedWorkspace(
page: Page,
worktreeId: string,
environmentId: string
): Promise<void> {
await page.evaluate(
({ worktreeId, hostId }) => {
const state = window.__store!.getState()
state.setActiveView('terminal')
state.setActiveWorktree(worktreeId, hostId)
},
{ worktreeId, hostId: toRuntimeExecutionHostId(environmentId) }
)
await dismissTransientAnnouncement(page)
}
export function terminalTabIds(page: Page, worktreeId: string): Promise<string[]> {
return page.evaluate(
(id) => (window.__store?.getState().tabsByWorktree[id] ?? []).map((tab) => tab.id),
worktreeId
)
}
/** Opens a terminal the way the tab bar's + does and waits for its pane to mount. */
export async function openTerminalTab(page: Page, worktreeId: string): Promise<string> {
const before = new Set(await terminalTabIds(page, worktreeId))
await page.evaluate(async (worktreeId) => {
const state = window.__store!.getState()
const groupId = state.activeGroupIdByWorktree[worktreeId]
if (!groupId) {
throw new Error(`No active tab group for ${worktreeId}`)
}
await state.openNewTerminalTabInActiveWorkspace(groupId)
}, worktreeId)
let created: string | undefined
await expect
.poll(
async () => {
created = (await terminalTabIds(page, worktreeId)).find((id) => !before.has(id))
return created ?? null
},
{ timeout: 60_000, message: 'the new terminal tab never appeared' }
)
.not.toBeNull()
await waitForPane(page, created!)
// Keys typed before the shell's first prompt can be lost, as they would be for a user too.
await expect
.poll(() => readPairedPaneContent(page, created!), {
timeout: 60_000,
message: `the shell in ${created} never printed a prompt`
})
.toMatch(/[#$] /)
return created!
}
export async function waitForPane(page: Page, tabId: string): Promise<void> {
await expect
.poll(() => page.evaluate((id) => window.__paneManagers?.has(id) ?? false, tabId), {
timeout: 60_000,
message: `pane for ${tabId} did not mount`
})
.toBe(true)
}
export function selectTerminalTab(page: Page, tabId: string): Promise<void> {
return page.evaluate((id) => {
const state = window.__store!.getState()
state.setActiveTab(id)
state.setActiveTabType('terminal', state.activeWorktreeId)
}, tabId)
}
/** Types `command` into the tab's pane and presses Enter, as a user at the keyboard would. */
export async function typeInTerminal(page: Page, tabId: string, command: string): Promise<void> {
await selectTerminalTab(page, tabId)
await waitForPane(page, tabId)
await focusActiveTerminalInput(page)
await page.keyboard.insertText(command)
await page.keyboard.press('Enter')
}
export async function expectPaneText(
page: Page,
tabId: string,
text: string,
timeout = 60_000
): Promise<void> {
await expect
.poll(() => readPairedPaneContent(page, tabId), {
timeout,
message: `pane ${tabId} never showed ${text}`
})
.toContain(text)
}
/** Echoes a unique marker and waits for the output; returns the marker. */
export async function echoMarker(
page: Page,
tabId: string,
prefix: string,
timeout = 60_000
): Promise<string> {
const marker = `${prefix}_${Date.now()}`
// Quoted apart so the typed command line itself can never satisfy the check.
await typeInTerminal(page, tabId, `echo ${prefix}''_${marker.slice(prefix.length + 1)}`)
await expectPaneText(page, tabId, marker, timeout)
return marker
}
/** Proves the shell reads input again: retypes a fresh marker until one is echoed back. */
export async function expectTerminalAnswers(
page: Page,
tabId: string,
timeout = 120_000
): Promise<void> {
await expect(async () => {
await echoMarker(page, tabId, 'ANSWER', 10_000)
}).toPass({ timeout })
}
@@ -6,7 +6,9 @@ import { focusActiveTerminalInput, getTerminalContent } from './terminal'
export async function openTerminalWorkspaceRootLink(
page: Page,
testInfo: TestInfo,
rootPath: string
rootPath: string,
/** `open` Ctrl/Cmd-clicks, which opens a file link directly instead of its actions menu. */
click: 'actions' | 'open' = 'actions'
): Promise<{ x: number; y: number }> {
await focusActiveTerminalInput(page)
const readyMarker = `ROOT_LINK_PRINTED_${Date.now()}`
@@ -63,9 +65,21 @@ export async function openTerminalWorkspaceRootLink(
}
await page.mouse.move(target.current.x, target.current.y)
await expect(
page.locator('.pane-link-tooltip:visible').filter({ hasText: `${rootPath} (Click for actions` })
page
.locator('.pane-link-tooltip:visible')
.filter({ hasText: click === 'actions' ? `${rootPath} (Click for actions` : `${rootPath} (` })
).toBeVisible()
await page.screenshot({ path: testInfo.outputPath('managed-terminal-root-link-hover.png') })
await page.mouse.click(target.current.x, target.current.y)
if (click === 'actions') {
await page.mouse.click(target.current.x, target.current.y)
return target.current
}
const modifier = process.platform === 'darwin' ? 'Meta' : 'Control'
await page.keyboard.down(modifier)
try {
await page.mouse.click(target.current.x, target.current.y)
} finally {
await page.keyboard.up(modifier)
}
return target.current
}
@@ -0,0 +1,216 @@
/**
* Terminal journeys on an orcad-managed SSH host, read from what each pane actually shows:
*
* 1. Reconnect: a pane keeps its scrollback and a running full-screen app, and a tab opened after
* a reconnect gets a fresh shell that survives the next reconnect.
* 2. Restore: tabs come back exactly once across relaunches and a renderer reload, and a tab
* closed while the host is disconnected stays closed.
* 3. Faults: a pane outlives a dropped SSH link and a frozen host, and answers input after both.
*/
import type { ElectronApplication, Page } from '@stablyai/playwright-test'
import { expect, test } from './helpers/orca-app'
import { reconnect } from './helpers/orcad-convert-flow'
import { readPairedPaneContent } from './helpers/paired-host-terminal'
import { waitForSessionReady, waitForStartupWorktreeRefresh } from './helpers/store'
import {
activateManagedWorkspace,
echoMarker,
expectPaneText,
expectTerminalAnswers,
launchManagedWorkspace,
openTerminalTab,
ORCAD_TEMPLATE,
selectTerminalTab,
terminalTabIds,
typeInTerminal,
waitForPane,
withManagedHost
} from './helpers/orcad-managed-workspace'
test.skip(
!ORCAD_TEMPLATE || process.env.ORCA_E2E_SSH_DOCKER !== '1',
'Needs Docker and server template'
)
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
function relaunchable(app: ElectronApplication | null): ElectronApplication {
if (!app) {
throw new Error('No live launch to close')
}
return app
}
function sshStatus(page: Page, targetId: string): Promise<string | null> {
return page.evaluate(
(id) => window.__store?.getState().sshConnectionStates.get(id)?.status ?? null,
targetId
)
}
/** A live full-screen app redraws; a frozen or replayed frame does not. */
async function expectTopRedrawing(page: Page, tabId: string): Promise<void> {
await expectPaneText(page, tabId, 'load average')
const frame = await readPairedPaneContent(page, tabId)
await expect.poll(() => readPairedPaneContent(page, tabId), { timeout: 30_000 }).not.toBe(frame)
expect(await readPairedPaneContent(page, tabId)).toContain('PID')
}
test('a managed pane keeps its scrollback and a running TUI across reconnects', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(12 * 60_000)
await withManagedHost(testInfo, async (fixture) => {
// The fixture shell emits no OSC title; a real user's shell does, and the new-tab check reads it.
fixture.host.exec!(`printf '%s\\n' 'PS1="\\[\\e]0;\\u@\\h: \\w\\a\\]$PS1"' >> /root/.bashrc`)
const { page, seeded } = await launchManagedWorkspace(fixture)
const first = await openTerminalTab(page, seeded.worktreeId)
const marker = await echoMarker(page, first, 'RECONNECT_MARKER')
await reconnect(page, seeded.targetId)
await selectTerminalTab(page, first)
await waitForPane(page, first)
await expectPaneText(page, first, marker)
await typeInTerminal(page, first, 'top')
await expectTopRedrawing(page, first)
for (let round = 0; round < 2; round += 1) {
await reconnect(page, seeded.targetId)
await selectTerminalTab(page, first)
await waitForPane(page, first)
await expectTopRedrawing(page, first)
}
// A tab opened after a reconnect gets its own shell, and survives the next reconnect.
const fresh = await openTerminalTab(page, seeded.worktreeId)
expect(fresh).not.toBe(first)
await expectTerminalAnswers(page, fresh)
expect(await readPairedPaneContent(page, fresh)).not.toContain(marker)
await expect
.poll(
() =>
page.evaluate(
({ worktreeId, tabId }) =>
window.__store?.getState().tabsByWorktree[worktreeId]?.find((tab) => tab.id === tabId)
?.title ?? null,
{ worktreeId: seeded.worktreeId, tabId: fresh }
),
{ timeout: 60_000, message: 'the new tab kept its placeholder title' }
)
.not.toMatch(/^Terminal \d+$/)
await reconnect(page, seeded.targetId)
expect(await terminalTabIds(page, seeded.worktreeId)).toEqual(
expect.arrayContaining([first, fresh])
)
await expectTerminalAnswers(page, fresh)
await page.screenshot({ path: testInfo.outputPath('managed-reconnect-panes.png') })
})
})
/** Samples until the tab set agrees three times running, so a late restore cannot slip past. */
async function settledTabIds(page: Page, worktreeId: string): Promise<string[]> {
let previous = ''
let agreements = 0
let latest: string[] = []
await expect
.poll(
async () => {
latest = (await terminalTabIds(page, worktreeId)).sort()
const key = JSON.stringify(latest)
agreements = key === previous && latest.length > 0 ? agreements + 1 : 0
previous = key
return agreements >= 2
},
{ timeout: 120_000, intervals: [2_000] }
)
.toBe(true)
return latest
}
test('managed tabs restore exactly once, and a tab closed while disconnected stays closed', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(15 * 60_000)
await withManagedHost(testInfo, async (fixture) => {
const launched = await launchManagedWorkspace(fixture)
const { seeded, environmentId } = launched
let page = launched.page
const opened = [
await openTerminalTab(page, seeded.worktreeId),
await openTerminalTab(page, seeded.worktreeId),
await openTerminalTab(page, seeded.worktreeId)
]
const marker = await echoMarker(page, opened[0], 'RESTORE_MARKER')
const expected = await settledTabIds(page, seeded.worktreeId)
expect(expected).toEqual(expect.arrayContaining(opened))
for (let cycle = 0; cycle < 2; cycle += 1) {
await fixture.session.close(relaunchable(fixture.app))
fixture.app = null
const relaunched = await fixture.session.launch()
fixture.app = relaunched.app
page = relaunched.page
await waitForSessionReady(page)
await waitForStartupWorktreeRefresh(page)
await activateManagedWorkspace(page, seeded.worktreeId, environmentId)
expect(await settledTabIds(page, seeded.worktreeId), `relaunch ${cycle + 1}`).toEqual(
expected
)
await selectTerminalTab(page, opened[0])
await expectPaneText(page, opened[0], marker)
}
// A renderer reload remounts every restored tab onto its own shell.
await page.evaluate(() => window.dispatchEvent(new Event('beforeunload')))
await page.reload()
await waitForSessionReady(page, 60_000)
await activateManagedWorkspace(page, seeded.worktreeId, environmentId)
expect(await settledTabIds(page, seeded.worktreeId)).toEqual(expected)
for (const tabId of opened) {
await expectTerminalAnswers(page, tabId)
}
await expectPaneText(page, opened[0], marker)
// The lid-closed shape: close a tab while disconnected, then reconnect.
const closed = opened[2]
await page.evaluate((targetId) => window.api.ssh.disconnect({ targetId }), seeded.targetId)
await page.evaluate((tabId) => window.__store!.getState().closeTab(tabId), closed)
await reconnect(page, seeded.targetId)
const remaining = expected.filter((id) => id !== closed)
expect(await settledTabIds(page, seeded.worktreeId)).toEqual(remaining)
await fixture.session.close(relaunchable(fixture.app))
fixture.app = null
const final = await fixture.session.launch()
fixture.app = final.app
await waitForSessionReady(final.page)
await waitForStartupWorktreeRefresh(final.page)
await activateManagedWorkspace(final.page, seeded.worktreeId, environmentId)
expect(await settledTabIds(final.page, seeded.worktreeId)).toEqual(remaining)
})
})
test('a managed pane outlives a dropped SSH link and a frozen host', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(15 * 60_000)
await withManagedHost(testInfo, async (fixture) => {
const { page, seeded } = await launchManagedWorkspace(fixture)
const tabId = await openTerminalTab(page, seeded.worktreeId)
const marker = await echoMarker(page, tabId, 'FAULT_MARKER')
expect(fixture.host.dropSshConnections!()).toBeGreaterThan(0)
await expectTerminalAnswers(page, tabId, 180_000)
await expectPaneText(page, tabId, marker)
// Silence, not a reset: the client must call the host lost instead of wedging on it.
fixture.host.setFrozen!(true)
try {
await expect
.poll(() => sshStatus(page, seeded.targetId), { timeout: 120_000, intervals: [2_000] })
.not.toBe('connected')
} finally {
fixture.host.setFrozen!(false)
}
await reconnect(page, seeded.targetId)
await selectTerminalTab(page, tabId)
await expectTerminalAnswers(page, tabId, 180_000)
await expectPaneText(page, tabId, marker)
await page.screenshot({ path: testInfo.outputPath('managed-fault-recovery.png') })
})
})
@@ -0,0 +1,213 @@
import { openSidebarWorkspaceComposer } from './helpers/sidebar-project-dialog'
import { execFileSync } from 'node:child_process'
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { homedir, tmpdir } from 'node:os'
import path from 'node:path'
import { expect, test } from './helpers/orca-app'
import { ensureDockerSshRelayImage } from './helpers/docker-ssh-relay-image'
import {
cleanupDockerSshRelayTarget,
execDockerSshRelayTargetCommand,
shellQuote,
startDockerSshRelayTarget,
DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
type DockerSshRelayTarget
} from './helpers/docker-ssh-relay-target'
import { ensureTerminalVisible, waitForSessionReady } from './helpers/store'
import { ORCAD_TEMPLATE } from './helpers/orcad-managed-workspace'
/** A recipe VM's SSH root, served by a managed orcad the first connect deploys. */
test.skip(
!ORCAD_TEMPLATE || process.env.ORCA_E2E_SSH_DOCKER !== '1',
'Needs Docker and server template'
)
test.skip(process.platform === 'win32', 'The recipe scripts are POSIX shell')
test.use({
seedTestRepo: false,
orcaAppExtraEnv: { ORCA_ORCAD_TEMPLATE_PATH: ORCAD_TEMPLATE ?? '' }
})
test('adopts a recipe-provisioned SSH root without creating a linked worktree', async ({
orcaPage
}, testInfo) => {
test.setTimeout(240_000)
let target: DockerSshRelayTarget | null = null
const sourceRepo = mkdtempSync(path.join(tmpdir(), 'orca-provisioned-root-source-'))
try {
ensureDockerSshRelayImage(process.cwd())
target = startDockerSshRelayTarget(testInfo)
const expectedRefHead = seedRecipeRepo(sourceRepo, target)
await waitForSessionReady(orcaPage)
const sourceRepoId = await addRecipeRepo(orcaPage, sourceRepo)
await openSidebarWorkspaceComposer(orcaPage)
const dialog = orcaPage.getByRole('dialog', { name: /Create (Workspace|Worktree)/i })
await expect(dialog).toBeVisible()
await dialog.getByRole('combobox', { name: 'Run on' }).click()
await orcaPage.getByRole('option', { name: /Per-Workspace Environment/ }).click()
await orcaPage
.getByRole('listbox', { name: 'Per-Workspace Environment' })
.getByText('Docker provisioned root', { exact: true })
.click()
const workspaceName = `provisioned-root-${Date.now()}`
await dialog.getByPlaceholder(/Type a name/i).fill(workspaceName)
await dialog.getByRole('button', { name: /Create (Workspace|Worktree)/i }).click()
const trustDialog = orcaPage.getByRole('dialog', { name: /Run VM recipe/ })
await expect(trustDialog).toBeVisible()
await trustDialog.getByRole('button', { name: 'Run hooks' }).click()
await expect(dialog).toBeHidden({ timeout: 60_000 })
await expect(orcaPage.getByRole('option', { name: new RegExp(workspaceName) })).toBeVisible({
timeout: 60_000
})
await ensureTerminalVisible(orcaPage)
const adopted = await orcaPage.evaluate(
({ sourceRepoId, workspaceName }) => {
const state = window.__store!.getState()
return Object.values(state.worktreesByRepo)
.flat()
.find(
(worktree) => worktree.displayName === workspaceName && worktree.repoId !== sourceRepoId
)
},
{ sourceRepoId, workspaceName }
)
expect(adopted).toMatchObject({
path: DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
isMainWorktree: true,
ephemeralVmCheckoutMode: 'provisioned-root'
})
expect(adopted?.hostId).toMatch(/^(?:ssh:runtime-ssh-|runtime:)/)
expect(
execDockerSshRelayTargetCommand(
target,
`git -C ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} worktree list --porcelain | grep -c '^worktree '`
)
).toBe('1')
expect(
execDockerSshRelayTargetCommand(
target,
`git -C ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} branch --show-current`
)
).toBe(workspaceName)
expect(
execDockerSshRelayTargetCommand(
target,
`git -C ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} rev-parse HEAD`
)
).toBe(expectedRefHead)
const removeDialog = orcaPage.getByRole('dialog', { name: 'Remove Project' })
const removeMenuItem = orcaPage.getByRole('menuitem', { name: 'Remove Project from Orca' })
await expect(async () => {
await orcaPage
.getByRole('option', { name: new RegExp(workspaceName) })
.click({ button: 'right' })
await expect(removeMenuItem).toBeVisible({ timeout: 1_000 })
await removeMenuItem.click({ force: true, timeout: 1_000 })
await expect(removeDialog).toBeVisible({ timeout: 1_000 })
}).toPass({ timeout: 10_000 })
await expect(removeDialog).toContainText(
'Its VM recipe determines whether the environment and its files are permanently deleted.'
)
await removeDialog.getByRole('button', { name: 'Remove', exact: true }).click()
await expect
.poll(
() =>
orcaPage.evaluate(
(repoId) => window.__store!.getState().repos.some((repo) => repo.id === repoId),
adopted!.repoId
),
{ timeout: 30_000 }
)
.toBe(false)
const destroyed = target
expect(() => execDockerSshRelayTargetCommand(destroyed, 'true')).toThrow()
} finally {
cleanupDockerSshRelayTarget(target)
rmSync(sourceRepo, { recursive: true, force: true })
}
})
async function addRecipeRepo(page: Parameters<typeof waitForSessionReady>[0], repoPath: string) {
return page.evaluate(async (pathValue) => {
const result = await window.api.repos.add({ path: pathValue })
if ('error' in result) {
throw new Error(result.error)
}
const store = window.__store!
await store.getState().fetchRepos({ runtimeEnvironmentId: null })
await store.getState().updateSettings({ experimentalEphemeralVms: true })
store.getState().setActiveRepo(result.repo.id)
return result.repo.id
}, repoPath)
}
function seedRecipeRepo(repoPath: string, target: DockerSshRelayTarget): string {
const createScript = path.join(repoPath, 'create.sh')
const destroyScript = path.join(repoPath, 'destroy.sh')
// The recipe's isolated HOME must still address the engine that owns the fixture container.
const docker = `docker --config ${shellQuote(process.env.DOCKER_CONFIG ?? path.join(homedir(), '.docker'))}`
writeFileSync(
createScript,
`#!/usr/bin/env bash
set -euo pipefail
[ "\${ORCA_RECIPE_RESULT_SCHEMA_VERSION:-}" = 2 ]
[ -n "\${ORCA_REPO_URL:-}" ]
[ -n "\${ORCA_REPO_REF:-}" ]
[ -n "\${ORCA_REPO_REF_HEAD:-}" ]
[ -n "\${ORCA_REPO_BRANCH:-}" ]
${docker} exec ${shellQuote(target.containerName)} git -C ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} cat-file -e "$ORCA_REPO_REF_HEAD^{commit}"
${docker} exec ${shellQuote(target.containerName)} git -C ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} checkout -B "$ORCA_REPO_BRANCH" "$ORCA_REPO_REF_HEAD" >&2
node -e 'console.log(JSON.stringify({schemaVersion:2,checkoutMode:"provisioned-root",connection:{type:"ssh",projectRoot:process.argv[1],target:{label:"Docker provisioned root",host:process.argv[2],port:Number(process.argv[3]),username:"root",identityFile:process.argv[4],identitiesOnly:true}}}))' ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} ${shellQuote(target.host)} ${target.port} ${shellQuote(target.identityFile)}
`
)
writeFileSync(
destroyScript,
`#!/usr/bin/env bash
set -euo pipefail
cat >/dev/null
${docker} rm -f ${shellQuote(target.containerName)} >/dev/null
`
)
chmodSync(createScript, 0o755)
chmodSync(destroyScript, 0o755)
writeFileSync(
path.join(repoPath, 'orca.yaml'),
`environmentRecipes:
- id: docker-provisioned-root
name: Docker provisioned root
checkoutMode: provisioned-root
create: ./create.sh
destroy: ./destroy.sh
`
)
execFileSync('git', ['init'], { cwd: repoPath })
execFileSync('git', ['config', 'user.email', 'e2e@test.local'], { cwd: repoPath })
execFileSync('git', ['config', 'user.name', 'Orca E2E'], { cwd: repoPath })
execFileSync('git', ['remote', 'add', 'origin', 'https://github.com/stablyai/orca.git'], {
cwd: repoPath
})
execFileSync('git', ['add', '.'], { cwd: repoPath })
execFileSync('git', ['commit', '-m', 'seed recipe'], { cwd: repoPath })
const expectedRefHead = execFileSync('git', ['rev-parse', 'HEAD'], {
cwd: repoPath,
encoding: 'utf8'
}).trim()
execDockerSshRelayTargetCommand(
target,
`rm -rf ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)} && mkdir -p ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`
)
execFileSync('docker', [
'cp',
`${repoPath}${path.sep}.`,
`${target.containerName}:${DOCKER_SSH_RELAY_REMOTE_REPO_PATH}`
])
execDockerSshRelayTargetCommand(
target,
`chown -R root:root ${shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH)}`
)
return expectedRefHead
}
+198
View File
@@ -0,0 +1,198 @@
/**
* Workspace tools on an orcad-managed SSH host: Quick Open over a monorepo-sized tree, the host's
* port scan and an SSH port forward, an image outside the workspace opened from a terminal link, and
* agent status reported from a remote terminal.
*/
import { expect, test } from './helpers/orca-app'
import { reconnect } from './helpers/orcad-convert-flow'
import { shellQuote } from './helpers/docker-ssh-relay-target'
import { callEnvironment, createPairedHostTerminal } from './helpers/paired-host-terminal'
import { reserveLocalPort } from './helpers/ssh-port-forward-snapshot-barrier'
import { openTerminalWorkspaceRootLink } from './helpers/terminal-workspace-root-link'
import {
echoMarker,
expectTerminalAnswers,
launchManagedWorkspace,
openTerminalTab,
ORCAD_TEMPLATE,
typeInTerminal,
withManagedHost
} from './helpers/orcad-managed-workspace'
test.skip(
!ORCAD_TEMPLATE || process.env.ORCA_E2E_SSH_DOCKER !== '1',
'Needs Docker and server template'
)
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
test('Quick Open finds a file in a monorepo-sized managed workspace', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(10 * 60_000)
await withManagedHost(testInfo, async (fixture) => {
const repo = shellQuote(fixture.host.remoteRepoPath)
// ~22.6k tracked paths: past the size whose single reply once overflowed the transport (#12547).
fixture.host.exec!(
[
`cd ${repo}`,
'for d in $(seq 1 226); do mkdir -p packages/pkg-$d/src && (cd packages/pkg-$d/src && ' +
'touch $(seq -f "module-with-a-long-descriptive-name-%05g.ts" 1 100)); done',
'touch packages/pkg-200/src/quick-open-needle-target.ts',
'git add -A',
'git -c user.email=e2e@test.local -c user.name=e2e commit -q -m "monorepo-sized tree"'
].join(' && ')
)
const { page } = await launchManagedWorkspace(fixture)
await page.evaluate(() => window.__store?.getState().openModal('quick-open'))
const dialog = page.getByRole('dialog', { name: 'Go to file' })
await expect(dialog).toBeVisible()
const input = dialog.getByPlaceholder('Go to file...')
await input.fill('quick-open-needle-target')
await expect(
dialog.getByRole('option').filter({ hasText: 'quick-open-needle-target.ts' })
).toHaveCount(1, { timeout: 90_000 })
await input.fill('module-with-a-long-descriptive-name-00042')
await expect(dialog.getByRole('option').first()).toBeVisible({ timeout: 60_000 })
await expect(dialog.getByText('Loading files...')).toHaveCount(0)
await expect(dialog).not.toContainText('Remote Orca runtime closed the connection')
await page.screenshot({ path: testInfo.outputPath('managed-quick-open.png') })
})
})
for (const acrossReconnect of [false, true]) {
test(`a managed host reports its listener, and an SSH forward reaches it${acrossReconnect ? ' across a reconnect' : ''}`, async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
// Why fixme: saved forwards are re-established only by the relay session's connect callbacks.
test.fixme(acrossReconnect, 'A managed host reconnect never calls restorePortForwards')
test.setTimeout(10 * 60_000)
const remotePort = 7860
const marker = `ORCA_FORWARD_${Date.now()}`
const reservation = await reserveLocalPort()
await withManagedHost(testInfo, async (fixture) => {
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
const script = `require('node:http').createServer((_q, r) => r.end(${JSON.stringify(marker)})).listen(${remotePort}, '127.0.0.1')`
// Started from the workspace so the scan attributes it to this workspace.
fixture.host.exec!(
`cd ${shellQuote(fixture.host.remoteRepoPath)} && nohup node -e ${shellQuote(script)} >/tmp/orca-http.log 2>&1 < /dev/null &`
)
// The scan the Ports panel shows for a workspace on this host.
await expect
.poll(
async () =>
JSON.stringify(await callEnvironment(page, environmentId, 'workspacePorts.scan', {})),
{ timeout: 60_000, message: 'the managed host never reported the listener' }
)
.toContain(`"port":${remotePort}`)
const localPort = reservation.port
await reservation.release()
const forward = await page.evaluate(
(args) => window.api.ssh.addPortForward({ ...args, remoteHost: '127.0.0.1' }),
{ targetId: seeded.targetId, localPort, remotePort, label: 'managed e2e' }
)
const fetchMarker = async (): Promise<string> => {
try {
return await (await fetch(`http://127.0.0.1:${localPort}/`)).text()
} catch (error) {
return String(error)
}
}
await expect.poll(fetchMarker, { timeout: 30_000 }).toBe(marker)
if (acrossReconnect) {
await reconnect(page, seeded.targetId)
await expect.poll(fetchMarker, { timeout: 60_000 }).toBe(marker)
}
await page.evaluate((id) => window.api.ssh.removePortForward({ id }), forward.id)
await expect.poll(fetchMarker, { timeout: 30_000 }).not.toBe(marker)
}).finally(() => reservation.release())
})
}
test('a terminal link opens an image outside the managed workspace', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
// Why fixme: in-workspace links work (ssh-orcad-terminal-root-owner); this path gets no link at all.
test.fixme(true, 'A managed terminal does not linkify a path outside its workspace')
test.setTimeout(10 * 60_000)
const imagePath = '/tmp/orca-ssh-external-preview.png'
const imageBase64 =
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFklEQVR4AWN8z8DwnwEJMDGgAcICAO2mBAXmO4drAAAAAElFTkSuQmCC'
await withManagedHost(testInfo, async (fixture) => {
fixture.host.exec!(
`printf '%s' ${shellQuote(imageBase64)} | base64 -d > ${shellQuote(imagePath)}`
)
const { page, seeded } = await launchManagedWorkspace(fixture)
// A link is only clickable once the shell has printed it, so wait for one that answers.
await expectTerminalAnswers(page, await openTerminalTab(page, seeded.worktreeId))
await openTerminalWorkspaceRootLink(page, testInfo, imagePath, 'open')
const preview = page.locator(`img[alt="${imagePath.split('/').at(-1)}"]`)
await expect(preview).toBeVisible({ timeout: 30_000 })
expect(await preview.evaluate((element: HTMLImageElement) => element.naturalWidth)).toBe(2)
await expect(page.getByText('Unable to load file', { exact: true })).toHaveCount(0)
await page.screenshot({ path: testInfo.outputPath('managed-external-image.png') })
})
})
test('a managed terminal reports agent status from hooks and from Pi-style titles', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(10 * 60_000)
await withManagedHost(testInfo, async (fixture) => {
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
const tabId = await openTerminalTab(page, seeded.worktreeId)
await echoMarker(page, tabId, 'HOOK_READY')
// Backgrounded: a foreground post ends the shell command, which rightly clears the turn.
const prompt = `orca managed hook ${Date.now()}`
const payload = JSON.stringify({ hook_event_name: 'UserPromptSubmit', prompt })
await typeInTerminal(
page,
tabId,
[
'( sleep 0.2; curl -sS -X POST "http://127.0.0.1:${ORCA_AGENT_HOOK_PORT}/hook/codex"',
'-H "X-Orca-Agent-Hook-Token: ${ORCA_AGENT_HOOK_TOKEN}"',
'--data-urlencode "paneKey=${ORCA_PANE_KEY}" --data-urlencode "tabId=${ORCA_TAB_ID}"',
'--data-urlencode "worktreeId=${ORCA_WORKTREE_ID}" --data-urlencode "env=${ORCA_AGENT_HOOK_ENV}"',
'--data-urlencode "version=${ORCA_AGENT_HOOK_VERSION}"',
`--data-urlencode ${shellQuote(`payload=${payload}`)} >/tmp/orca-hook.log 2>&1 ) &`
].join(' ')
)
await expect
.poll(
() =>
page.evaluate(
(prompt) =>
Object.values(window.__store?.getState().agentStatusByPaneKey ?? {}).some(
(entry) =>
entry.prompt === prompt &&
entry.agentType === 'codex' &&
entry.state === 'working'
),
prompt
),
{ timeout: 30_000, message: 'the host hook never reached the agent status store' }
)
.toBe(true)
// The host classifies agent titles a remote terminal emits.
const terminal = await createPairedHostTerminal(page, environmentId, seeded.worktreeId, 'bash')
const status = async (): Promise<unknown> => {
const result = await callEnvironment(page, environmentId, 'terminal.agentStatus', {
terminal: terminal.terminal
})
return typeof result === 'object' && result !== null && 'agentStatus' in result
? result.agentStatus
: result
}
for (const [title, expected] of [
['⠋ OMP', 'working'],
['OMP ready', 'idle'],
['⠋ Pi', 'working']
] as const) {
await callEnvironment(page, environmentId, 'terminal.send', {
terminal: terminal.terminal,
text: `printf '\\033]0;${title}\\007'`,
enter: true
})
await expect
.poll(status, { timeout: 15_000, message: `title ${title}` })
.toMatchObject({ isRunningAgent: true, status: expected })
}
})
})