mirror of
https://github.com/stablyai/orca.git
synced 2026-10-27 08:02:51 +00:00
fix(remote): recover stalled terminal streams
This commit is contained in:
OrcaWin
+233
-17
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"updatedAt": "2026-07-28",
|
||||
"updatedAt": "2026-07-29",
|
||||
"policy": {
|
||||
"maturityLevels": [
|
||||
"experimental",
|
||||
@@ -3248,6 +3248,199 @@
|
||||
],
|
||||
"demotionRule": "Keep experimental or demote to protection none if output differs across replay slices, header-only restores consume a replay slot, retained payload bytes exceed the cap, or the focused gate flakes."
|
||||
},
|
||||
{
|
||||
"id": "terminal-performance.remote-hidden-retention-budget",
|
||||
"title": "Paired remote terminals paint retained output and bound hidden renderer state",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "terminal-runtime",
|
||||
"layer": "paired-headed-runtime",
|
||||
"surfaces": [
|
||||
"paired remote terminal first paint",
|
||||
"stalled paired terminal stream recovery",
|
||||
"hidden remote worktree retention",
|
||||
"remote terminal reveal and input",
|
||||
"manual server disconnect"
|
||||
],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["paired-runtime", "ssh"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["paired-runtime"],
|
||||
"coverageNotes": "Deterministic headed macOS runs launch an isolated Orca desktop server and a separate paired web client. They prove a truncated retained history still paints its latest-screen snapshot; an ACK-starved stream can stop client delivery while its host PTY cursor advances, then recover without replacing the PTY; and six real paired host-backed worktrees with bounded scrollback shrink to the configured manager/cell budget before reveal restores the original PTY and live input. The same stream-stall oracle passes against an isolated headless `orca serve` host. A separate headed memory test measures staged xterm buffer release for remote-shaped terminals. Unit tests cover stream timeouts and diagnostic reasons, SSH and paired classification, plus desktop and browser manual-disconnect fencing. Linux, Windows, mixed-version paired hosts, live SSH, headless retention, and production-scale paired hosts remain gaps.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/issues/8652",
|
||||
"https://github.com/stablyai/orca/pull/10625"
|
||||
],
|
||||
"invariant": "Historical retained-tail overflow must not invalidate a complete serialized or latest-screen first-paint snapshot. A paired terminal stream whose delivery credits or command responses stop progressing must replace only that stream, preserve its tab and PTY identity, and repaint authoritative host output. With terminal parking enabled, hidden unparkable SSH and paired-runtime worktrees must retain at most the configured count and none beyond the TTL, including paired tabs with a host-backed remote PTY and stale activation residue. Force-parking must preserve the captured screen and later PTY input/output. Manual disconnect must retain pairing while preventing passive calls and subscriptions from recreating transport until explicit Connect.",
|
||||
"oracle": "Launch a headed desktop host and a separate paired client. Produce more than the retained terminal-read limit, require terminal.read to report truncated, reload the client, and require the final marker to paint before new output. Separately withhold ACKs until one stream exhausts its 512 KiB window, require the host to finish bounded output and advance its cursor after accepted input while the client lacks the marker, then require command-response recovery to repaint it with the same tab and PTY. Run the byte-identical oracle with recovery disabled and require it to stay stalled. Create six live host PTYs in distinct paired worktrees, stage bounded high-output scrollback in every mounted xterm, disable the budget and require all six managers to survive, then enable a limit of two and require exactly two managers plus at most 45% of staged buffer cells to remain without material heap growth or a renderer timer stall. Reveal an evicted tab, require the same PTY identity and fresh live echo. Disconnect transport, exercise passive status, call, and subscribe paths, and require no client reconstruction until explicit Connect.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts",
|
||||
"ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_REMOTE_TERMINAL_STALL_RECOVERY=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"SKIP_BUILD=1 pnpm exec playwright test tests/e2e/terminal-parked-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/renderer/src/web/web-preload-api.test.ts",
|
||||
"src/main/ipc/runtime-environments.test.ts",
|
||||
"src/main/runtime/rpc/terminal-multiplex.test.ts",
|
||||
"src/main/runtime/rpc/terminal-subscribe-buffer.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts",
|
||||
"src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts",
|
||||
"src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts",
|
||||
"tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts",
|
||||
"tests/e2e/paired-remote-terminal-stall-recovery.spec.ts",
|
||||
"tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts",
|
||||
"tests/e2e/paired-remote-terminal-retention-memory.spec.ts",
|
||||
"tests/e2e/terminal-parked-memory.spec.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "tests/e2e/paired-remote-terminal-stall-recovery.spec.ts",
|
||||
"assertions": [
|
||||
"restarts one ACK-starved paired terminal stream without replacing its PTY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts",
|
||||
"assertions": [
|
||||
"recovers an ACK-starved stream from an isolated headless Orca host"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts",
|
||||
"assertions": [
|
||||
"paints a paired remote terminal when only its retained text tail overflowed",
|
||||
"bounds hidden paired-runtime pane managers and restores PTY output"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/paired-remote-terminal-retention-memory.spec.ts",
|
||||
"assertions": [
|
||||
"bounds retained buffers across real paired worktrees and preserves PTYs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/terminal-parked-memory.spec.ts",
|
||||
"assertions": [
|
||||
"releases un-parkable hidden worktree buffers only once the retention budget engages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts",
|
||||
"assertions": [
|
||||
"treats a host-backed paired PTY as settled despite activation residue",
|
||||
"preserves real startup work and non-paired activation guards",
|
||||
"force-parks the least-recently-hidden candidates beyond the retention limit"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/terminal-multiplex.test.ts",
|
||||
"assertions": [
|
||||
"binary first paint remains valid when only retained history was truncated"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/web/web-preload-api.test.ts",
|
||||
"assertions": [
|
||||
"keeps pairing while manual disconnect fences passive reconnects",
|
||||
"fences a web runtime response that completes after manual disconnect"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 5.98,
|
||||
"summary": "Six focused files passed 202 tests covering transport truncation semantics, paired retention classification, deterministic overrides, and desktop/browser disconnect fencing."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"result": "passed",
|
||||
"durationSeconds": 18.5,
|
||||
"summary": "Both headed paired-server scenarios passed: truncated retained history painted after reload, and the retention control/candidate run evicted exactly the older hidden manager before successful reveal and live PTY input."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_REMOTE_TERMINAL_STALL_RECOVERY=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"result": "failed",
|
||||
"durationSeconds": 36.6,
|
||||
"summary": "With stream recovery disabled, the host cursor advanced after accepted input while the paired client remained frozen around flood row 5,396 and never painted the later marker."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"result": "passed",
|
||||
"durationSeconds": 18,
|
||||
"summary": "The byte-identical headed oracle exhausted one paired stream, proved host/client divergence, then repainted the marker while preserving the original PTY and tab."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"result": "passed",
|
||||
"durationSeconds": 14.8,
|
||||
"summary": "Six real paired worktrees and host PTYs retained bounded high-output client buffers with the budget disabled, shrank to exactly two managers and under 45% of staged cells when enabled, stayed within heap and timer-lag budgets, then restored live output on the original evicted PTY."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
|
||||
"result": "passed",
|
||||
"durationSeconds": 14.4,
|
||||
"summary": "An isolated `orca serve` host and paired web renderer exhausted one terminal stream, recovered it, and preserved the original PTY and tab without exposing readiness or pairing material."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 360,
|
||||
"scope": "focused units plus isolated headed paired-runtime and renderer-memory scenarios"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "unknown",
|
||||
"evidence": "The paired headed oracle is deterministic locally but has no CI or soak history yet."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "complete",
|
||||
"evidence": "Before the snapshot fix, the headed paired reload stayed blank because retained-tail truncation invalidated first paint. Before the paired-retention follow-up, the live control/candidate oracle kept remote managers mounted because stale pendingActivationSpawn residue excluded host-backed worktrees. With stalled-stream recovery disabled, the host cursor advanced but the client remained frozen; enabling it repainted the marker with the same PTY and tab. The six-worktree retention oracle keeps all managers and staged cells when the budget is disabled, then releases four managers and their buffers when enabled."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "The production policy retains at most 12 hidden unparkable worktrees and force-parks any hidden beyond 45 minutes. Stream recovery adds no polling: timers exist only while an entered command awaits a frame or renderer delivery credit remains unsettled, and restart is scoped to one stream. The headed paired memory oracle requires exact manager reduction, at least 55% staged xterm-cell release, no more than 16 MiB heap growth, and under 500 ms timer drift; all paired oracles use bounded output and deterministic barriers rather than an overnight sleep."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect 100 consecutive CI passes or 14 days of soak history for both headed commands.",
|
||||
"Run the paired topology on Windows and Linux and add one live SSH retention run.",
|
||||
"Run the stream-stall oracle with a v1.4.160-rc.3 host and v1.4.160-rc.4 client.",
|
||||
"Add a production-scale paired run with dozens of worktrees and explicit event-loop and renderer-memory budgets.",
|
||||
"Keep the budget-off control, exact manager-count assertion, reveal marker, and post-reveal PTY input in one headed run."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Remote reveal restores only the serialized screen or captured tail because no full remote scrollback RPC exists.",
|
||||
"The scaled paired headed retention scenario uses six worktrees and a cap of two; it proves real host-backed eviction and buffer release, not a 100-worktree soak or the production cap of 12.",
|
||||
"Display-off reveal latency is a separate atlas and viewport-reflow class and is not covered by this gate.",
|
||||
"Historical field trace archives are still required to order replay-wedge events against renderer heartbeat loss and deduplicate archived records.",
|
||||
"The live windows-issues incident proves a healthy host PTY and dead existing client stream, but lacks per-stream ACK counters; ACK starvation is reproduced, not yet proven as that field incident's unique boundary."
|
||||
],
|
||||
"demotionRule": "Keep experimental or demote to protection none if paired first paint is blank, passive work reconnects after manual disconnect, an eligible hidden remote manager survives the cap or TTL, reveal loses the captured screen or live PTY, retained buffer cells do not fall, or either headed scenario flakes."
|
||||
},
|
||||
{
|
||||
"id": "terminal-performance.no-hot-list-sessions",
|
||||
"title": "Hot terminal interactions do not call global PTY session listing",
|
||||
@@ -3376,13 +3569,14 @@
|
||||
"id": "terminal-observability.lifecycle-breadcrumbs",
|
||||
"title": "Terminal lifecycle anomalies enter crash diagnostics as compact breadcrumbs",
|
||||
"maturity": "experimental",
|
||||
"protection": "none",
|
||||
"protection": "partial",
|
||||
"owner": "terminal-runtime",
|
||||
"layer": "renderer-observability",
|
||||
"surfaces": [
|
||||
"terminal lifecycle",
|
||||
"reattach",
|
||||
"restore",
|
||||
"replay-wedge identity",
|
||||
"provider ownership",
|
||||
"diagnostics bundle"
|
||||
],
|
||||
@@ -3398,34 +3592,56 @@
|
||||
"wsl",
|
||||
"remote-runtime"
|
||||
],
|
||||
"coveredPlatforms": [],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": [],
|
||||
"coverageNotes": "Registered gap on main. The crash-breadcrumb recording and its test exist only on the pending reliability stack. It registers here with its owning split PR.",
|
||||
"coverageNotes": "Renderer unit coverage proves replay-guard lost-completion and certified-wedge events carry correlatable tab, worktree, durable leaf, pane, and redacted PTY identity without exposing path-bearing values. Full provider lifecycle attribution and diagnostics-bundle artifact proof remain gaps.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/pull/6800",
|
||||
"https://github.com/stablyai/orca/issues/6773"
|
||||
],
|
||||
"invariant": "Terminal lifecycle anomalies around reattach, restore, provider ownership, stale liveness, and fallback routing must leave compact, deduped, privacy-safe breadcrumbs in crash diagnostics so future reports can be attributed from evidence.",
|
||||
"oracle": "The current executable slice calls warnTerminalLifecycleAnomaly with terminal identity, provider, PTY id, binding epoch, and reason, then asserts the existing console warning is preserved and a compact terminal_lifecycle_anomaly crash breadcrumb is recorded once per lifecycle identity. Full pane transition traces and diagnostics-bundle artifact proof remain follow-ups.",
|
||||
"commands": [],
|
||||
"testFiles": [],
|
||||
"assertionRefs": [],
|
||||
"evidenceRuns": [],
|
||||
"invariant": "Terminal lifecycle anomalies around reattach, restore, replay wedges, provider ownership, stale liveness, and fallback routing must leave compact, deduped, privacy-safe breadcrumbs in crash diagnostics so future reports can be attributed from evidence. Replay anomalies must distinguish pane managers and PTYs without recording path-bearing worktree or session identities.",
|
||||
"oracle": "Drop a replay write completion while allowing its FIFO probe to parse, then require the lost-completion breadcrumb to include pane ID, stable hashes for tab/worktree/leaf identity, and a path-redacted PTY ID. Existing wedge tests require both lost-completion and certified-dead paths to record their distinct event names. Full pane transition traces and diagnostics-bundle artifact proof remain follow-ups.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/replay-guard.test.ts"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/renderer/src/components/terminal-pane/replay-guard.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/replay-guard.test.ts",
|
||||
"assertions": [
|
||||
"records correlatable replay identity without exposing worktree or PTY paths",
|
||||
"releases after the probe itself never parses (wedged pipeline) and reports it"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-07-29",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/replay-guard.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 0.201,
|
||||
"summary": "The focused file passed 29 tests; replay anomaly breadcrumbs preserved event classification while adding hashed tab/worktree/leaf correlation and a path-redacted PTY identity."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 10,
|
||||
"scope": "renderer observability unit test"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "unknown",
|
||||
"evidence": "Focused unit slice passed locally once; no CI soak history yet."
|
||||
"evidence": "The replay-guard unit slice passed locally once; no CI soak history yet."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "Tests would fail if lifecycle anomalies stopped recording crash breadcrumbs or stopped deduping repeated identities. Needs diagnostics-bundle artifact proof and full transition-trace evidence before promotion."
|
||||
"evidence": "Tests fail if replay anomalies stop recording their event name, omit pane/terminal correlation, or expose the fixture's path-bearing worktree and PTY prefixes. Needs diagnostics-bundle artifact proof and full transition-trace evidence before promotion."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Breadcrumb recording is deduped and capped by the existing lifecycle anomaly guard; full trace buffers must include size and event-count caps before promotion."
|
||||
"evidence": "Replay identity hashing is synchronous and only runs when replay writes are queued; it adds no polling or provider calls. Breadcrumb storage remains bounded by the existing crash reporter. Full trace buffers must include size and event-count caps before promotion."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Add full compact pane lifecycle trace buffer with event-count caps.",
|
||||
@@ -3433,10 +3649,10 @@
|
||||
"Add forbidden-transition tests for stale close, unknown owner fallback, and stuck zero-size panes."
|
||||
],
|
||||
"knownGaps": [
|
||||
"No executable coverage on main yet; the slice lives on the pending fix-terminal-reliability stack.",
|
||||
"Current command records anomaly breadcrumbs only, not a full pane lifecycle state-machine trace.",
|
||||
"Current command does not prove crash/diagnostics bundle export includes the breadcrumb.",
|
||||
"Current command does not assert forbidden transitions across live Electron/provider flows."
|
||||
"The replay identity schema has not yet been exercised in a live Electron/provider failure.",
|
||||
"Current coverage records anomaly breadcrumbs only, not a full pane lifecycle state-machine trace.",
|
||||
"Current coverage does not prove crash/diagnostics bundle export includes the breadcrumb.",
|
||||
"Current coverage does not assert forbidden transitions across live Electron/provider flows."
|
||||
],
|
||||
"demotionRule": "Cannot promote if diagnostics are console-only, unbounded, or missing from support artifacts."
|
||||
},
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
import { ipcMain } from 'electron'
|
||||
import {
|
||||
addEnvironmentFromPairingCode,
|
||||
listEnvironments,
|
||||
removeEnvironment,
|
||||
resolveEnvironment
|
||||
} from '../../shared/runtime-environment-store'
|
||||
import {
|
||||
redactRuntimeEnvironment,
|
||||
type PublicKnownRuntimeEnvironment
|
||||
} from '../../shared/runtime-environments'
|
||||
import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope'
|
||||
import type { RuntimeStatus } from '../../shared/runtime-types'
|
||||
import type { Store } from '../persistence'
|
||||
import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections'
|
||||
import {
|
||||
callRuntimeEnvironment,
|
||||
clearSharedControlSupport,
|
||||
getRuntimeEnvironmentStatus
|
||||
} from './runtime-environment-transport-routing'
|
||||
|
||||
const manuallyDisconnectedEnvironmentIds = new Set<string>()
|
||||
|
||||
function manuallyDisconnectedResponse(
|
||||
environment: ReturnType<typeof resolveEnvironment>
|
||||
): RuntimeRpcResponse<never> {
|
||||
return {
|
||||
id: 'runtime.manualDisconnect',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'runtime_manually_disconnected',
|
||||
message: 'Runtime environment is manually disconnected.'
|
||||
},
|
||||
_meta: { runtimeId: environment.runtimeId }
|
||||
}
|
||||
}
|
||||
|
||||
export function isRuntimeEnvironmentManuallyDisconnected(environmentId: string): boolean {
|
||||
return manuallyDisconnectedEnvironmentIds.has(environmentId)
|
||||
}
|
||||
|
||||
type ConnectivityHandlerOptions = {
|
||||
store: Store
|
||||
getUserDataPath: () => string
|
||||
invalidateTransport: (environmentId: string) => void
|
||||
}
|
||||
|
||||
export function registerRuntimeEnvironmentConnectivityHandlers({
|
||||
store,
|
||||
getUserDataPath,
|
||||
invalidateTransport
|
||||
}: ConnectivityHandlerOptions): void {
|
||||
ipcMain.handle('runtimeEnvironments:list', () =>
|
||||
listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment)
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:addFromPairingCode',
|
||||
(
|
||||
_event,
|
||||
args: { name: string; pairingCode: string }
|
||||
): { environment: PublicKnownRuntimeEnvironment } => {
|
||||
const environment = addEnvironmentFromPairingCode(getUserDataPath(), args)
|
||||
manuallyDisconnectedEnvironmentIds.delete(environment.id)
|
||||
return { environment: redactRuntimeEnvironment(environment) }
|
||||
}
|
||||
)
|
||||
ipcMain.handle('runtimeEnvironments:resolve', (_event, args: { selector: string }) =>
|
||||
redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector))
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:remove',
|
||||
(_event, args: { selector: string }): { removed: PublicKnownRuntimeEnvironment } => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
if (store.getSettings().activeRuntimeEnvironmentId === environment.id) {
|
||||
throw new Error('Choose another Active Server in Advanced before removing this server.')
|
||||
}
|
||||
const removed = removeEnvironment(getUserDataPath(), args.selector)
|
||||
manuallyDisconnectedEnvironmentIds.delete(removed.id)
|
||||
invalidateTransport(removed.id)
|
||||
closeLegacySelectorTransport(args.selector, removed.id)
|
||||
return { removed: redactRuntimeEnvironment(removed) }
|
||||
}
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:disconnect',
|
||||
(_event, args: { selector: string }): { disconnected: PublicKnownRuntimeEnvironment } => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
manuallyDisconnectedEnvironmentIds.add(environment.id)
|
||||
invalidateTransport(environment.id)
|
||||
closeLegacySelectorTransport(args.selector, environment.id)
|
||||
return { disconnected: redactRuntimeEnvironment(environment) }
|
||||
}
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:connect',
|
||||
async (
|
||||
_event,
|
||||
args: { selector: string; timeoutMs?: number }
|
||||
): Promise<RuntimeRpcResponse<RuntimeStatus>> => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
manuallyDisconnectedEnvironmentIds.delete(environment.id)
|
||||
return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export function registerRuntimeEnvironmentPassiveHandlers(getUserDataPath: () => string): void {
|
||||
registerPassiveStatusHandler(getUserDataPath)
|
||||
registerPassiveCallHandler(getUserDataPath)
|
||||
}
|
||||
|
||||
function closeLegacySelectorTransport(selector: string, environmentId: string): void {
|
||||
if (selector === environmentId) {
|
||||
return
|
||||
}
|
||||
closeRemoteRuntimeRequestConnection(selector)
|
||||
clearSharedControlSupport(selector)
|
||||
}
|
||||
|
||||
function registerPassiveStatusHandler(getUserDataPath: () => string): void {
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:getStatus',
|
||||
async (
|
||||
_event,
|
||||
args: { selector: string; timeoutMs?: number }
|
||||
): Promise<RuntimeRpcResponse<RuntimeStatus>> => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
const response = await getRuntimeEnvironmentStatus(
|
||||
getUserDataPath(),
|
||||
environment.id,
|
||||
args.timeoutMs
|
||||
)
|
||||
return isRuntimeEnvironmentManuallyDisconnected(environment.id)
|
||||
? manuallyDisconnectedResponse(environment)
|
||||
: response
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function registerPassiveCallHandler(getUserDataPath: () => string): void {
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:call',
|
||||
async (
|
||||
_event,
|
||||
args: {
|
||||
selector: string
|
||||
method: string
|
||||
params?: unknown
|
||||
timeoutMs?: number
|
||||
expectedEnvironmentPairingRevision?: number
|
||||
}
|
||||
): Promise<RuntimeRpcResponse<unknown>> => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
const response = await callRuntimeEnvironment(
|
||||
getUserDataPath(),
|
||||
environment.id,
|
||||
args.method,
|
||||
args.params,
|
||||
args.timeoutMs,
|
||||
args.expectedEnvironmentPairingRevision
|
||||
)
|
||||
return isRuntimeEnvironmentManuallyDisconnected(environment.id)
|
||||
? manuallyDisconnectedResponse(environment)
|
||||
: response
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -135,6 +135,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
'runtimeEnvironments:resolve',
|
||||
'runtimeEnvironments:remove',
|
||||
'runtimeEnvironments:disconnect',
|
||||
'runtimeEnvironments:connect',
|
||||
'runtimeEnvironments:retryConnectionsNow',
|
||||
'runtimeEnvironments:getStatus',
|
||||
'runtimeEnvironments:call',
|
||||
@@ -155,6 +156,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
'runtimeEnvironments:resolve',
|
||||
'runtimeEnvironments:remove',
|
||||
'runtimeEnvironments:disconnect',
|
||||
'runtimeEnvironments:connect',
|
||||
'runtimeEnvironments:getStatus',
|
||||
'runtimeEnvironments:call',
|
||||
'runtimeEnvironments:subscribe',
|
||||
@@ -230,6 +232,12 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
|
||||
it('disconnects a saved runtime without removing it', async () => {
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
sendRemoteRuntimeRequestMock.mockResolvedValue({
|
||||
id: 'status',
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-remote' },
|
||||
_meta: { runtimeId: 'runtime-remote' }
|
||||
})
|
||||
|
||||
const add = handler<
|
||||
{ name: string; pairingCode: string },
|
||||
@@ -250,6 +258,39 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
|
||||
const list = handler<undefined, { id: string; name: string }[]>('runtimeEnvironments:list')
|
||||
expect(await list(null, undefined)).toMatchObject([{ id: added.environment.id, name: 'desk' }])
|
||||
|
||||
const getStatus = handler<{ selector: string }, { ok: boolean; error?: { code: string } }>(
|
||||
'runtimeEnvironments:getStatus'
|
||||
)
|
||||
await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'runtime_manually_disconnected' }
|
||||
})
|
||||
const call = handler<
|
||||
{ selector: string; method: string },
|
||||
{ ok: boolean; error?: { code: string } }
|
||||
>('runtimeEnvironments:call')
|
||||
await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'runtime_manually_disconnected' }
|
||||
})
|
||||
const subscribe = handler<{ selector: string; method: string }, { subscriptionId: string }>(
|
||||
'runtimeEnvironments:subscribe'
|
||||
)
|
||||
await expect(
|
||||
subscribe(null, { selector: 'desk', method: 'terminal.multiplex' })
|
||||
).rejects.toThrow('runtime_manually_disconnected')
|
||||
expect(sendRemoteRuntimeRequestMock).not.toHaveBeenCalled()
|
||||
expect(subscribeRemoteRuntimeRequestMock).not.toHaveBeenCalled()
|
||||
|
||||
const connect = handler<{ selector: string }, { ok: boolean; result?: { runtimeId: string } }>(
|
||||
'runtimeEnvironments:connect'
|
||||
)
|
||||
await expect(connect(null, { selector: 'desk' })).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-remote' }
|
||||
})
|
||||
expect(sendRemoteRuntimeRequestMock).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('marks environments owned by ephemeral VM runtimes in the public list', async () => {
|
||||
@@ -1155,9 +1196,12 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
{ disconnected: { id: string; name: string } }
|
||||
>('runtimeEnvironments:disconnect')
|
||||
await disconnect(null, { selector: 'desk' })
|
||||
const connect = handler<{ selector: string }, { ok: boolean }>('runtimeEnvironments:connect')
|
||||
await connect(null, { selector: 'desk' })
|
||||
await call(null, { selector: 'desk', method: 'repo.list' })
|
||||
|
||||
expect(sendRemoteRuntimeRequestMock.mock.calls.map((call) => call[1])).toEqual([
|
||||
'status.get',
|
||||
'status.get',
|
||||
'status.get'
|
||||
])
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
import { app, ipcMain } from 'electron'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
addEnvironmentFromPairingCode,
|
||||
listEnvironments,
|
||||
removeEnvironment,
|
||||
resolveEnvironment
|
||||
} from '../../shared/runtime-environment-store'
|
||||
import {
|
||||
redactRuntimeEnvironment,
|
||||
type PublicKnownRuntimeEnvironment
|
||||
} from '../../shared/runtime-environments'
|
||||
import type { RuntimeStatus } from '../../shared/runtime-types'
|
||||
import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope'
|
||||
import { resolveEnvironment } from '../../shared/runtime-environment-store'
|
||||
import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client'
|
||||
import type { Store } from '../persistence'
|
||||
import {
|
||||
isRuntimeEnvironmentManuallyDisconnected,
|
||||
registerRuntimeEnvironmentConnectivityHandlers,
|
||||
registerRuntimeEnvironmentPassiveHandlers
|
||||
} from './runtime-environment-connectivity-handlers'
|
||||
import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections'
|
||||
import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler'
|
||||
import {
|
||||
@@ -21,9 +15,7 @@ import {
|
||||
getRuntimeEnvironmentTransportGeneration
|
||||
} from './runtime-environment-transport-generation'
|
||||
import {
|
||||
callRuntimeEnvironment,
|
||||
clearSharedControlSupport,
|
||||
getRuntimeEnvironmentStatus,
|
||||
resetSharedControlSupport,
|
||||
subscribeRuntimeEnvironment
|
||||
} from './runtime-environment-transport-routing'
|
||||
@@ -34,6 +26,7 @@ const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [
|
||||
'runtimeEnvironments:resolve',
|
||||
'runtimeEnvironments:remove',
|
||||
'runtimeEnvironments:disconnect',
|
||||
'runtimeEnvironments:connect',
|
||||
'runtimeEnvironments:getStatus',
|
||||
'runtimeEnvironments:call',
|
||||
'runtimeEnvironments:subscribe',
|
||||
@@ -66,11 +59,6 @@ export function invalidateRuntimeEnvironmentTransport(environmentId: string): vo
|
||||
closeSubscriptionsForEnvironment(environmentId)
|
||||
}
|
||||
|
||||
function listPublicRuntimeEnvironments(): PublicKnownRuntimeEnvironment[] {
|
||||
// Why: a corrupt VM store must not break persisted environment listing.
|
||||
return listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment)
|
||||
}
|
||||
|
||||
export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
// Why: keep direct re-registration safe even though register-core-handlers
|
||||
// normally guards this path; otherwise the binary send listener can stack.
|
||||
@@ -80,81 +68,13 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
}
|
||||
ipcMain.removeAllListeners('runtimeEnvironments:subscriptionBinary')
|
||||
|
||||
ipcMain.handle('runtimeEnvironments:list', listPublicRuntimeEnvironments)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:addFromPairingCode',
|
||||
(
|
||||
_event,
|
||||
args: { name: string; pairingCode: string }
|
||||
): { environment: PublicKnownRuntimeEnvironment } => ({
|
||||
environment: redactRuntimeEnvironment(addEnvironmentFromPairingCode(getUserDataPath(), args))
|
||||
})
|
||||
)
|
||||
ipcMain.handle('runtimeEnvironments:resolve', (_event, args: { selector: string }) =>
|
||||
redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector))
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:remove',
|
||||
(_event, args: { selector: string }): { removed: PublicKnownRuntimeEnvironment } => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
if (store.getSettings().activeRuntimeEnvironmentId === environment.id) {
|
||||
throw new Error('Choose another Active Server in Advanced before removing this server.')
|
||||
}
|
||||
const removed = removeEnvironment(getUserDataPath(), args.selector)
|
||||
invalidateRuntimeEnvironmentTransport(removed.id)
|
||||
if (args.selector !== removed.id) {
|
||||
closeRemoteRuntimeRequestConnection(args.selector)
|
||||
clearSharedControlSupport(args.selector)
|
||||
}
|
||||
return { removed: redactRuntimeEnvironment(removed) }
|
||||
}
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:disconnect',
|
||||
(_event, args: { selector: string }): { disconnected: PublicKnownRuntimeEnvironment } => {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
// Why: disconnect is intentionally non-destructive; it drops live
|
||||
// transport state while keeping the paired server available for later.
|
||||
invalidateRuntimeEnvironmentTransport(environment.id)
|
||||
if (args.selector !== environment.id) {
|
||||
closeRemoteRuntimeRequestConnection(args.selector)
|
||||
clearSharedControlSupport(args.selector)
|
||||
}
|
||||
return { disconnected: redactRuntimeEnvironment(environment) }
|
||||
}
|
||||
)
|
||||
registerRuntimeEnvironmentConnectivityHandlers({
|
||||
store,
|
||||
getUserDataPath,
|
||||
invalidateTransport: invalidateRuntimeEnvironmentTransport
|
||||
})
|
||||
registerRuntimeEnvironmentRecoveryHandler()
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:getStatus',
|
||||
async (
|
||||
_event,
|
||||
args: { selector: string; timeoutMs?: number }
|
||||
): Promise<RuntimeRpcResponse<RuntimeStatus>> => {
|
||||
return getRuntimeEnvironmentStatus(getUserDataPath(), args.selector, args.timeoutMs)
|
||||
}
|
||||
)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:call',
|
||||
async (
|
||||
_event,
|
||||
args: {
|
||||
selector: string
|
||||
method: string
|
||||
params?: unknown
|
||||
timeoutMs?: number
|
||||
expectedEnvironmentPairingRevision?: number
|
||||
}
|
||||
): Promise<RuntimeRpcResponse<unknown>> => {
|
||||
return callRuntimeEnvironment(
|
||||
getUserDataPath(),
|
||||
args.selector,
|
||||
args.method,
|
||||
args.params,
|
||||
args.timeoutMs,
|
||||
args.expectedEnvironmentPairingRevision
|
||||
)
|
||||
}
|
||||
)
|
||||
registerRuntimeEnvironmentPassiveHandlers(getUserDataPath)
|
||||
ipcMain.handle(
|
||||
'runtimeEnvironments:subscribe',
|
||||
async (
|
||||
@@ -176,6 +96,9 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
throw new Error('Runtime environment subscription id already exists')
|
||||
}
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
|
||||
throw new Error('runtime_manually_disconnected')
|
||||
}
|
||||
const pairingRevision = environment.pairingRevision ?? environment.createdAt
|
||||
if (
|
||||
args.expectedEnvironmentPairingRevision !== undefined &&
|
||||
|
||||
@@ -2273,9 +2273,8 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
rows: serialized?.rows ?? size?.rows,
|
||||
displayMode,
|
||||
seq: layoutSeq,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read))
|
||||
// Why: retained-tail truncation loses history, not the authoritative latest-screen fallback.
|
||||
truncated: initialOutputOverflowed
|
||||
})
|
||||
sendSnapshotFrames((opcode, payload) => sendFrame(request.streamId, opcode, payload), {
|
||||
kind: 'scrollback',
|
||||
@@ -2284,9 +2283,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
displayMode,
|
||||
seq: snapshotFrameSeq,
|
||||
cwd: serialized?.cwd,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
|
||||
truncated: initialOutputOverflowed,
|
||||
truncatedByByteBudget: serialized?.truncatedByByteBudget,
|
||||
source: serialized?.source,
|
||||
oscLinks: serialized?.oscLinks,
|
||||
@@ -3026,8 +3023,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
streamId,
|
||||
lines: read.tail,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
|
||||
initialOutputOverflowed || (!sendBinary && isTerminalReadPayloadIncomplete(read)),
|
||||
cols: serialized?.cols ?? size?.cols,
|
||||
rows: serialized?.rows ?? size?.rows,
|
||||
displayMode,
|
||||
@@ -3040,9 +3036,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
displayMode,
|
||||
seq: snapshotFrameSeq,
|
||||
cwd: serialized?.cwd,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
|
||||
truncated: initialOutputOverflowed,
|
||||
truncatedByByteBudget: serialized?.truncatedByByteBudget,
|
||||
oscLinks: serialized?.oscLinks,
|
||||
data: serialized?.data ?? ''
|
||||
|
||||
@@ -1901,7 +1901,7 @@ describe('terminal multiplex RPC', () => {
|
||||
await dispatchPromise
|
||||
})
|
||||
|
||||
it('marks multiplex fallback snapshots truncated when the uncursored read is limited', async () => {
|
||||
it('keeps a limited retained-tail fallback usable for multiplex first paint', async () => {
|
||||
const messages: string[] = []
|
||||
const binaryFrames: Uint8Array<ArrayBufferLike>[] = []
|
||||
const handlers = new Map<
|
||||
@@ -1984,7 +1984,7 @@ describe('terminal multiplex RPC', () => {
|
||||
expect(subscribed).toMatchObject({
|
||||
type: 'subscribed',
|
||||
streamId: 11,
|
||||
truncated: true
|
||||
truncated: false
|
||||
})
|
||||
|
||||
const decodedFrames = binaryFrames.map((frame) => decodeTerminalStreamFrame(frame))
|
||||
@@ -1992,7 +1992,7 @@ describe('terminal multiplex RPC', () => {
|
||||
(frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart && frame.streamId === 11
|
||||
)
|
||||
expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({
|
||||
truncated: true
|
||||
truncated: false
|
||||
})
|
||||
const snapshotData = decodedFrames
|
||||
.filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk)
|
||||
@@ -2004,6 +2004,50 @@ describe('terminal multiplex RPC', () => {
|
||||
await dispatchPromise
|
||||
})
|
||||
|
||||
it('does not mark a serialized multiplex snapshot truncated from an overflowed read', async () => {
|
||||
const harness = startDesktopMultiplexSubscribe({
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['old retained line'],
|
||||
truncated: true,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue({
|
||||
data: 'authoritative current screen\r\n',
|
||||
cols: 120,
|
||||
rows: 40
|
||||
})
|
||||
})
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe(
|
||||
true
|
||||
)
|
||||
)
|
||||
sendDesktopMultiplexSubscribe(harness.handlers)
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed')
|
||||
).toBe(true)
|
||||
)
|
||||
|
||||
const snapshotStart = harness.binaryFrames
|
||||
.map((bytes) => decodeTerminalStreamFrame(bytes))
|
||||
.find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart && frame.streamId === 7)
|
||||
expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({
|
||||
truncated: false
|
||||
})
|
||||
expect(
|
||||
harness.binaryFrames
|
||||
.map((bytes) => decodeTerminalStreamFrame(bytes))
|
||||
.filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk)
|
||||
.map((frame) => (frame ? decodeTerminalStreamText(frame.payload) : ''))
|
||||
.join('')
|
||||
).toBe('authoritative current screen\r\n')
|
||||
|
||||
harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.()
|
||||
await harness.dispatchPromise
|
||||
})
|
||||
|
||||
it('falls back to smaller requested snapshots when serialized data exceeds the send budget', async () => {
|
||||
const messages: string[] = []
|
||||
const binaryFrames: Uint8Array<ArrayBufferLike>[] = []
|
||||
|
||||
@@ -316,7 +316,7 @@ describe('terminal subscribe buffering', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('marks binary subscribed previews truncated when the uncursored read is limited', async () => {
|
||||
it('keeps a limited retained-tail fallback usable for binary first paint', async () => {
|
||||
const messages: string[] = []
|
||||
const binaryFrames: Uint8Array<ArrayBufferLike>[] = []
|
||||
const cleanups = new Map<string, () => void>()
|
||||
@@ -374,20 +374,20 @@ describe('terminal subscribe buffering', () => {
|
||||
expect(subscribed).toMatchObject({
|
||||
type: 'subscribed',
|
||||
lines: ['line 120'],
|
||||
truncated: true
|
||||
truncated: false
|
||||
})
|
||||
const snapshotStart = binaryFrames
|
||||
.map((frame) => decodeTerminalStreamFrame(frame))
|
||||
.find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart)
|
||||
expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({
|
||||
truncated: true
|
||||
truncated: false
|
||||
})
|
||||
|
||||
runtime.cleanupSubscription('terminal-1:desktop-1')
|
||||
await dispatchPromise
|
||||
})
|
||||
|
||||
it('does not mark binary snapshot frames truncated from a limited read when serialized data is available', async () => {
|
||||
it('does not mark binary snapshot frames truncated from an overflowed read when serialized data is available', async () => {
|
||||
const messages: string[] = []
|
||||
const binaryFrames: Uint8Array<ArrayBufferLike>[] = []
|
||||
const cleanups = new Map<string, () => void>()
|
||||
@@ -395,7 +395,7 @@ describe('terminal subscribe buffering', () => {
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
truncated: true,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue({
|
||||
|
||||
@@ -3303,6 +3303,10 @@ export type PreloadApi = {
|
||||
disconnect: (args: {
|
||||
selector: string
|
||||
}) => Promise<{ disconnected: PublicKnownRuntimeEnvironment }>
|
||||
connect: (args: {
|
||||
selector: string
|
||||
timeoutMs?: number
|
||||
}) => Promise<RuntimeRpcResponse<RuntimeStatus>>
|
||||
getStatus: (args: {
|
||||
selector: string
|
||||
timeoutMs?: number
|
||||
|
||||
@@ -4261,6 +4261,11 @@ const api = {
|
||||
selector: string
|
||||
}): Promise<{ disconnected: PublicKnownRuntimeEnvironment }> =>
|
||||
ipcRenderer.invoke('runtimeEnvironments:disconnect', args),
|
||||
connect: (args: {
|
||||
selector: string
|
||||
timeoutMs?: number
|
||||
}): Promise<RuntimeRpcResponse<RuntimeStatus>> =>
|
||||
ipcRenderer.invoke('runtimeEnvironments:connect', args),
|
||||
getStatus: (args: {
|
||||
selector: string
|
||||
timeoutMs?: number
|
||||
|
||||
@@ -93,13 +93,17 @@ import {
|
||||
} from './terminal-pane/terminal-hidden-view-parking'
|
||||
import {
|
||||
TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS,
|
||||
hasPendingRetentionSpawnWork,
|
||||
selectForceParkEvictableTabIds,
|
||||
selectRetentionForceParkedTerminalWorktrees,
|
||||
type TerminalWorktreeRetentionCandidate
|
||||
} from './terminal-pane/terminal-hidden-worktree-retention'
|
||||
import { captureForceParkedWorktreeBuffers } from './terminal-pane/force-park-buffer-capture'
|
||||
import { warnTerminalLifecycleAnomaly } from './terminal-pane/terminal-lifecycle-diagnostics'
|
||||
import { getTerminalParkingPolicyOverrides } from './terminal-pane/terminal-parking-e2e-overrides'
|
||||
import {
|
||||
getTerminalParkingPolicyOverrides,
|
||||
recordTerminalWorktreeParkingDebugVerdicts
|
||||
} from './terminal-pane/terminal-parking-e2e-overrides'
|
||||
import { selectEvictionExemptTerminalTabIds } from './terminal-pane/terminal-eviction-exempt-tabs'
|
||||
import {
|
||||
canWatcherCoverParkedTerminalTab,
|
||||
@@ -1006,14 +1010,11 @@ function Terminal(): React.JSX.Element | null {
|
||||
isVisible: candidate.isVisible,
|
||||
shouldMeasureHiddenWorktree: candidate.shouldMeasureHiddenWorktree,
|
||||
hasActivityTerminalPortal: candidate.hasActivityTerminalPortal,
|
||||
parkCooldownUntilMs: candidate.parkCooldownUntilMs,
|
||||
parkCooldownUntilMs: candidate.parkCooldownUntilMs ?? null,
|
||||
ordinaryParkingCovers:
|
||||
parkEligible && worktreeTabsAreWatcherCovered(candidate.worktreeId, tabs),
|
||||
hasPendingSpawnWork: tabs.some(
|
||||
(tab) =>
|
||||
pendingStartupByTabId[tab.id] !== undefined ||
|
||||
tab.pendingActivationSpawn === true ||
|
||||
(typeof tab.pendingActivationSpawn === 'number' && tab.pendingActivationSpawn > 0)
|
||||
hasPendingSpawnWork: tabs.some((tab) =>
|
||||
hasPendingRetentionSpawnWork(tab, pendingStartupByTabId)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1025,6 +1026,13 @@ function Terminal(): React.JSX.Element | null {
|
||||
nowMs,
|
||||
...overrides
|
||||
})
|
||||
recordTerminalWorktreeParkingDebugVerdicts(
|
||||
retentionBudgetCandidates.map((candidate) => ({
|
||||
...candidate,
|
||||
parkCooldownUntilMs: candidate.parkCooldownUntilMs ?? null,
|
||||
forceParked: forceParkedWorktreeIds.has(candidate.worktreeId)
|
||||
}))
|
||||
)
|
||||
const capturedForceParked = forceParkedCaptureDoneRef.current
|
||||
for (const id of Array.from(capturedForceParked)) {
|
||||
if (!forceParkedWorktreeIds.has(id)) {
|
||||
|
||||
@@ -600,7 +600,7 @@ export function RuntimeEnvironmentsPane({
|
||||
setConnectingId(environment.id)
|
||||
setSwitchError(null)
|
||||
try {
|
||||
const response = await window.api.runtimeEnvironments.getStatus({
|
||||
const response = await window.api.runtimeEnvironments.connect({
|
||||
selector: environment.id,
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
|
||||
@@ -20,6 +20,7 @@ import { isUserManagedRuntimeEnvironment } from '../../../../shared/runtime-envi
|
||||
import { RuntimeHostStatusRow, type RuntimeHostConnectionState } from './RuntimeHostStatusRow'
|
||||
import { SshTargetStatusRow } from './SshTargetStatusRow'
|
||||
import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types'
|
||||
import { connectRuntimeEnvironmentAndRecordStatus } from './runtime-environment-explicit-connect'
|
||||
|
||||
function isConnecting(status: SshConnectionStatus): boolean {
|
||||
return ['connecting', 'deploying-relay', 'reconnecting'].includes(status)
|
||||
@@ -177,7 +178,6 @@ export function SshStatusSegment({
|
||||
const runtimeStatusByEnvironmentId = useAppStore((s) => s.runtimeStatusByEnvironmentId)
|
||||
const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus)
|
||||
const hydrateRuntimeEnvironmentStatuses = useAppStore((s) => s.hydrateRuntimeEnvironmentStatuses)
|
||||
const refreshRuntimeEnvironmentStatus = useAppStore((s) => s.refreshRuntimeEnvironmentStatus)
|
||||
const remoteWorkspaceSyncStatusByTargetId = useAppStore(
|
||||
(s) => s.remoteWorkspaceSyncStatusByTargetId
|
||||
)
|
||||
@@ -232,7 +232,7 @@ export function SshStatusSegment({
|
||||
const store = useAppStore.getState()
|
||||
const reachable = await connectRuntimeHostForNavigation({
|
||||
environmentId,
|
||||
refreshStatus: refreshRuntimeEnvironmentStatus,
|
||||
refreshStatus: connectRuntimeEnvironmentAndRecordStatus,
|
||||
fetchRepos: store.fetchRuntimeEnvironmentRepos,
|
||||
fetchWorktrees: store.fetchWorktrees,
|
||||
fetchLineage: store.fetchWorktreeLineage
|
||||
@@ -248,7 +248,7 @@ export function SshStatusSegment({
|
||||
}
|
||||
recordFeatureInteraction('ssh')
|
||||
},
|
||||
[recordFeatureInteraction, refreshRuntimeEnvironmentStatus]
|
||||
[recordFeatureInteraction]
|
||||
)
|
||||
const disconnectRuntimeHost = useCallback(
|
||||
async (environmentId: string): Promise<void> => {
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { RuntimeStatus } from '../../../../shared/runtime-types'
|
||||
import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client'
|
||||
import { useAppStore } from '../../store'
|
||||
|
||||
export async function connectRuntimeEnvironmentAndRecordStatus(
|
||||
environmentId: string,
|
||||
timeoutMs: number
|
||||
): Promise<boolean> {
|
||||
const setStatus = useAppStore.getState().setRuntimeEnvironmentStatus
|
||||
try {
|
||||
const response = await window.api.runtimeEnvironments.connect({
|
||||
selector: environmentId,
|
||||
timeoutMs
|
||||
})
|
||||
const status = unwrapRuntimeRpcResult<RuntimeStatus>(response)
|
||||
setStatus(environmentId, { status, checkedAt: Date.now() })
|
||||
return true
|
||||
} catch {
|
||||
setStatus(environmentId, { status: null, checkedAt: Date.now() })
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -2050,6 +2050,11 @@ export function connectPanePty(
|
||||
// surviving shell then receives pointer moves as typed SGR reports; the
|
||||
// replay guard keeps xterm's auto-replies from leaking to the shell.
|
||||
replayIntoTerminal(pane, deps.replayingPanesRef, POST_REPLAY_REATTACH_RESET, {
|
||||
breadcrumbIdentity: {
|
||||
tabId: deps.tabId,
|
||||
worktreeId: deps.worktreeId,
|
||||
ptyId: transport.getPtyId()
|
||||
},
|
||||
shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously
|
||||
})
|
||||
if (reason === 'visible-pty') {
|
||||
@@ -2579,6 +2584,11 @@ export function connectPanePty(
|
||||
// eats every click and keystroke against a dead transport — disarm the
|
||||
// modes now and arm the reveal-time wake.
|
||||
replayIntoTerminal(pane, deps.replayingPanesRef, POST_REPLAY_MODE_RESET, {
|
||||
breadcrumbIdentity: {
|
||||
tabId: deps.tabId,
|
||||
worktreeId: deps.worktreeId,
|
||||
ptyId: transport.getPtyId()
|
||||
},
|
||||
shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously
|
||||
})
|
||||
hibernatedWakeTarget = { ptyId, record: sleepingRecordEntry.record }
|
||||
@@ -5332,6 +5342,11 @@ export function connectPanePty(
|
||||
// scheduler's deferred drain cannot land older bytes on top of the replay.
|
||||
flushTerminalOutput(pane.terminal)
|
||||
replayIntoTerminal(pane, deps.replayingPanesRef, data, {
|
||||
breadcrumbIdentity: {
|
||||
tabId: deps.tabId,
|
||||
worktreeId: deps.worktreeId,
|
||||
ptyId: transport.getPtyId()
|
||||
},
|
||||
shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously,
|
||||
shouldReleaseRenderPause: () => deps.isVisibleRef.current
|
||||
})
|
||||
@@ -5342,6 +5357,11 @@ export function connectPanePty(
|
||||
// merely after the write was queued.
|
||||
flushTerminalOutput(pane.terminal)
|
||||
return replayIntoTerminalAsync(pane, deps.replayingPanesRef, data, {
|
||||
breadcrumbIdentity: {
|
||||
tabId: deps.tabId,
|
||||
worktreeId: deps.worktreeId,
|
||||
ptyId: transport.getPtyId()
|
||||
},
|
||||
shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously,
|
||||
shouldReleaseRenderPause: () => deps.isVisibleRef.current
|
||||
})
|
||||
|
||||
@@ -495,6 +495,43 @@ describe('replay-guard stall handling (probe-certified release)', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('records correlatable replay identity without exposing worktree or PTY paths', () => {
|
||||
vi.useFakeTimers()
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
try {
|
||||
const ref = makeRef()
|
||||
const { pane, terminal } = makeFakePane(1)
|
||||
pane.leafId = 'leaf-private-identity' as ManagedPane['leafId']
|
||||
|
||||
replayIntoTerminal(pane, ref, 'restored bytes', {
|
||||
breadcrumbIdentity: {
|
||||
tabId: 'tab-private-identity',
|
||||
worktreeId: 'repo::/Users/alice/private-worktree',
|
||||
ptyId: '/Users/alice/private-worktree@@ab12cd34'
|
||||
},
|
||||
stallCheckMs: 1_000
|
||||
})
|
||||
terminal.pendingCallbacks.shift()
|
||||
vi.advanceTimersByTime(1_000)
|
||||
terminal.flush()
|
||||
|
||||
const breadcrumbData = mocks.recordRendererCrashBreadcrumb.mock.calls[0]?.[1]
|
||||
expect(mocks.recordRendererCrashBreadcrumb).toHaveBeenCalledWith(
|
||||
'terminal_replay_guard_lost_completion',
|
||||
{
|
||||
paneId: 1,
|
||||
leafIdHash: expect.stringMatching(/^[0-9a-f]{8}$/),
|
||||
tabIdHash: expect.stringMatching(/^[0-9a-f]{8}$/),
|
||||
worktreeIdHash: expect.stringMatching(/^[0-9a-f]{8}$/),
|
||||
ptyId: '…@@ab12cd34'
|
||||
}
|
||||
)
|
||||
expect(JSON.stringify(breadcrumbData)).not.toContain('/Users/alice')
|
||||
} finally {
|
||||
errorSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('releases after the probe itself never parses (wedged pipeline) and reports it', () => {
|
||||
vi.useFakeTimers()
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
notifyUndeliverableWrite,
|
||||
recordTerminalParseProgress
|
||||
} from '@/lib/pane-manager/terminal-write-pipeline-health'
|
||||
import { redactPtyIdForDiagnostics } from '../../../../shared/pty-delivery-diagnostics'
|
||||
|
||||
// Why this guard exists: xterm auto-replies to query sequences (DA1/DECRQM/OSC 10-11/CPR) via onData → shell stdin, so replaying recorded PTY bytes leaks stray replies onto the new shell's prompt.
|
||||
// No wasUserInput flag distinguishes replay replies from real keystrokes, so a per-pane in-flight counter gates onData; bounded by xterm's parse completion (not a timer), only auto-replies from replayed bytes are dropped.
|
||||
@@ -20,11 +21,53 @@ export type ReplayingPanesRef = React.RefObject<Map<number, number>>
|
||||
const REPLAY_GUARD_STALL_CHECK_MS = 10_000
|
||||
|
||||
type ReplayTerminalOptions = {
|
||||
breadcrumbIdentity?: {
|
||||
tabId?: string
|
||||
worktreeId?: string
|
||||
ptyId?: string | null
|
||||
}
|
||||
shouldRefreshViewportSynchronously?: () => boolean
|
||||
shouldReleaseRenderPause?: () => boolean
|
||||
stallCheckMs?: number
|
||||
}
|
||||
|
||||
type ReplayGuardBreadcrumbData = {
|
||||
paneId: number
|
||||
tabIdHash?: string
|
||||
worktreeIdHash?: string
|
||||
leafIdHash?: string
|
||||
ptyId?: string
|
||||
}
|
||||
|
||||
function hashReplayIdentity(value: string): string {
|
||||
let hash = 0x811c9dc5
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash ^= value.charCodeAt(index)
|
||||
hash = Math.imul(hash, 0x01000193)
|
||||
}
|
||||
return (hash >>> 0).toString(16).padStart(8, '0')
|
||||
}
|
||||
|
||||
function replayGuardBreadcrumbData(
|
||||
pane: ManagedPane,
|
||||
identity: ReplayTerminalOptions['breadcrumbIdentity']
|
||||
): ReplayGuardBreadcrumbData {
|
||||
const data: ReplayGuardBreadcrumbData = { paneId: pane.id }
|
||||
if (pane.leafId) {
|
||||
data.leafIdHash = hashReplayIdentity(pane.leafId)
|
||||
}
|
||||
if (identity?.tabId) {
|
||||
data.tabIdHash = hashReplayIdentity(identity.tabId)
|
||||
}
|
||||
if (identity?.worktreeId) {
|
||||
data.worktreeIdHash = hashReplayIdentity(identity.worktreeId)
|
||||
}
|
||||
if (identity?.ptyId) {
|
||||
data.ptyId = redactPtyIdForDiagnostics(identity.ptyId)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
export function isPaneReplaying(ref: ReplayingPanesRef, paneId: number): boolean {
|
||||
return (ref.current.get(paneId) ?? 0) > 0
|
||||
}
|
||||
@@ -45,6 +88,7 @@ function engageReplayGuard(
|
||||
paneId: number,
|
||||
terminal: ReplayGuardWriteTarget,
|
||||
stallCheckMs: number,
|
||||
breadcrumbData: ReplayGuardBreadcrumbData,
|
||||
onRelease?: () => void
|
||||
): ReplayGuardWriteCallbacks {
|
||||
map.set(paneId, (map.get(paneId) ?? 0) + 1)
|
||||
@@ -69,12 +113,12 @@ function engageReplayGuard(
|
||||
console.error(
|
||||
`[terminal] replay guard released for pane ${paneId} — the probe write parsed but the replay completion never arrived (lost write callback)`
|
||||
)
|
||||
recordRendererCrashBreadcrumb('terminal_replay_guard_lost_completion', { paneId })
|
||||
recordRendererCrashBreadcrumb('terminal_replay_guard_lost_completion', breadcrumbData)
|
||||
} else if (reason === 'wedged') {
|
||||
console.error(
|
||||
`[terminal] replay guard released for pane ${paneId} — xterm rejected the replay write or its probe never parsed (undeliverable write pipeline; pane likely needs recovery)`
|
||||
)
|
||||
recordRendererCrashBreadcrumb('terminal_replay_guard_wedged_release', { paneId })
|
||||
recordRendererCrashBreadcrumb('terminal_replay_guard_wedged_release', breadcrumbData)
|
||||
// Why: a rejected replay or silent probe makes the pipeline undeliverable; recover instead of a fossil that eats input.
|
||||
notifyUndeliverableWrite(terminal, 'replay-wedged')
|
||||
}
|
||||
@@ -144,7 +188,8 @@ export function replayIntoTerminal(
|
||||
replayingPanesRef.current,
|
||||
pane.id,
|
||||
pane.terminal,
|
||||
options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS
|
||||
options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS,
|
||||
replayGuardBreadcrumbData(pane, options.breadcrumbIdentity)
|
||||
)
|
||||
// Why: hidden/snapshot replay skips the foreground path; WebGL/canvas still need a post-parse repaint to drop stale cells.
|
||||
writeForegroundTerminalChunk(pane.terminal, data, {
|
||||
@@ -178,6 +223,7 @@ export function replayIntoTerminalAsync(
|
||||
pane.id,
|
||||
pane.terminal,
|
||||
options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS,
|
||||
replayGuardBreadcrumbData(pane, options.breadcrumbIdentity),
|
||||
resolve
|
||||
)
|
||||
writeForegroundTerminalChunk(pane.terminal, data, {
|
||||
|
||||
@@ -2,12 +2,46 @@ import { describe, expect, it } from 'vitest'
|
||||
import { TERMINAL_WORKTREE_PARK_DELAY_MS } from './terminal-hidden-view-parking'
|
||||
import {
|
||||
TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS,
|
||||
hasPendingRetentionSpawnWork,
|
||||
isEvictionExemptTerminalPty,
|
||||
selectForceParkEvictableTabIds,
|
||||
selectRetentionForceParkedTerminalWorktrees,
|
||||
type TerminalWorktreeRetentionCandidate
|
||||
} from './terminal-hidden-worktree-retention'
|
||||
|
||||
describe('hasPendingRetentionSpawnWork', () => {
|
||||
const remoteTab = {
|
||||
id: 'tab-remote',
|
||||
ptyId: 'remote:env-1@@terminal-1',
|
||||
pendingActivationSpawn: true as const
|
||||
}
|
||||
|
||||
it('treats a host-backed paired PTY as settled despite activation residue', () => {
|
||||
expect(hasPendingRetentionSpawnWork(remoteTab, {})).toBe(false)
|
||||
expect(hasPendingRetentionSpawnWork({ ...remoteTab, pendingActivationSpawn: 2 }, {})).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves real startup work and non-paired activation guards', () => {
|
||||
expect(hasPendingRetentionSpawnWork(remoteTab, { [remoteTab.id]: ['echo', 'pending'] })).toBe(
|
||||
true
|
||||
)
|
||||
expect(
|
||||
hasPendingRetentionSpawnWork(
|
||||
{ id: 'tab-local', ptyId: 'pty-local', pendingActivationSpawn: true },
|
||||
{}
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
hasPendingRetentionSpawnWork(
|
||||
{ id: 'tab-unbound', ptyId: null, pendingActivationSpawn: true },
|
||||
{}
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('isEvictionExemptTerminalPty', () => {
|
||||
const worktreeId = 'repo::/worktree'
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type ColdParkRetainCandidate,
|
||||
type TerminalColdParkPolicyOverrides
|
||||
} from './terminal-hidden-view-parking'
|
||||
import type { TerminalTab } from '../../../../shared/types'
|
||||
|
||||
// Why these sizes: a retained hidden pane costs a measured ~2.5MB of V8 heap
|
||||
// at the 5k-row default scrollback and ~19MB at 50k (plus per-pane queues),
|
||||
@@ -28,6 +29,18 @@ import {
|
||||
export const TERMINAL_HIDDEN_WORKTREE_RETENTION_LIMIT = 12
|
||||
export const TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS = 45 * 60_000
|
||||
|
||||
export function hasPendingRetentionSpawnWork(
|
||||
tab: Pick<TerminalTab, 'id' | 'ptyId' | 'pendingActivationSpawn'>,
|
||||
pendingStartupByTabId: Readonly<Record<string, unknown>>
|
||||
): boolean {
|
||||
if (pendingStartupByTabId[tab.id] !== undefined) {
|
||||
return true
|
||||
}
|
||||
// Why: paired mirrors never spawn locally; their host-backed PTY id proves
|
||||
// activation's sort-suppression residue cannot represent unfinished work.
|
||||
return Boolean(tab.pendingActivationSpawn && (!tab.ptyId || !isRemoteRuntimePtyId(tab.ptyId)))
|
||||
}
|
||||
|
||||
// Why: an eviction-exempt pty is a live local one a remount could not reattach
|
||||
// (daemon-fail-open separator-less ids, ptys minted under another worktree) — a
|
||||
// fresh spawn would orphan the live shell. Its TAB keeps its mounted pane when
|
||||
|
||||
@@ -5,6 +5,20 @@ import {
|
||||
} from './terminal-hidden-view-parking'
|
||||
import { getParkedTerminalWatcherTabIds } from './terminal-parked-tab-watchers'
|
||||
|
||||
export type TerminalWorktreeParkingDebugVerdict = {
|
||||
worktreeId: string
|
||||
forceParked: boolean
|
||||
hasActivityTerminalPortal: boolean
|
||||
hasPendingSpawnWork: boolean
|
||||
hiddenSinceMs: number | null
|
||||
isVisible: boolean
|
||||
ordinaryParkingCovers: boolean
|
||||
parkCooldownUntilMs: number | null
|
||||
shouldMeasureHiddenWorktree: boolean
|
||||
}
|
||||
|
||||
let worktreeVerdicts: TerminalWorktreeParkingDebugVerdict[] = []
|
||||
|
||||
// Why: ORCA_E2E_TERMINAL_PARKING_DELAY_MS must shrink BOTH the cold-park
|
||||
// hysteresis and the hot-retain window — recently hidden tabs otherwise sit
|
||||
// in the hot-retain working set for 5 minutes and never park within a test
|
||||
@@ -36,7 +50,17 @@ export function registerTerminalParkingDebugHandle(): void {
|
||||
window.__terminalParkingDebug = {
|
||||
parkDelayMs:
|
||||
getTerminalParkingPolicyOverrides().coldParkDelayMs ?? TERMINAL_TAB_COLD_PARK_DELAY_MS,
|
||||
parkedTabIds: () => getParkedTerminalWatcherTabIds()
|
||||
parkedTabIds: () => getParkedTerminalWatcherTabIds(),
|
||||
retentionLimit: getTerminalParkingPolicyOverrides().retentionLimit ?? null,
|
||||
worktreeVerdicts: () => worktreeVerdicts
|
||||
}
|
||||
}
|
||||
|
||||
export function recordTerminalWorktreeParkingDebugVerdicts(
|
||||
verdicts: TerminalWorktreeParkingDebugVerdict[]
|
||||
): void {
|
||||
if (e2eConfig.exposeStore) {
|
||||
worktreeVerdicts = verdicts
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Vendored
+12
@@ -71,6 +71,18 @@ declare global {
|
||||
__terminalParkingDebug?: {
|
||||
parkDelayMs: number
|
||||
parkedTabIds: () => string[]
|
||||
retentionLimit: number | null
|
||||
worktreeVerdicts: () => {
|
||||
worktreeId: string
|
||||
forceParked: boolean
|
||||
hasActivityTerminalPortal: boolean
|
||||
hasPendingSpawnWork: boolean
|
||||
hiddenSinceMs: number | null
|
||||
isVisible: boolean
|
||||
ordinaryParkingCovers: boolean
|
||||
parkCooldownUntilMs: number | null
|
||||
shouldMeasureHiddenWorktree: boolean
|
||||
}[]
|
||||
}
|
||||
__monacoEditorE2E?: MonacoE2EProbe
|
||||
__e2ePtyAppliedSizeReadDelayMs?: number
|
||||
|
||||
@@ -1,11 +1,24 @@
|
||||
import { createE2EConfig } from '../../../shared/e2e-config'
|
||||
|
||||
const rendererE2EExposeStore = String(import.meta.env.VITE_EXPOSE_STORE) === 'true'
|
||||
// Why: the paired web API installs after static modules initialize, so its
|
||||
// build-gated fallback must read the test URL before caching this config.
|
||||
const rendererE2EQuery =
|
||||
rendererE2EExposeStore && typeof window !== 'undefined'
|
||||
? new URLSearchParams(window.location.search)
|
||||
: null
|
||||
export const e2eDisableRemoteTerminalStallRecovery =
|
||||
rendererE2EExposeStore &&
|
||||
rendererE2EQuery?.get('orcaE2EDisableRemoteTerminalStallRecovery') === '1'
|
||||
const rendererFallbackE2EConfig = createE2EConfig({
|
||||
exposeStore: rendererE2EExposeStore,
|
||||
terminalParkingDelayMs: Number(rendererE2EQuery?.get('orcaE2ETerminalParkingDelayMs')) || null,
|
||||
terminalRetentionLimit: Number(rendererE2EQuery?.get('orcaE2ETerminalRetentionLimit')) || null
|
||||
})
|
||||
|
||||
// Why: preload owns the Electron startup contract, so renderer code should
|
||||
// consume the bridged E2E config from window.api instead of reading env vars.
|
||||
export const e2eConfig =
|
||||
typeof window !== 'undefined' && window.api?.e2e
|
||||
? window.api.e2e.getConfig()
|
||||
: createE2EConfig({
|
||||
// Why: paired browser E2E has no preload bridge, so its build flag is the only safe test-hook signal.
|
||||
exposeStore: String(import.meta.env.VITE_EXPOSE_STORE) === 'true'
|
||||
})
|
||||
: rendererFallbackE2EConfig
|
||||
|
||||
@@ -10,7 +10,8 @@ import {
|
||||
encodeTerminalStreamJson,
|
||||
encodeTerminalStreamText
|
||||
} from '../../../shared/terminal-stream-protocol'
|
||||
import { e2eConfig } from '@/lib/e2e-config'
|
||||
import { e2eConfig, e2eDisableRemoteTerminalStallRecovery } from '@/lib/e2e-config'
|
||||
import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder'
|
||||
import { deliverTerminalDataWithDeferredCredit } from '@/lib/pane-manager/terminal-delivery-credit'
|
||||
import { unwrapRuntimeRpcResult } from './runtime-rpc-client'
|
||||
import { getRuntimeEnvironmentRevision } from './runtime-environment-revision'
|
||||
@@ -18,6 +19,10 @@ import {
|
||||
TERMINAL_MULTIPLEX_ACK_BATCH_BYTES,
|
||||
TERMINAL_MULTIPLEX_ACK_FLUSH_MS
|
||||
} from '../../../shared/terminal-multiplex-flow-control'
|
||||
import {
|
||||
createRemoteTerminalStreamWatchdog,
|
||||
type RemoteTerminalStreamWatchdog
|
||||
} from './remote-terminal-stream-watchdog'
|
||||
|
||||
type RuntimeEnvironmentSubscriptionHandle = {
|
||||
unsubscribe: () => void
|
||||
@@ -101,6 +106,7 @@ type RemoteRuntimeMultiplexedTerminalState = {
|
||||
resyncPendingSend: boolean
|
||||
resyncTimer: ReturnType<typeof setTimeout> | null
|
||||
resyncAttempts: number
|
||||
watchdog: RemoteTerminalStreamWatchdog
|
||||
}
|
||||
|
||||
type RemoteRuntimeSnapshotInfo = {
|
||||
@@ -155,6 +161,7 @@ type E2eRemoteTerminalMultiplexAckGateSnapshot = {
|
||||
type E2eRemoteTerminalMultiplexAckGateApi = {
|
||||
hold: (terminals: string[]) => void
|
||||
release: () => void
|
||||
sendInput: (terminal: string, text: string) => number
|
||||
snapshot: () => E2eRemoteTerminalMultiplexAckGateSnapshot
|
||||
}
|
||||
|
||||
@@ -208,6 +215,13 @@ function exposeE2eRemoteTerminalMultiplexAckGate(): void {
|
||||
}
|
||||
},
|
||||
release: releaseE2eRemoteTerminalAcks,
|
||||
sendInput: (terminal, value) => {
|
||||
let sent = 0
|
||||
for (const multiplexer of multiplexers.values()) {
|
||||
sent += multiplexer.sendInputForE2e(terminal, value)
|
||||
}
|
||||
return sent
|
||||
},
|
||||
snapshot: getE2eRemoteAckSnapshot
|
||||
}
|
||||
}
|
||||
@@ -267,14 +281,30 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
resyncInFlight: false,
|
||||
resyncPendingSend: false,
|
||||
resyncTimer: null,
|
||||
resyncAttempts: 0
|
||||
resyncAttempts: 0,
|
||||
watchdog: createRemoteTerminalStreamWatchdog((stall) => {
|
||||
if (!e2eDisableRemoteTerminalStallRecovery) {
|
||||
recordRendererCrashBreadcrumb('remote_terminal_stream_stall_recovery', {
|
||||
environmentId: this.environmentId,
|
||||
expectedSeq: state.expectedSeq ?? null,
|
||||
inactiveForMs: stall.inactiveForMs,
|
||||
outstandingDeliveryBytes: stall.outstandingDeliveryBytes,
|
||||
pendingAckBytes: state.pendingAckBytes,
|
||||
reason: stall.reason,
|
||||
resyncAttempts: state.resyncAttempts,
|
||||
snapshotPending: state.pendingSnapshotRequest !== null,
|
||||
streamId: state.streamId,
|
||||
terminal: state.terminal
|
||||
})
|
||||
this.recoverStalledStream(state)
|
||||
}
|
||||
})
|
||||
}
|
||||
this.streams.set(streamId, state)
|
||||
|
||||
const stream: RemoteRuntimeMultiplexedTerminal = {
|
||||
streamId,
|
||||
sendInput: (text) =>
|
||||
this.sendFrame(streamId, TerminalStreamOpcode.Input, encodeTerminalStreamText(text)),
|
||||
sendInput: (text) => this.sendInput(state, text),
|
||||
resize: (cols, rows) =>
|
||||
this.sendFrame(
|
||||
streamId,
|
||||
@@ -301,6 +331,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
close: () => {
|
||||
if (this.streams.get(streamId) === state) {
|
||||
discardOutputAcknowledgements(state)
|
||||
state.watchdog.dispose()
|
||||
this.sendFrame(streamId, TerminalStreamOpcode.Unsubscribe)
|
||||
clearResyncTimer(state)
|
||||
rejectPendingSnapshotRequest(state, 'Remote terminal stream closed.')
|
||||
@@ -437,8 +468,10 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
if (!stream) {
|
||||
return
|
||||
}
|
||||
stream.watchdog.recordInbound()
|
||||
if (event.type === 'end') {
|
||||
discardOutputAcknowledgements(stream)
|
||||
stream.watchdog.dispose()
|
||||
clearSnapshot(stream)
|
||||
clearResyncTimer(stream)
|
||||
rejectPendingSnapshotRequest(stream, 'Remote terminal stream ended.')
|
||||
@@ -508,6 +541,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
}
|
||||
return
|
||||
}
|
||||
stream.watchdog.recordInbound()
|
||||
if (
|
||||
frame.opcode === TerminalStreamOpcode.Output ||
|
||||
frame.opcode === TerminalStreamOpcode.OutputSpan
|
||||
@@ -577,7 +611,9 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
return
|
||||
}
|
||||
try {
|
||||
const settleWatchdog = stream.watchdog.beginOutputDelivery(frame.payload.byteLength)
|
||||
deliverTerminalDataWithDeferredCredit(() => {
|
||||
settleWatchdog()
|
||||
if (shouldHoldE2eRemoteTerminalAck(stream.terminal)) {
|
||||
stream.heldAckBytes += frame.payload.byteLength
|
||||
} else {
|
||||
@@ -846,7 +882,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
if (stream.pendingSnapshotRequest?.timer === timer) {
|
||||
clearPendingSnapshotRequest(stream)
|
||||
reject(new Error('Remote terminal snapshot timed out.'))
|
||||
this.sendDeferredResyncSnapshot(stream)
|
||||
this.recoverStalledStream(stream)
|
||||
}
|
||||
}, REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS)
|
||||
if (typeof timer.unref === 'function') {
|
||||
@@ -881,6 +917,37 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
)
|
||||
}
|
||||
|
||||
private sendInput(stream: RemoteRuntimeMultiplexedTerminalState, text: string): boolean {
|
||||
const sent = this.sendFrame(
|
||||
stream.streamId,
|
||||
TerminalStreamOpcode.Input,
|
||||
encodeTerminalStreamText(text)
|
||||
)
|
||||
if (sent) {
|
||||
stream.watchdog.recordCommandInput(text)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
|
||||
private recoverStalledStream(stream: RemoteRuntimeMultiplexedTerminalState): void {
|
||||
if (this.streams.get(stream.streamId) !== stream) {
|
||||
return
|
||||
}
|
||||
stream.watchdog.dispose()
|
||||
discardOutputAcknowledgements(stream)
|
||||
clearSnapshot(stream)
|
||||
clearResyncTimer(stream)
|
||||
rejectPendingSnapshotRequest(stream, 'Remote terminal stream stopped responding.')
|
||||
this.streams.delete(stream.streamId)
|
||||
this.sendFrame(stream.streamId, TerminalStreamOpcode.Unsubscribe)
|
||||
if (stream.callbacks.onTransportClose) {
|
||||
stream.callbacks.onTransportClose({ recoverable: true })
|
||||
} else {
|
||||
stream.callbacks.onError?.('Remote terminal stream stopped responding.')
|
||||
}
|
||||
this.closeIfIdle()
|
||||
}
|
||||
|
||||
private queueOutputAcknowledgement(
|
||||
stream: RemoteRuntimeMultiplexedTerminalState,
|
||||
bytes: number
|
||||
@@ -927,6 +994,16 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
return released
|
||||
}
|
||||
|
||||
sendInputForE2e(terminal: string, text: string): number {
|
||||
let sent = 0
|
||||
for (const stream of this.streams.values()) {
|
||||
if (stream.terminal === terminal && this.sendInput(stream, text)) {
|
||||
sent += 1
|
||||
}
|
||||
}
|
||||
return sent
|
||||
}
|
||||
|
||||
private sendFrame(
|
||||
streamId: number,
|
||||
opcode: TerminalStreamOpcode,
|
||||
@@ -983,6 +1060,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
||||
this.releaseIfCurrent(this.environmentId, this)
|
||||
for (const stream of streams) {
|
||||
discardOutputAcknowledgements(stream)
|
||||
stream.watchdog.dispose()
|
||||
clearSnapshot(stream)
|
||||
clearResyncTimer(stream)
|
||||
rejectPendingSnapshotRequest(stream, message ?? 'Remote runtime connection closed.')
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
encodeTerminalStreamFrame,
|
||||
encodeTerminalStreamText
|
||||
} from '../../../shared/terminal-stream-protocol'
|
||||
import {
|
||||
REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS,
|
||||
REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS
|
||||
} from './remote-terminal-stream-watchdog'
|
||||
|
||||
describe('remote terminal stalled stream recovery', () => {
|
||||
const sendBinary = vi.fn()
|
||||
const unsubscribe = vi.fn()
|
||||
const recordBreadcrumb = vi.fn()
|
||||
let callbacks: {
|
||||
onResponse: (response: unknown) => void
|
||||
onBinary: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
} | null = null
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.resetModules()
|
||||
sendBinary.mockReset()
|
||||
unsubscribe.mockReset()
|
||||
recordBreadcrumb.mockReset()
|
||||
callbacks = null
|
||||
vi.stubGlobal('window', {
|
||||
api: {
|
||||
crashReports: {
|
||||
recordBreadcrumb
|
||||
},
|
||||
runtimeEnvironments: {
|
||||
subscribe: vi.fn(async (_args, nextCallbacks) => {
|
||||
callbacks = nextCallbacks
|
||||
queueMicrotask(() => {
|
||||
callbacks?.onResponse({ ok: true, result: { type: 'ready' } })
|
||||
})
|
||||
return { unsubscribe, sendBinary }
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('restarts only the stream whose renderer delivery credit never settles', async () => {
|
||||
const { getRemoteRuntimeTerminalMultiplexer } =
|
||||
await import('./remote-runtime-terminal-multiplexer')
|
||||
const { takeCurrentTerminalDeliveryCredit } =
|
||||
await import('../lib/pane-manager/terminal-delivery-credit')
|
||||
const stalledCredits: (() => void)[] = []
|
||||
const onTransportClose = vi.fn()
|
||||
const multiplexer = getRemoteRuntimeTerminalMultiplexer('windows-test')
|
||||
const stalled = await multiplexer.subscribeTerminal({
|
||||
terminal: 'term-stalled',
|
||||
client: { id: 'mac-viewer', type: 'desktop' },
|
||||
callbacks: {
|
||||
onData: () => {
|
||||
const credit = takeCurrentTerminalDeliveryCredit()
|
||||
if (credit) {
|
||||
stalledCredits.push(credit)
|
||||
}
|
||||
},
|
||||
onSnapshot: vi.fn(),
|
||||
onTransportClose
|
||||
}
|
||||
})
|
||||
const healthy = await multiplexer.subscribeTerminal({
|
||||
terminal: 'term-healthy',
|
||||
client: { id: 'mac-viewer', type: 'desktop' },
|
||||
callbacks: { onData: vi.fn(), onSnapshot: vi.fn() }
|
||||
})
|
||||
sendBinary.mockClear()
|
||||
|
||||
emitOutput(stalled.streamId, 'host output that xterm never parses')
|
||||
expect(stalledCredits).toHaveLength(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS)
|
||||
|
||||
expect(onTransportClose).toHaveBeenCalledWith({ recoverable: true })
|
||||
expect(sentUnsubscribeStreamIds()).toEqual([stalled.streamId])
|
||||
expect(unsubscribe).not.toHaveBeenCalled()
|
||||
expect(recordBreadcrumb).toHaveBeenCalledWith({
|
||||
name: 'remote_terminal_stream_stall_recovery',
|
||||
data: expect.objectContaining({
|
||||
inactiveForMs: REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS,
|
||||
outstandingDeliveryBytes: 'host output that xterm never parses'.length,
|
||||
reason: 'delivery-credit-timeout',
|
||||
streamId: stalled.streamId,
|
||||
terminal: 'term-stalled'
|
||||
})
|
||||
})
|
||||
healthy.close()
|
||||
})
|
||||
|
||||
it('restarts a stream when an entered command receives no frames', async () => {
|
||||
const { getRemoteRuntimeTerminalMultiplexer } =
|
||||
await import('./remote-runtime-terminal-multiplexer')
|
||||
const onTransportClose = vi.fn()
|
||||
const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({
|
||||
terminal: 'term-silent',
|
||||
client: { id: 'mac-viewer', type: 'desktop' },
|
||||
callbacks: { onData: vi.fn(), onSnapshot: vi.fn(), onTransportClose }
|
||||
})
|
||||
sendBinary.mockClear()
|
||||
|
||||
expect(stream.sendInput('ls\r')).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS)
|
||||
|
||||
expect(onTransportClose).toHaveBeenCalledWith({ recoverable: true })
|
||||
expect(sentUnsubscribeStreamIds()).toEqual([stream.streamId])
|
||||
expect(recordBreadcrumb).toHaveBeenCalledWith({
|
||||
name: 'remote_terminal_stream_stall_recovery',
|
||||
data: expect.objectContaining({
|
||||
inactiveForMs: REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS,
|
||||
outstandingDeliveryBytes: 0,
|
||||
reason: 'command-response-timeout',
|
||||
streamId: stream.streamId,
|
||||
terminal: 'term-silent'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps a command stream when any host frame proves it is responsive', async () => {
|
||||
const { getRemoteRuntimeTerminalMultiplexer } =
|
||||
await import('./remote-runtime-terminal-multiplexer')
|
||||
const onTransportClose = vi.fn()
|
||||
const onData = vi.fn()
|
||||
const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({
|
||||
terminal: 'term-responsive',
|
||||
client: { id: 'mac-viewer', type: 'desktop' },
|
||||
callbacks: { onData, onSnapshot: vi.fn(), onTransportClose }
|
||||
})
|
||||
sendBinary.mockClear()
|
||||
|
||||
expect(stream.sendInput('ls\r')).toBe(true)
|
||||
emitOutput(stream.streamId, 'responsive output')
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS)
|
||||
|
||||
expect(onData).toHaveBeenCalledWith('responsive output', {
|
||||
seq: 'responsive output'.length,
|
||||
rawLength: 'responsive output'.length
|
||||
})
|
||||
expect(onTransportClose).not.toHaveBeenCalled()
|
||||
expect(sentUnsubscribeStreamIds()).toEqual([])
|
||||
stream.close()
|
||||
})
|
||||
|
||||
function emitOutput(streamId: number, text: string): void {
|
||||
callbacks?.onBinary(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode: TerminalStreamOpcode.Output,
|
||||
streamId,
|
||||
seq: text.length,
|
||||
payload: encodeTerminalStreamText(text)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function sentUnsubscribeStreamIds(): number[] {
|
||||
return sendBinary.mock.calls.flatMap(([bytes]) => {
|
||||
const frame = decodeTerminalStreamFrame(bytes)
|
||||
return frame?.opcode === TerminalStreamOpcode.Unsubscribe ? [frame.streamId] : []
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,98 @@
|
||||
export const REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS = 10_000
|
||||
export const REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS = 30_000
|
||||
|
||||
export type RemoteTerminalStreamStall = {
|
||||
inactiveForMs: number
|
||||
outstandingDeliveryBytes: number
|
||||
reason: 'command-response-timeout' | 'delivery-credit-timeout'
|
||||
}
|
||||
|
||||
export type RemoteTerminalStreamWatchdog = {
|
||||
beginOutputDelivery: (bytes: number) => () => void
|
||||
recordCommandInput: (text: string) => void
|
||||
recordInbound: () => void
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
export function createRemoteTerminalStreamWatchdog(
|
||||
onStall: (stall: RemoteTerminalStreamStall) => void
|
||||
): RemoteTerminalStreamWatchdog {
|
||||
let responseTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let deliveryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let outstandingDeliveryBytes = 0
|
||||
let lastInboundAtMs = Date.now()
|
||||
let disposed = false
|
||||
|
||||
const clearResponseTimer = (): void => {
|
||||
if (responseTimer) {
|
||||
clearTimeout(responseTimer)
|
||||
responseTimer = null
|
||||
}
|
||||
}
|
||||
const clearDeliveryTimer = (): void => {
|
||||
if (deliveryTimer) {
|
||||
clearTimeout(deliveryTimer)
|
||||
deliveryTimer = null
|
||||
}
|
||||
}
|
||||
const trip = (reason: RemoteTerminalStreamStall['reason']): void => {
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
disposed = true
|
||||
clearResponseTimer()
|
||||
clearDeliveryTimer()
|
||||
onStall({
|
||||
inactiveForMs: Math.max(0, Date.now() - lastInboundAtMs),
|
||||
outstandingDeliveryBytes,
|
||||
reason
|
||||
})
|
||||
}
|
||||
const armDeliveryTimer = (): void => {
|
||||
clearDeliveryTimer()
|
||||
if (outstandingDeliveryBytes <= 0 || disposed) {
|
||||
return
|
||||
}
|
||||
deliveryTimer = setTimeout(
|
||||
() => trip('delivery-credit-timeout'),
|
||||
REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
beginOutputDelivery(bytes) {
|
||||
outstandingDeliveryBytes += bytes
|
||||
if (!deliveryTimer) {
|
||||
armDeliveryTimer()
|
||||
}
|
||||
let settled = false
|
||||
return () => {
|
||||
if (settled || disposed) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
outstandingDeliveryBytes = Math.max(0, outstandingDeliveryBytes - bytes)
|
||||
armDeliveryTimer()
|
||||
}
|
||||
},
|
||||
recordCommandInput(text) {
|
||||
if (disposed || responseTimer || !/[\r\n]/u.test(text)) {
|
||||
return
|
||||
}
|
||||
responseTimer = setTimeout(
|
||||
() => trip('command-response-timeout'),
|
||||
REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS
|
||||
)
|
||||
},
|
||||
recordInbound() {
|
||||
lastInboundAtMs = Date.now()
|
||||
clearResponseTimer()
|
||||
},
|
||||
dispose() {
|
||||
disposed = true
|
||||
clearResponseTimer()
|
||||
clearDeliveryTimer()
|
||||
outstandingDeliveryBytes = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -235,6 +235,7 @@ describe('web runtime environment identity', () => {
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
vi.doUnmock('./web-runtime-client')
|
||||
})
|
||||
|
||||
it('does not resolve an old server selector through a differently keyed server', async () => {
|
||||
@@ -359,6 +360,111 @@ describe('web runtime environment identity', () => {
|
||||
globals.window.api.runtimeEnvironments.resolve({ selector: 'web-server-old' })
|
||||
).rejects.toThrow('Unknown Orca runtime environment: web-server-old')
|
||||
})
|
||||
|
||||
it('keeps pairing while manual disconnect fences passive reconnects', async () => {
|
||||
const calls: string[] = []
|
||||
const close = vi.fn()
|
||||
let clientCount = 0
|
||||
vi.doMock('./web-runtime-client', () => ({
|
||||
WebRuntimeClient: class {
|
||||
constructor() {
|
||||
clientCount += 1
|
||||
}
|
||||
|
||||
call(method: string): Promise<RuntimeRpcResponse<unknown>> {
|
||||
calls.push(method)
|
||||
return Promise.resolve({
|
||||
id: method,
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-1' },
|
||||
_meta: { runtimeId: 'runtime-1' }
|
||||
})
|
||||
}
|
||||
|
||||
close(): void {
|
||||
close()
|
||||
}
|
||||
}
|
||||
}))
|
||||
const globals = installBrowserGlobals('Linux')
|
||||
writeStoredRuntimeEnvironment(globals.storage, 'web-server-a')
|
||||
const { installWebPreloadApi } = await import('./web-preload-api')
|
||||
installWebPreloadApi()
|
||||
|
||||
await expect(
|
||||
globals.window.api.runtimeEnvironments.getStatus({ selector: 'web-server-a' })
|
||||
).resolves.toMatchObject({ ok: true })
|
||||
await globals.window.api.runtimeEnvironments.disconnect({ selector: 'web-server-a' })
|
||||
|
||||
await expect(globals.window.api.runtimeEnvironments.list()).resolves.toMatchObject([
|
||||
{ id: 'web-server-a' }
|
||||
])
|
||||
await expect(
|
||||
globals.window.api.runtimeEnvironments.getStatus({ selector: 'web-server-a' })
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'runtime_manually_disconnected' }
|
||||
})
|
||||
await expect(
|
||||
globals.window.api.runtimeEnvironments.call({
|
||||
selector: 'web-server-a',
|
||||
method: 'repos.list'
|
||||
})
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'runtime_manually_disconnected' }
|
||||
})
|
||||
await expect(
|
||||
globals.window.api.runtimeEnvironments.subscribe(
|
||||
{ selector: 'web-server-a', method: 'terminal.subscribe' },
|
||||
{ onResponse: vi.fn() }
|
||||
)
|
||||
).rejects.toThrow('runtime_manually_disconnected')
|
||||
expect(clientCount).toBe(1)
|
||||
expect(calls).toEqual(['status.get'])
|
||||
expect(close).toHaveBeenCalledOnce()
|
||||
|
||||
await expect(
|
||||
globals.window.api.runtimeEnvironments.connect({ selector: 'web-server-a' })
|
||||
).resolves.toMatchObject({ ok: true })
|
||||
expect(clientCount).toBe(2)
|
||||
expect(calls).toEqual(['status.get', 'status.get'])
|
||||
})
|
||||
|
||||
it('fences a web runtime response that completes after manual disconnect', async () => {
|
||||
let resolveCall!: (response: RuntimeRpcResponse<unknown>) => void
|
||||
const pendingCall = new Promise<RuntimeRpcResponse<unknown>>((resolve) => {
|
||||
resolveCall = resolve
|
||||
})
|
||||
const call = vi.fn(() => pendingCall)
|
||||
vi.doMock('./web-runtime-client', () => ({
|
||||
WebRuntimeClient: class {
|
||||
call = call
|
||||
close(): void {}
|
||||
}
|
||||
}))
|
||||
const globals = installBrowserGlobals('Linux')
|
||||
writeStoredRuntimeEnvironment(globals.storage, 'web-server-a')
|
||||
const { installWebPreloadApi } = await import('./web-preload-api')
|
||||
installWebPreloadApi()
|
||||
|
||||
const status = globals.window.api.runtimeEnvironments.getStatus({
|
||||
selector: 'web-server-a'
|
||||
})
|
||||
await vi.waitFor(() => expect(call).toHaveBeenCalledOnce())
|
||||
await globals.window.api.runtimeEnvironments.disconnect({ selector: 'web-server-a' })
|
||||
resolveCall({
|
||||
id: 'status.get',
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-1' },
|
||||
_meta: { runtimeId: 'runtime-1' }
|
||||
})
|
||||
|
||||
await expect(status).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'runtime_manually_disconnected' }
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('web browser-local port capability', () => {
|
||||
|
||||
@@ -148,6 +148,14 @@ const SESSION_STORAGE_KEY = 'orca.web.workspaceSession.v1'
|
||||
const ONBOARDING_STORAGE_KEY = 'orca.web.onboarding.v1'
|
||||
const GITHUB_CACHE_STORAGE_KEY = 'orca.web.githubCache.v1'
|
||||
const KEYBINDINGS_STORAGE_KEY = 'orca.web.keybindings.v1'
|
||||
// Why: paired web clients lack Electron env/preload state; the E2E build gate keeps URL overrides out of releases.
|
||||
const webE2EExposeStore = String(import.meta.env.VITE_EXPOSE_STORE) === 'true'
|
||||
const webE2EQuery = webE2EExposeStore ? new URLSearchParams(window.location.search) : null
|
||||
const webE2EConfig = createE2EConfig({
|
||||
exposeStore: webE2EExposeStore,
|
||||
terminalParkingDelayMs: Number(webE2EQuery?.get('orcaE2ETerminalParkingDelayMs')) || null,
|
||||
terminalRetentionLimit: Number(webE2EQuery?.get('orcaE2ETerminalRetentionLimit')) || null
|
||||
})
|
||||
// Why: paired clients need parity for large dev sessions; the runtime default stays capped for lower-level RPC callers.
|
||||
const WEB_RUNTIME_WORKTREE_LIST_LIMIT = 10_000
|
||||
const MAX_CLIPBOARD_IMAGE_BASE64_CHARS = CLIPBOARD_IMAGE_MAX_BASE64_CHARS
|
||||
@@ -160,6 +168,7 @@ const CLIPBOARD_IMAGE_SAVE_TIMEOUT_MS = 30_000
|
||||
let activeEnvironment: StoredWebRuntimeEnvironment | null = readStoredWebRuntimeEnvironment()
|
||||
let activeClient: WebRuntimeClient | null = null
|
||||
let activeClientEnvironmentId: string | null = null
|
||||
const manuallyDisconnectedEnvironmentIds = new Set<string>()
|
||||
let cachedWorktrees: { loadedAt: number; worktrees: Worktree[] } | null = null
|
||||
let cachedDetectedWorktrees: { loadedAt: number; worktrees: Worktree[] } | null = null
|
||||
const runtimeCallQueuePool = new RuntimeRpcCallQueuePool()
|
||||
@@ -648,7 +657,7 @@ function createWebPreloadApi(): Partial<PreloadApi> {
|
||||
orgMemberRemove: async () => ({ status: 'unconfigured' })
|
||||
},
|
||||
e2e: {
|
||||
getConfig: () => createE2EConfig({})
|
||||
getConfig: () => webE2EConfig
|
||||
},
|
||||
settings: {
|
||||
get: async () => getRuntimeBackedStoredSettings(),
|
||||
@@ -1348,6 +1357,7 @@ function createRuntimeEnvironmentsApi(): NonNullable<Partial<PreloadApi>['runtim
|
||||
const previousEnvironment = activeEnvironment
|
||||
closeActiveRuntimeClients()
|
||||
activeEnvironment = createStoredWebRuntimeEnvironment({ name, offer, previousEnvironment })
|
||||
manuallyDisconnectedEnvironmentIds.clear()
|
||||
saveStoredWebRuntimeEnvironment(activeEnvironment)
|
||||
return { environment: redactStoredWebRuntimeEnvironment(activeEnvironment) }
|
||||
},
|
||||
@@ -1356,17 +1366,29 @@ function createRuntimeEnvironmentsApi(): NonNullable<Partial<PreloadApi>['runtim
|
||||
remove: async ({ selector }) => {
|
||||
const environment = resolveEnvironment(selector)
|
||||
if (activeEnvironment?.id === environment.id) {
|
||||
disconnectActiveRuntimeEnvironment()
|
||||
removeActiveRuntimeEnvironment()
|
||||
}
|
||||
manuallyDisconnectedEnvironmentIds.delete(environment.id)
|
||||
return { removed: redactStoredWebRuntimeEnvironment(environment) }
|
||||
},
|
||||
disconnect: async ({ selector }) => {
|
||||
const environment = resolveEnvironment(selector)
|
||||
if (activeEnvironment?.id === environment.id) {
|
||||
manuallyDisconnectedEnvironmentIds.add(environment.id)
|
||||
disconnectActiveRuntimeEnvironment()
|
||||
}
|
||||
return { disconnected: redactStoredWebRuntimeEnvironment(environment) }
|
||||
},
|
||||
connect: ({ selector, timeoutMs }) => {
|
||||
const environment = resolveEnvironment(selector)
|
||||
manuallyDisconnectedEnvironmentIds.delete(environment.id)
|
||||
return callEnvironmentEnvelope<RuntimeStatus>(
|
||||
environment.id,
|
||||
'status.get',
|
||||
undefined,
|
||||
timeoutMs
|
||||
)
|
||||
},
|
||||
getStatus: ({ selector, timeoutMs }) =>
|
||||
callEnvironmentEnvelope<RuntimeStatus>(selector, 'status.get', undefined, timeoutMs),
|
||||
call: ({ selector, method, params, timeoutMs }) =>
|
||||
@@ -1374,7 +1396,12 @@ function createRuntimeEnvironmentsApi(): NonNullable<Partial<PreloadApi>['runtim
|
||||
subscribe: async ({ selector, method, params, timeoutMs }, callbacks) => {
|
||||
const environment = resolveEnvironment(selector)
|
||||
const client = getClientForEnvironment(environment)
|
||||
return client.subscribe(method, params, callbacks, { timeoutMs })
|
||||
const subscription = await client.subscribe(method, params, callbacks, { timeoutMs })
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
subscription.unsubscribe()
|
||||
throw new Error('runtime_manually_disconnected')
|
||||
}
|
||||
return subscription
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3139,9 +3166,15 @@ async function callRuntimeEnvelope<TResult = unknown>(
|
||||
timeoutMs?: number
|
||||
): Promise<RuntimeRpcResponse<TResult>> {
|
||||
const environment = requireActiveEnvironment()
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
const response = await runtimeCallQueuePool.enqueue(environment.id, method, () =>
|
||||
getClientForEnvironment(environment).call(method, params, { timeoutMs })
|
||||
)
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
updateEnvironmentFromResponse(environment, response)
|
||||
return response as RuntimeRpcResponse<TResult>
|
||||
}
|
||||
@@ -3153,9 +3186,15 @@ async function callEnvironmentEnvelope<TResult = unknown>(
|
||||
timeoutMs?: number
|
||||
): Promise<RuntimeRpcResponse<TResult>> {
|
||||
const environment = resolveEnvironment(selector)
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
const response = await runtimeCallQueuePool.enqueue(environment.id, method, () =>
|
||||
getClientForEnvironment(environment).call(method, params, { timeoutMs })
|
||||
)
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
return manuallyDisconnectedResponse(environment)
|
||||
}
|
||||
updateEnvironmentFromResponse(environment, response)
|
||||
return response as RuntimeRpcResponse<TResult>
|
||||
}
|
||||
@@ -3332,6 +3371,9 @@ async function getRemoteRuntimeStatus(): Promise<RuntimeStatus> {
|
||||
}
|
||||
|
||||
function getClientForEnvironment(environment: StoredWebRuntimeEnvironment): WebRuntimeClient {
|
||||
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
|
||||
throw new Error('runtime_manually_disconnected')
|
||||
}
|
||||
if (!activeClient || activeClientEnvironmentId !== environment.id) {
|
||||
activeClient?.close()
|
||||
activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment))
|
||||
@@ -3349,10 +3391,28 @@ function closeActiveRuntimeClients(): void {
|
||||
|
||||
function disconnectActiveRuntimeEnvironment(): void {
|
||||
closeActiveRuntimeClients()
|
||||
}
|
||||
|
||||
function removeActiveRuntimeEnvironment(): void {
|
||||
disconnectActiveRuntimeEnvironment()
|
||||
clearStoredWebRuntimeEnvironment()
|
||||
activeEnvironment = null
|
||||
}
|
||||
|
||||
function manuallyDisconnectedResponse(
|
||||
environment: StoredWebRuntimeEnvironment
|
||||
): RuntimeRpcResponse<never> {
|
||||
return {
|
||||
id: 'runtime.manualDisconnect',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'runtime_manually_disconnected',
|
||||
message: 'Runtime environment is manually disconnected.'
|
||||
},
|
||||
_meta: { runtimeId: environment.runtimeId }
|
||||
}
|
||||
}
|
||||
|
||||
function resolveEnvironment(selector: string): StoredWebRuntimeEnvironment {
|
||||
const environment = requireActiveEnvironment()
|
||||
if (selector === environment.id || selector === environment.name || selector === 'active') {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import type { RuntimeTerminalRead } from '../../src/shared/runtime-types'
|
||||
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
|
||||
import { launchPairedWebClient } from './helpers/paired-electron-client'
|
||||
import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal'
|
||||
|
||||
const MIN_EXHAUSTED_ACK_BYTES = 400 * 1024
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-headless-stalled-stream-'))
|
||||
const fixturePath = path.join(scratch, 'headless-stalled-stream.mjs')
|
||||
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
"process.stdout.write('HEADLESS_STALL_READY\\r\\n')",
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"let pending = ''",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' pending += data',
|
||||
' const commands = pending.split(/\\r\\n|\\r|\\n/)',
|
||||
' pending = commands.pop() ?? ""',
|
||||
' for (const input of commands) {',
|
||||
" if (input === 'GO') {",
|
||||
" for (let row = 0; row < 16_000; row += 1) process.stdout.write(`headless-${row}-${'x'.repeat(80)}\\r\\n`)",
|
||||
" process.stdout.write('HEADLESS_FLOOD_COMPLETE\\r\\n')",
|
||||
' continue',
|
||||
' }',
|
||||
' process.stdout.write(`LIVE:${input}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(): string {
|
||||
const command = [process.execPath, fixturePath]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(page: Page, method: string, params: unknown): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ method, params }) => {
|
||||
const response = await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
test('recovers an ACK-starved stream from an isolated headless Orca host @headful', async ({
|
||||
testRepoPath
|
||||
}) => {
|
||||
test.setTimeout(180_000)
|
||||
const host = await launchHeadlessPairedRuntimeHost()
|
||||
const client = await launchPairedWebClient(host.app, host.offer, { waitForWorkspace: false })
|
||||
let terminal: string | null = null
|
||||
try {
|
||||
await host.client.call('repo.add', { path: testRepoPath, kind: 'git' })
|
||||
try {
|
||||
await client.page.locator('[data-worktree-sidebar]').waitFor({
|
||||
state: 'visible',
|
||||
timeout: 30_000
|
||||
})
|
||||
} catch {
|
||||
const boot = await client.page.evaluate(() => ({
|
||||
bodyChildren: document.body?.children.length ?? 0,
|
||||
bodyTextLength: document.body?.innerText.length ?? 0,
|
||||
hasApi: Boolean(window.api),
|
||||
hasRoot: Boolean(document.querySelector('#root')),
|
||||
hasStore: Boolean(window.__store),
|
||||
readyState: document.readyState,
|
||||
title: document.title
|
||||
}))
|
||||
throw new Error(`Headless paired web client did not boot: ${JSON.stringify(boot)}`)
|
||||
}
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(() => {
|
||||
const worktrees = window.__store?.getState().allWorktrees() ?? []
|
||||
return worktrees[0]?.id ?? null
|
||||
}),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.not.toBeNull()
|
||||
const worktreeId = await client.page.evaluate(
|
||||
() => window.__store?.getState().allWorktrees()[0]?.id ?? null
|
||||
)
|
||||
if (!worktreeId) {
|
||||
throw new Error('Headless paired client did not receive the host worktree')
|
||||
}
|
||||
const created = await callRuntime<{
|
||||
tab: { parentTabId: string; terminal: string | null }
|
||||
}>(client.page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixtureCommand(),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
terminal = created.tab.terminal
|
||||
if (!terminal) {
|
||||
throw new Error('Headless paired host did not publish the fixture terminal')
|
||||
}
|
||||
const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await client.page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId)
|
||||
const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`)
|
||||
await expect(tab).toBeVisible({ timeout: 30_000 })
|
||||
await tab.click()
|
||||
const originalPtyId = await waitForActivePanePtyId(client.page, 30_000)
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain('HEADLESS_STALL_READY')
|
||||
|
||||
await client.page.evaluate((target) => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: { hold: (terminals: string[]) => void }
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
if (!gate) {
|
||||
throw new Error('Remote terminal multiplex ACK gate is unavailable')
|
||||
}
|
||||
gate.hold([target])
|
||||
}, terminal)
|
||||
const textarea = client.page.locator('.xterm-helper-textarea:visible').first()
|
||||
await textarea.focus()
|
||||
await client.page.keyboard.type('GO')
|
||||
await client.page.keyboard.press('Enter')
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(() => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: {
|
||||
snapshot: () => { heldAckChars: number }
|
||||
}
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
return gate?.snapshot().heldAckChars ?? 0
|
||||
}),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBeGreaterThan(MIN_EXHAUSTED_ACK_BYTES)
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const result = await callRuntime<{ terminal: RuntimeTerminalRead }>(
|
||||
client.page,
|
||||
'terminal.read',
|
||||
{ terminal }
|
||||
)
|
||||
return result.terminal.tail.join('\n')
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toContain('HEADLESS_FLOOD_COMPLETE')
|
||||
|
||||
const marker = `HEADLESS_RECOVERED_${Date.now()}`
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: marker,
|
||||
enter: true,
|
||||
client: { id: 'headless-stalled-stream-e2e', type: 'desktop' }
|
||||
})
|
||||
expect(await getTerminalContent(client.page)).not.toContain(marker)
|
||||
expect(
|
||||
await client.page.evaluate(
|
||||
({ target }) => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: {
|
||||
sendInput: (terminal: string, text: string) => number
|
||||
}
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
return gate?.sendInput(target, '\r') ?? 0
|
||||
},
|
||||
{ target: terminal }
|
||||
)
|
||||
).toBe(1)
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${marker}`)
|
||||
expect(await waitForActivePanePtyId(client.page, 30_000)).toBe(originalPtyId)
|
||||
await expect(tab).toHaveAttribute('data-active', 'true')
|
||||
} finally {
|
||||
await client.page
|
||||
.evaluate(() => {
|
||||
;(
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: { release: () => void }
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate?.release()
|
||||
})
|
||||
.catch(() => undefined)
|
||||
if (terminal) {
|
||||
await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
await host.dispose()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { _electron as electron, type ElectronApplication } from '@stablyai/playwright-test'
|
||||
import { RuntimeClient } from '../../../src/cli/runtime/client'
|
||||
import { getE2ECompletedOnboardingProfile } from './e2e-completed-onboarding-profile'
|
||||
import { getOrcaElectronLaunchArgs } from './electron-launch-args'
|
||||
import { cleanupE2EDaemons, closeElectronAppForE2E } from './electron-process-shutdown'
|
||||
import {
|
||||
assertElectronResolvedIsolatedHome,
|
||||
createElectronHomeIsolation
|
||||
} from './electron-home-isolation'
|
||||
import type { RuntimeDesktopPairingOffer } from './paired-electron-client'
|
||||
|
||||
type ServeReady = {
|
||||
type?: unknown
|
||||
pairing?: {
|
||||
available?: unknown
|
||||
url?: unknown
|
||||
webClientUrl?: unknown
|
||||
}
|
||||
}
|
||||
|
||||
export type HeadlessPairedRuntimeHost = {
|
||||
app: ElectronApplication
|
||||
client: RuntimeClient
|
||||
dispose: () => Promise<void>
|
||||
offer: RuntimeDesktopPairingOffer
|
||||
}
|
||||
|
||||
async function readPairingOffer(app: ElectronApplication): Promise<RuntimeDesktopPairingOffer> {
|
||||
const stdout = app.process().stdout
|
||||
if (!stdout) {
|
||||
throw new Error('Headless runtime stdout is unavailable')
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
let buffered = ''
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup()
|
||||
reject(new Error('Headless runtime did not publish pairing readiness'))
|
||||
}, 60_000)
|
||||
const cleanup = (): void => {
|
||||
clearTimeout(timeout)
|
||||
stdout.off('data', onData)
|
||||
}
|
||||
const onData = (chunk: Buffer): void => {
|
||||
buffered += chunk.toString()
|
||||
const lines = buffered.split(/\r?\n/)
|
||||
buffered = lines.pop() ?? ''
|
||||
for (const line of lines) {
|
||||
let readiness: ServeReady
|
||||
try {
|
||||
readiness = JSON.parse(line) as ServeReady
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const pairing = readiness.pairing
|
||||
if (
|
||||
readiness.type !== 'orca_server_ready' ||
|
||||
pairing?.available !== true ||
|
||||
typeof pairing.url !== 'string' ||
|
||||
typeof pairing.webClientUrl !== 'string'
|
||||
) {
|
||||
continue
|
||||
}
|
||||
cleanup()
|
||||
resolve({ pairingUrl: pairing.url, webClientUrl: pairing.webClientUrl })
|
||||
return
|
||||
}
|
||||
}
|
||||
stdout.on('data', onData)
|
||||
})
|
||||
}
|
||||
|
||||
export async function launchHeadlessPairedRuntimeHost(): Promise<HeadlessPairedRuntimeHost> {
|
||||
const userDataDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-headless-paired-'))
|
||||
writeFileSync(
|
||||
path.join(userDataDir, 'orca-data.json'),
|
||||
`${JSON.stringify(getE2ECompletedOnboardingProfile(), null, 2)}\n`
|
||||
)
|
||||
const { ELECTRON_RUN_AS_NODE: _unused, ...cleanEnv } = process.env
|
||||
void _unused
|
||||
const isolation = createElectronHomeIsolation({
|
||||
inheritedEnv: cleanEnv,
|
||||
launchEnv: {
|
||||
NODE_ENV: 'development',
|
||||
ORCA_E2E_ENFORCE_SINGLE_INSTANCE_LOCK: '1',
|
||||
ORCA_E2E_HEADLESS: '1'
|
||||
},
|
||||
extraEnv: {},
|
||||
userDataDir,
|
||||
codexRealHomeEnabled: false
|
||||
})
|
||||
const mainPath = path.join(process.cwd(), 'out', 'main', 'index.js')
|
||||
const app = await electron.launch({
|
||||
args: [
|
||||
...getOrcaElectronLaunchArgs(mainPath, false),
|
||||
'--serve',
|
||||
'--serve-json',
|
||||
'--serve-port',
|
||||
'0',
|
||||
'--serve-pairing-address',
|
||||
'127.0.0.1'
|
||||
],
|
||||
env: isolation.env
|
||||
})
|
||||
try {
|
||||
assertElectronResolvedIsolatedHome(
|
||||
await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')),
|
||||
isolation
|
||||
)
|
||||
const offer = await readPairingOffer(app)
|
||||
return {
|
||||
app,
|
||||
client: new RuntimeClient(userDataDir, 5_000),
|
||||
offer,
|
||||
dispose: async () => {
|
||||
await closeElectronAppForE2E(app)
|
||||
await cleanupE2EDaemons(userDataDir)
|
||||
rmSync(userDataDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await closeElectronAppForE2E(app)
|
||||
await cleanupE2EDaemons(userDataDir)
|
||||
rmSync(userDataDir, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
replaceRuntimePairingInPlace,
|
||||
type SameIdPairingReplacement
|
||||
} from './nested-runtime-same-id-pairing'
|
||||
import { createPairedWebClientUrl, type PairedWebClientOptions } from './paired-web-client-url'
|
||||
|
||||
export type { SameIdPairingReplacement } from './nested-runtime-same-id-pairing'
|
||||
|
||||
@@ -92,11 +93,13 @@ export async function createRuntimeDesktopPairingOffer(
|
||||
|
||||
export async function launchPairedWebClient(
|
||||
hubApp: ElectronApplication,
|
||||
offer: RuntimeDesktopPairingOffer
|
||||
offer: RuntimeDesktopPairingOffer,
|
||||
options: PairedWebClientOptions = {}
|
||||
): Promise<PairedWebClient> {
|
||||
if (!offer.webClientUrl) {
|
||||
throw new Error('HUB runtime did not provide a paired web client URL')
|
||||
}
|
||||
const clientUrl = createPairedWebClientUrl(offer.webClientUrl, options)
|
||||
const pagePromise = hubApp.waitForEvent('window')
|
||||
await hubApp.evaluate(
|
||||
async ({ BrowserWindow }, { partition, url }) => {
|
||||
@@ -115,11 +118,13 @@ export async function launchPairedWebClient(
|
||||
},
|
||||
{
|
||||
partition: `e2e-nested-runtime-web-${randomUUID()}`,
|
||||
url: offer.webClientUrl
|
||||
url: clientUrl
|
||||
}
|
||||
)
|
||||
const page = await pagePromise
|
||||
await page.locator('[data-worktree-sidebar]').waitFor({ state: 'visible', timeout: 30_000 })
|
||||
if (options.waitForWorkspace !== false) {
|
||||
await page.locator('[data-worktree-sidebar]').waitFor({ state: 'visible', timeout: 30_000 })
|
||||
}
|
||||
return { page, dispose: () => page.close() }
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
export type PairedWebClientOptions = {
|
||||
disableRemoteTerminalStallRecovery?: boolean
|
||||
terminalParkingDelayMs?: number
|
||||
terminalRetentionLimit?: number
|
||||
waitForWorkspace?: boolean
|
||||
}
|
||||
|
||||
export function createPairedWebClientUrl(
|
||||
offerUrl: string,
|
||||
options: PairedWebClientOptions
|
||||
): string {
|
||||
const clientUrl = new URL(offerUrl)
|
||||
if (options.disableRemoteTerminalStallRecovery) {
|
||||
clientUrl.searchParams.set('orcaE2EDisableRemoteTerminalStallRecovery', '1')
|
||||
}
|
||||
if (options.terminalParkingDelayMs !== undefined) {
|
||||
clientUrl.searchParams.set('orcaE2ETerminalParkingDelayMs', `${options.terminalParkingDelayMs}`)
|
||||
}
|
||||
if (options.terminalRetentionLimit !== undefined) {
|
||||
clientUrl.searchParams.set('orcaE2ETerminalRetentionLimit', `${options.terminalRetentionLimit}`)
|
||||
}
|
||||
return clientUrl.href
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
createRuntimeDesktopPairingOffer,
|
||||
launchPairedWebClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal'
|
||||
import {
|
||||
readPairedRetentionSample,
|
||||
startRendererLagProbe
|
||||
} from './paired-runtime-retention-metrics'
|
||||
|
||||
const RETENTION_LIMIT = 2
|
||||
const TARGET_WORKTREE_COUNT = 6
|
||||
const PARK_DELAY_MS = 100
|
||||
const FILL_ROWS = 6_000
|
||||
const MIN_STAGED_BUFFER_CELLS = 1_000_000
|
||||
const MAX_RETAINED_CELL_FRACTION = 0.45
|
||||
const MAX_EVICTION_LAG_MS = 500
|
||||
const MAX_HEAP_GROWTH_BYTES = 16 * 1024 * 1024
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-retention-memory-'))
|
||||
const fixturePath = path.join(scratch, 'paired-retention-memory.mjs')
|
||||
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
'const marker = process.argv[2]',
|
||||
'process.stdout.write(`READY:${marker}\\r\\n`)',
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' for (const command of data.split(/\\r\\n|\\r|\\n/).filter(Boolean)) {',
|
||||
" if (command === 'FILL') {",
|
||||
` for (let row = 0; row < ${FILL_ROWS}; row += 1) process.stdout.write(\`fill-${'${marker}'}-${'${row}'}-${'x'.repeat(80)}\\r\\n\`)`,
|
||||
' process.stdout.write(`FILLED:${marker}\\r\\n`)',
|
||||
' continue',
|
||||
' }',
|
||||
' process.stdout.write(`LIVE:${command}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(marker: string): string {
|
||||
const command = [process.execPath, fixturePath, marker]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(page: Page, method: string, params: unknown): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ method, params }) => {
|
||||
const response = await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
type RemoteTab = {
|
||||
marker: string
|
||||
originalPtyId: string
|
||||
tabId: string
|
||||
terminal: string
|
||||
worktreeId: string
|
||||
}
|
||||
|
||||
test('bounds retained buffers across real paired worktrees and preserves PTYs @headful', async ({
|
||||
electronApp,
|
||||
orcaPage
|
||||
}) => {
|
||||
test.setTimeout(240_000)
|
||||
const seed = await orcaPage.evaluate(() => {
|
||||
const state = window.__store?.getState()
|
||||
const worktrees = state?.allWorktrees() ?? []
|
||||
const active = worktrees.find((worktree) => worktree.id === state?.activeWorktreeId)
|
||||
if (!active) {
|
||||
throw new Error('Paired retention host has no active seeded worktree')
|
||||
}
|
||||
return { repoId: active.repoId, worktreeIds: worktrees.map((worktree) => worktree.id) }
|
||||
})
|
||||
const offer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const client = await launchPairedWebClient(electronApp, offer, {
|
||||
terminalParkingDelayMs: PARK_DELAY_MS,
|
||||
terminalRetentionLimit: RETENTION_LIMIT
|
||||
})
|
||||
const createdWorktreeIds: string[] = []
|
||||
const remoteTabs: RemoteTab[] = []
|
||||
try {
|
||||
await client.page.evaluate(async () => {
|
||||
await window.__store
|
||||
?.getState()
|
||||
.updateSettings({ terminalHiddenWorktreeRetentionBudget: false })
|
||||
})
|
||||
const worktreeIds = seed.worktreeIds.slice(0, TARGET_WORKTREE_COUNT)
|
||||
while (worktreeIds.length < TARGET_WORKTREE_COUNT) {
|
||||
const suffix = `${Date.now()}-${worktreeIds.length}`
|
||||
const created = await callRuntime<{ worktree: { id: string } }>(
|
||||
client.page,
|
||||
'worktree.create',
|
||||
{
|
||||
repo: seed.repoId,
|
||||
name: `paired-retention-${suffix}`,
|
||||
setupDecision: 'skip',
|
||||
activate: false,
|
||||
noParent: true
|
||||
}
|
||||
)
|
||||
worktreeIds.push(created.worktree.id)
|
||||
createdWorktreeIds.push(created.worktree.id)
|
||||
}
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(ids) =>
|
||||
ids.every((id) =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.some((w) => w.id === id)
|
||||
),
|
||||
worktreeIds
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
|
||||
for (const [index, worktreeId] of worktreeIds.entries()) {
|
||||
const marker = `PAIR_RETENTION_${index}`
|
||||
const created = await callRuntime<{
|
||||
tab: { parentTabId: string; terminal: string | null }
|
||||
}>(client.page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixtureCommand(marker),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
if (!created.tab.terminal) {
|
||||
throw new Error(`Paired retention terminal ${index} was not published`)
|
||||
}
|
||||
const terminal = created.tab.terminal
|
||||
const tabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await client.page.evaluate(
|
||||
(id) => window.__store?.getState().setActiveWorktree(id),
|
||||
worktreeId
|
||||
)
|
||||
const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${tabId}"]`)
|
||||
await expect(tab).toBeVisible({ timeout: 30_000 })
|
||||
await tab.click()
|
||||
const originalPtyId = await waitForActivePanePtyId(client.page, 30_000)
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: 'FILL',
|
||||
enter: true,
|
||||
client: { id: 'paired-retention-memory-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`FILLED:${marker}`)
|
||||
remoteTabs.push({ marker, originalPtyId, tabId, terminal, worktreeId })
|
||||
}
|
||||
|
||||
await client.page.evaluate(() => window.__store?.getState().setActiveView('tasks'))
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(ids) => ids.filter((id) => window.__paneManagers?.has(id)).length,
|
||||
remoteTabs.map((tab) => tab.tabId)
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toBe(TARGET_WORKTREE_COUNT)
|
||||
const baseline = await readPairedRetentionSample(
|
||||
client.page,
|
||||
remoteTabs.map((tab) => tab.tabId)
|
||||
)
|
||||
expect(baseline.bufferCells).toBeGreaterThan(MIN_STAGED_BUFFER_CELLS)
|
||||
|
||||
const lagProbe = await startRendererLagProbe(client.page)
|
||||
await client.page.evaluate(async () => {
|
||||
await window.__store
|
||||
?.getState()
|
||||
.updateSettings({ terminalHiddenWorktreeRetentionBudget: true })
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(ids) => ids.filter((id) => window.__paneManagers?.has(id)).length,
|
||||
remoteTabs.map((tab) => tab.tabId)
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toBe(RETENTION_LIMIT)
|
||||
const maxLagMs = await lagProbe.evaluate((probe) => probe.stop())
|
||||
await lagProbe.dispose()
|
||||
const after = await readPairedRetentionSample(
|
||||
client.page,
|
||||
remoteTabs.map((tab) => tab.tabId)
|
||||
)
|
||||
expect(after.bufferCells).toBeLessThanOrEqual(baseline.bufferCells * MAX_RETAINED_CELL_FRACTION)
|
||||
expect(after.mountedTargetManagers).toBe(RETENTION_LIMIT)
|
||||
expect(maxLagMs).toBeLessThan(MAX_EVICTION_LAG_MS)
|
||||
if (baseline.heapBytes !== null && after.heapBytes !== null) {
|
||||
expect(after.heapBytes).toBeLessThanOrEqual(baseline.heapBytes + MAX_HEAP_GROWTH_BYTES)
|
||||
}
|
||||
|
||||
const evicted = await client.page.evaluate(
|
||||
(tabs) => tabs.find((tab) => !window.__paneManagers?.has(tab.tabId)) ?? null,
|
||||
remoteTabs
|
||||
)
|
||||
if (!evicted) {
|
||||
throw new Error('Retention budget did not evict a paired terminal')
|
||||
}
|
||||
await client.page.evaluate((worktreeId) => {
|
||||
const state = window.__store?.getState()
|
||||
state?.setActiveView('terminal')
|
||||
state?.setActiveWorktree(worktreeId)
|
||||
}, evicted.worktreeId)
|
||||
const restored = client.page.locator(
|
||||
`[data-testid="sortable-tab"][data-tab-id="${evicted.tabId}"]`
|
||||
)
|
||||
await expect(restored).toBeVisible({ timeout: 30_000 })
|
||||
await restored.click()
|
||||
expect(await waitForActivePanePtyId(client.page, 30_000)).toBe(evicted.originalPtyId)
|
||||
const liveMarker = `AFTER_RETENTION_${Date.now()}`
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal: evicted.terminal,
|
||||
text: liveMarker,
|
||||
enter: true,
|
||||
client: { id: 'paired-retention-memory-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${liveMarker}`)
|
||||
} finally {
|
||||
for (const tab of remoteTabs) {
|
||||
await callRuntime(client.page, 'terminal.closeTab', { terminal: tab.terminal }).catch(
|
||||
() => undefined
|
||||
)
|
||||
}
|
||||
await client.page
|
||||
.evaluate((id) => window.__store?.getState().setActiveWorktree(id), seed.worktreeIds[0])
|
||||
.catch(() => undefined)
|
||||
for (const worktreeId of createdWorktreeIds.toReversed()) {
|
||||
await callRuntime(client.page, 'worktree.rm', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
force: true,
|
||||
runHooks: false
|
||||
}).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,251 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import type { RuntimeTerminalRead } from '../../src/shared/runtime-types'
|
||||
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
createRuntimeDesktopPairingOffer,
|
||||
launchPairedWebClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal'
|
||||
|
||||
const MIN_EXHAUSTED_ACK_BYTES = 400 * 1024
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-stalled-stream-'))
|
||||
const fixturePath = path.join(scratch, 'stalled-stream-terminal.mjs')
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
"process.stdout.write('PAIRED_STALL_READY\\r\\n')",
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"let pending = ''",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' pending += data',
|
||||
' const commands = pending.split(/\\r\\n|\\r|\\n/)',
|
||||
' pending = commands.pop() ?? ""',
|
||||
' for (const input of commands) {',
|
||||
" if (input === 'GO') {",
|
||||
" for (let row = 0; row < 16_000; row += 1) process.stdout.write(`flood-${row}-${'x'.repeat(80)}\\r\\n`)",
|
||||
" process.stdout.write('HOST_FLOOD_COMPLETE\\r\\n')",
|
||||
' continue',
|
||||
' }',
|
||||
' process.stdout.write(`LIVE:${input}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(): string {
|
||||
const command = [process.execPath, fixturePath]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(page: Page, method: string, params: unknown): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ method, params }) => {
|
||||
const response = await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
test('restarts one ACK-starved paired terminal stream without replacing its PTY @headful', async ({
|
||||
electronApp,
|
||||
orcaPage
|
||||
}) => {
|
||||
test.setTimeout(120_000)
|
||||
const liveMarker = `PAIRED_STALL_RECOVERED_${Date.now()}`
|
||||
const worktree = await orcaPage.evaluate(() => {
|
||||
const state = window.__store?.getState()
|
||||
const id = state?.activeWorktreeId
|
||||
const active = state?.allWorktrees().find((candidate) => candidate.id === id)
|
||||
if (!active) {
|
||||
throw new Error('Headed host did not select its seeded worktree')
|
||||
}
|
||||
return { id: active.id }
|
||||
})
|
||||
const offer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const client = await launchPairedWebClient(electronApp, offer, {
|
||||
disableRemoteTerminalStallRecovery:
|
||||
process.env.ORCA_E2E_DISABLE_REMOTE_TERMINAL_STALL_RECOVERY === '1'
|
||||
})
|
||||
let terminal: string | null = null
|
||||
try {
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(worktreeId) =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.some((candidate) => candidate.id === worktreeId),
|
||||
worktree.id
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
const created = await callRuntime<{
|
||||
tab: { parentTabId: string; terminal: string | null }
|
||||
}>(client.page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktree.id}`,
|
||||
command: fixtureCommand(),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
terminal = created.tab.terminal
|
||||
if (!terminal) {
|
||||
throw new Error('Paired runtime did not publish the stalled-stream fixture')
|
||||
}
|
||||
const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
({ tabId, worktreeId }) =>
|
||||
(window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some(
|
||||
(tab) => tab.id === tabId
|
||||
),
|
||||
{ tabId: webTabId, worktreeId: worktree.id }
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
await client.page.evaluate(
|
||||
(worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId),
|
||||
worktree.id
|
||||
)
|
||||
const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`)
|
||||
await expect(tab).toBeVisible({ timeout: 30_000 })
|
||||
await tab.click()
|
||||
const originalPtyId = await waitForActivePanePtyId(client.page, 30_000)
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain('PAIRED_STALL_READY')
|
||||
|
||||
await client.page.evaluate((target) => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: { hold: (terminals: string[]) => void }
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
if (!gate) {
|
||||
throw new Error('Remote terminal multiplex ACK gate is unavailable')
|
||||
}
|
||||
gate.hold([target])
|
||||
}, terminal)
|
||||
const textarea = client.page.locator('.xterm-helper-textarea:visible').first()
|
||||
await textarea.focus()
|
||||
await client.page.keyboard.type('GO')
|
||||
await client.page.keyboard.press('Enter')
|
||||
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(() => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: {
|
||||
snapshot: () => { heldAckChars: number }
|
||||
}
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
return gate?.snapshot().heldAckChars ?? 0
|
||||
}),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBeGreaterThan(MIN_EXHAUSTED_ACK_BYTES)
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const result = await callRuntime<{ terminal: RuntimeTerminalRead }>(
|
||||
client.page,
|
||||
'terminal.read',
|
||||
{ terminal }
|
||||
)
|
||||
return result.terminal.tail.join('\n').includes('HOST_FLOOD_COMPLETE')
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
|
||||
const beforeInput = await callRuntime<{ terminal: RuntimeTerminalRead }>(
|
||||
client.page,
|
||||
'terminal.read',
|
||||
{ terminal }
|
||||
)
|
||||
const sent = await callRuntime<{ send: { accepted: boolean } }>(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: liveMarker,
|
||||
enter: true,
|
||||
client: { id: 'paired-stalled-stream-e2e', type: 'desktop' }
|
||||
})
|
||||
expect(sent.send.accepted).toBe(true)
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const result = await callRuntime<{ terminal: RuntimeTerminalRead }>(
|
||||
client.page,
|
||||
'terminal.read',
|
||||
{ terminal }
|
||||
)
|
||||
return Number(result.terminal.latestCursor)
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBeGreaterThan(Number(beforeInput.terminal.latestCursor))
|
||||
expect(await getTerminalContent(client.page)).not.toContain(liveMarker)
|
||||
expect(
|
||||
await client.page.evaluate(
|
||||
({ target, text }) => {
|
||||
const gate = (
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: {
|
||||
sendInput: (terminal: string, text: string) => number
|
||||
}
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate
|
||||
return gate?.sendInput(target, text) ?? 0
|
||||
},
|
||||
{ target: terminal, text: '\r' }
|
||||
)
|
||||
).toBe(1)
|
||||
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${liveMarker}`)
|
||||
expect(await waitForActivePanePtyId(client.page, 30_000)).toBe(originalPtyId)
|
||||
await expect(tab).toHaveAttribute('data-active', 'true')
|
||||
} finally {
|
||||
await client.page
|
||||
.evaluate(() => {
|
||||
;(
|
||||
window as typeof window & {
|
||||
__remoteTerminalMultiplexAckGate?: { release: () => void }
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate?.release()
|
||||
})
|
||||
.catch(() => undefined)
|
||||
if (terminal) {
|
||||
await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,466 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import type { RuntimeTerminalRead } from '../../src/shared/runtime-types'
|
||||
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
createRuntimeDesktopPairingOffer,
|
||||
launchPairedWebClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal'
|
||||
|
||||
const RETENTION_PARK_DELAY_MS = 100
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-truncated-tail-'))
|
||||
const fixturePath = path.join(scratch, 'truncated-tail-terminal.mjs')
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
'const marker = process.argv[2]',
|
||||
'let flooded = false',
|
||||
"process.stdout.write('REMOTE_TRUNCATED_TAIL_READY\\r\\n')",
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
" if (!flooded && data.includes('GO')) {",
|
||||
' flooded = true',
|
||||
" for (let row = 0; row < 4_000; row += 1) process.stdout.write(`overflow-${row}-${'x'.repeat(80)}\\r\\n`)",
|
||||
' process.stdout.write(`${marker}\\r\\n`)',
|
||||
' return',
|
||||
' }',
|
||||
' process.stdout.write(`LIVE:${data.trim()}\\r\\n`)',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test.use({
|
||||
orcaAppExtraEnv: {
|
||||
ORCA_E2E_TERMINAL_PARKING_DELAY_MS: String(RETENTION_PARK_DELAY_MS),
|
||||
ORCA_E2E_TERMINAL_RETENTION_LIMIT: '1'
|
||||
}
|
||||
})
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(marker: string): string {
|
||||
const command = [process.execPath, fixturePath, marker]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(page: Page, method: string, params: unknown): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ method, params }) => {
|
||||
const response = await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
test('paints a paired remote terminal when only its retained text tail overflowed @headful', async ({
|
||||
electronApp,
|
||||
orcaPage
|
||||
}) => {
|
||||
test.setTimeout(120_000)
|
||||
const firstPaintMarker = `REMOTE_TRUNCATED_TAIL_FIRST_PAINT_${Date.now()}`
|
||||
const liveMarker = `REMOTE_TRUNCATED_TAIL_LIVE_${Date.now()}`
|
||||
const worktree = await orcaPage.evaluate(() => {
|
||||
const state = window.__store?.getState()
|
||||
const activeWorktreeId = state?.activeWorktreeId
|
||||
if (!activeWorktreeId) {
|
||||
throw new Error('Headed host did not select its seeded worktree')
|
||||
}
|
||||
const activeWorktree = state
|
||||
.allWorktrees()
|
||||
.find((candidate) => candidate.id === activeWorktreeId)
|
||||
if (!activeWorktree) {
|
||||
throw new Error('Headed host active worktree was absent from inventory')
|
||||
}
|
||||
return { id: activeWorktree.id, path: activeWorktree.path }
|
||||
})
|
||||
const offer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const client = await launchPairedWebClient(electronApp, offer)
|
||||
let terminal: string | null = null
|
||||
try {
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(worktreeId) =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.some((candidate) => candidate.id === worktreeId),
|
||||
worktree.id
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
const created = await callRuntime<{
|
||||
tab: {
|
||||
parentTabId: string
|
||||
leafId: string
|
||||
terminal: string | null
|
||||
}
|
||||
}>(client.page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktree.id}`,
|
||||
command: fixtureCommand(firstPaintMarker),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
terminal = created.tab.terminal
|
||||
if (!terminal) {
|
||||
throw new Error('Paired runtime did not publish the overflow fixture terminal')
|
||||
}
|
||||
|
||||
const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
({ tabId, worktreeId }) =>
|
||||
(window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some(
|
||||
(tab) => tab.id === tabId
|
||||
),
|
||||
{ tabId: webTabId, worktreeId: worktree.id }
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
await client.page.evaluate(
|
||||
(worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId),
|
||||
worktree.id
|
||||
)
|
||||
const remoteTab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`)
|
||||
await expect(remoteTab).toBeVisible({ timeout: 30_000 })
|
||||
await remoteTab.click()
|
||||
await expect(remoteTab).toHaveAttribute('data-active', 'true')
|
||||
await waitForActivePanePtyId(client.page, 30_000)
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: 'WARMUP',
|
||||
enter: true,
|
||||
client: { id: 'paired-truncated-tail-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain('LIVE:WARMUP')
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: 'GO',
|
||||
enter: true,
|
||||
client: { id: 'paired-truncated-tail-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(firstPaintMarker)
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const result = await callRuntime<{ terminal: RuntimeTerminalRead }>(
|
||||
client.page,
|
||||
'terminal.read',
|
||||
{ terminal }
|
||||
)
|
||||
return {
|
||||
marker: result.terminal.tail.join('\n').includes(firstPaintMarker),
|
||||
truncated: result.terminal.truncated
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toEqual({ marker: true, truncated: true })
|
||||
|
||||
await client.page.reload()
|
||||
await client.page.locator('[data-worktree-sidebar]').waitFor({
|
||||
state: 'visible',
|
||||
timeout: 30_000
|
||||
})
|
||||
await client.page.evaluate(
|
||||
(worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId),
|
||||
worktree.id
|
||||
)
|
||||
const restoredRemoteTab = client.page.locator(
|
||||
`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`
|
||||
)
|
||||
await expect(restoredRemoteTab).toBeVisible({ timeout: 30_000 })
|
||||
await restoredRemoteTab.click()
|
||||
await expect(restoredRemoteTab).toHaveAttribute('data-active', 'true')
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(firstPaintMarker)
|
||||
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal,
|
||||
text: liveMarker,
|
||||
enter: true,
|
||||
client: { id: 'paired-truncated-tail-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${liveMarker}`)
|
||||
} finally {
|
||||
if (terminal) {
|
||||
await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
test('bounds hidden paired-runtime pane managers and restores PTY output @headful', async ({
|
||||
electronApp,
|
||||
orcaPage
|
||||
}) => {
|
||||
test.setTimeout(120_000)
|
||||
const worktreeIds = await orcaPage.evaluate(() =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.slice(0, 2)
|
||||
.map((worktree) => worktree.id)
|
||||
)
|
||||
if (!worktreeIds || worktreeIds.length < 2) {
|
||||
throw new Error('Paired retention fixture requires two seeded worktrees')
|
||||
}
|
||||
const offer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const client = await launchPairedWebClient(electronApp, offer, {
|
||||
terminalParkingDelayMs: RETENTION_PARK_DELAY_MS,
|
||||
terminalRetentionLimit: 1
|
||||
})
|
||||
const createdTerminals: string[] = []
|
||||
try {
|
||||
expect(await client.page.evaluate(() => window.api.e2e.getConfig())).toMatchObject({
|
||||
exposeStore: true,
|
||||
terminalParkingDelayMs: RETENTION_PARK_DELAY_MS,
|
||||
terminalRetentionLimit: 1
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate((ids) => {
|
||||
const known = new Set(
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.map((worktree) => worktree.id)
|
||||
)
|
||||
return ids.every((id) => known.has(id))
|
||||
}, worktreeIds),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
await client.page.evaluate(async () => {
|
||||
await window.__store
|
||||
?.getState()
|
||||
.updateSettings({ terminalHiddenWorktreeRetentionBudget: false })
|
||||
})
|
||||
|
||||
const remoteTabs: { tabId: string; terminal: string; worktreeId: string; marker: string }[] = []
|
||||
for (const [index, worktreeId] of worktreeIds.entries()) {
|
||||
const marker = `PAIRED_RETENTION_${index}_${Date.now()}`
|
||||
const created = await callRuntime<{
|
||||
tab: { parentTabId: string; terminal: string | null }
|
||||
}>(client.page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixtureCommand(marker),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
if (!created.tab.terminal) {
|
||||
throw new Error(`Paired retention terminal ${index} was not published`)
|
||||
}
|
||||
createdTerminals.push(created.tab.terminal)
|
||||
const tabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
({ tabId, worktreeId }) =>
|
||||
(window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some(
|
||||
(tab) => tab.id === tabId
|
||||
),
|
||||
{ tabId, worktreeId }
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
await client.page.evaluate(
|
||||
(id) => window.__store?.getState().setActiveWorktree(id),
|
||||
worktreeId
|
||||
)
|
||||
const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${tabId}"]`)
|
||||
await expect(tab).toBeVisible({ timeout: 30_000 })
|
||||
await tab.click()
|
||||
await waitForActivePanePtyId(client.page, 30_000)
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal: created.tab.terminal,
|
||||
text: marker,
|
||||
enter: true,
|
||||
client: { id: 'paired-retention-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${marker}`)
|
||||
remoteTabs.push({ tabId, terminal: created.tab.terminal, worktreeId, marker })
|
||||
}
|
||||
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(() => ({
|
||||
parkDelayMs: window.__terminalParkingDebug?.parkDelayMs,
|
||||
retentionLimit: window.__terminalParkingDebug?.retentionLimit
|
||||
})),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toEqual({ parkDelayMs: RETENTION_PARK_DELAY_MS, retentionLimit: 1 })
|
||||
|
||||
await client.page.evaluate(() => window.__store?.getState().setActiveView('tasks'))
|
||||
const controlStartedAt = Date.now()
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
({ tabIds, controlStartedAt, delayMs }) => ({
|
||||
heldLongEnough: Date.now() - controlStartedAt >= delayMs * 4,
|
||||
mounted: tabIds.filter((tabId) => window.__paneManagers?.has(tabId)).length
|
||||
}),
|
||||
{
|
||||
tabIds: remoteTabs.map((tab) => tab.tabId),
|
||||
controlStartedAt,
|
||||
delayMs: RETENTION_PARK_DELAY_MS
|
||||
}
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toEqual({ heldLongEnough: true, mounted: 2 })
|
||||
|
||||
await client.page.evaluate(async () => {
|
||||
await window.__store
|
||||
?.getState()
|
||||
.updateSettings({ terminalHiddenWorktreeRetentionBudget: true })
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
({ delayMs, tabIds: [olderTabId, newerTabId], worktreeIds }) => {
|
||||
const state = window.__store?.getState()
|
||||
const terminalTabs = Object.values(state?.tabsByWorktree ?? {}).flat()
|
||||
const verdicts = window.__terminalParkingDebug?.worktreeVerdicts() ?? []
|
||||
return {
|
||||
activeView: state?.activeView,
|
||||
budgetEnabled: state?.settings?.terminalHiddenWorktreeRetentionBudget,
|
||||
newerMounted: window.__paneManagers?.has(newerTabId),
|
||||
olderMounted: window.__paneManagers?.has(olderTabId),
|
||||
remotePtys: [olderTabId, newerTabId].map((tabId) =>
|
||||
terminalTabs.find((tab) => tab.id === tabId)?.ptyId?.startsWith('remote:')
|
||||
),
|
||||
verdicts: worktreeIds.map((worktreeId) => {
|
||||
const verdict = verdicts.find((candidate) => candidate.worktreeId === worktreeId)
|
||||
return verdict
|
||||
? {
|
||||
forceParked: verdict.forceParked,
|
||||
hasActivityTerminalPortal: verdict.hasActivityTerminalPortal,
|
||||
hasPendingSpawnWork: verdict.hasPendingSpawnWork,
|
||||
hidden: verdict.hiddenSinceMs !== null,
|
||||
hiddenPastDelay:
|
||||
verdict.hiddenSinceMs !== null &&
|
||||
Date.now() - verdict.hiddenSinceMs >= delayMs,
|
||||
isVisible: verdict.isVisible,
|
||||
ordinaryParkingCovers: verdict.ordinaryParkingCovers,
|
||||
parkCooldown:
|
||||
verdict.parkCooldownUntilMs !== null &&
|
||||
Date.now() < verdict.parkCooldownUntilMs,
|
||||
shouldMeasureHiddenWorktree: verdict.shouldMeasureHiddenWorktree
|
||||
}
|
||||
: null
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
delayMs: RETENTION_PARK_DELAY_MS,
|
||||
tabIds: [remoteTabs[0]!.tabId, remoteTabs[1]!.tabId],
|
||||
worktreeIds
|
||||
}
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toEqual({
|
||||
activeView: 'tasks',
|
||||
budgetEnabled: true,
|
||||
newerMounted: true,
|
||||
olderMounted: false,
|
||||
remotePtys: [true, true],
|
||||
verdicts: [
|
||||
{
|
||||
forceParked: true,
|
||||
hasActivityTerminalPortal: false,
|
||||
hasPendingSpawnWork: false,
|
||||
hidden: true,
|
||||
hiddenPastDelay: true,
|
||||
isVisible: false,
|
||||
ordinaryParkingCovers: false,
|
||||
parkCooldown: false,
|
||||
shouldMeasureHiddenWorktree: false
|
||||
},
|
||||
{
|
||||
forceParked: false,
|
||||
hasActivityTerminalPortal: false,
|
||||
hasPendingSpawnWork: false,
|
||||
hidden: true,
|
||||
hiddenPastDelay: true,
|
||||
isVisible: false,
|
||||
ordinaryParkingCovers: false,
|
||||
parkCooldown: false,
|
||||
shouldMeasureHiddenWorktree: false
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const older = remoteTabs[0]!
|
||||
await client.page.evaluate((worktreeId) => {
|
||||
const state = window.__store?.getState()
|
||||
state?.setActiveView('terminal')
|
||||
state?.setActiveWorktree(worktreeId)
|
||||
}, older.worktreeId)
|
||||
const olderTab = client.page.locator(
|
||||
`[data-testid="sortable-tab"][data-tab-id="${older.tabId}"]`
|
||||
)
|
||||
await expect(olderTab).toBeVisible({ timeout: 30_000 })
|
||||
await olderTab.click()
|
||||
await waitForActivePanePtyId(client.page, 30_000)
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain(`LIVE:${older.marker}`)
|
||||
await callRuntime(client.page, 'terminal.send', {
|
||||
terminal: older.terminal,
|
||||
text: 'AFTER_RESTORE',
|
||||
enter: true,
|
||||
client: { id: 'paired-retention-e2e', type: 'desktop' }
|
||||
})
|
||||
await expect
|
||||
.poll(() => getTerminalContent(client.page), { timeout: 30_000 })
|
||||
.toContain('LIVE:AFTER_RESTORE')
|
||||
await expect(olderTab).toHaveAttribute('data-active', 'true')
|
||||
} finally {
|
||||
for (const terminal of createdTerminals) {
|
||||
await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { JSHandle, Page } from '@stablyai/playwright-test'
|
||||
|
||||
export type PairedRetentionSample = {
|
||||
bufferCells: number
|
||||
heapBytes: number | null
|
||||
mountedTargetManagers: number
|
||||
targetPanes: number
|
||||
}
|
||||
|
||||
export async function readPairedRetentionSample(
|
||||
page: Page,
|
||||
tabIds: string[]
|
||||
): Promise<PairedRetentionSample> {
|
||||
try {
|
||||
const session = await page.context().newCDPSession(page)
|
||||
await session.send('HeapProfiler.collectGarbage')
|
||||
await session.detach()
|
||||
} catch {
|
||||
// GC only improves measurement fidelity.
|
||||
}
|
||||
return page.evaluate((targets) => {
|
||||
let bufferCells = 0
|
||||
let mountedTargetManagers = 0
|
||||
let targetPanes = 0
|
||||
for (const tabId of targets) {
|
||||
const manager = window.__paneManagers?.get(tabId)
|
||||
if (!manager) {
|
||||
continue
|
||||
}
|
||||
mountedTargetManagers += 1
|
||||
for (const pane of manager.getPanes?.() ?? []) {
|
||||
const buffer = pane.terminal?.buffer?.active
|
||||
if (!buffer) {
|
||||
continue
|
||||
}
|
||||
targetPanes += 1
|
||||
bufferCells += buffer.length * pane.terminal.cols
|
||||
}
|
||||
}
|
||||
const memory = (performance as Performance & { memory?: { usedJSHeapSize?: number } }).memory
|
||||
return {
|
||||
bufferCells,
|
||||
heapBytes: memory?.usedJSHeapSize ?? null,
|
||||
mountedTargetManagers,
|
||||
targetPanes
|
||||
}
|
||||
}, tabIds)
|
||||
}
|
||||
|
||||
export async function startRendererLagProbe(page: Page): Promise<JSHandle<{ stop: () => number }>> {
|
||||
return page.evaluateHandle(() => {
|
||||
const sampleMs = 16
|
||||
let lastAt = performance.now()
|
||||
let maxDriftMs = 0
|
||||
const timer = window.setInterval(() => {
|
||||
const now = performance.now()
|
||||
maxDriftMs = Math.max(maxDriftMs, now - lastAt - sampleMs)
|
||||
lastAt = now
|
||||
}, sampleMs)
|
||||
return {
|
||||
stop: () => {
|
||||
window.clearInterval(timer)
|
||||
return maxDriftMs
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user