From 252dbd60eaa6a3e1d5c334bdcb05d5e575275f07 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:11:55 -0400 Subject: [PATCH] fix(terminal): restore lossy initial remote snapshots (#17113) * fix(terminal): restore lossy initial remote snapshots * test(terminal): strengthen lossy snapshot causal oracle --- .github/workflows/pr.yml | 1 + config/reliability-gates.jsonc | 43 ++- ...stream-extraction-characterization.test.ts | 73 +++++ .../terminal-multiplex-initial-snapshot.ts | 4 +- ...roups-section-label-disambiguation.test.ts | 8 +- ...time-pty-transport-snapshot-replay.test.ts | 46 +++ ...e-runtime-pty-transport-stream-fixtures.ts | 4 +- ...emote-runtime-terminal-binary-snapshots.ts | 8 +- .../remote-runtime-terminal-e2e-control.ts | 10 + .../host-terminal-runtime-stub.ts | 17 +- ...ported-lossy-initial-snapshot.unit.test.ts | 214 ++++++++++++++ ...te-terminal-lossy-initial-snapshot.spec.ts | 278 ++++++++++++++++++ 12 files changed, 685 insertions(+), 21 deletions(-) create mode 100644 tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts create mode 100644 tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 68246a085b3..a5ebf9e8808 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -542,6 +542,7 @@ jobs: tests/e2e/cross-version-wire/release-checkout.unit.test.ts tests/e2e/cross-version-wire/cross-version-browser-placement.unit.test.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts + tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts managed_hook_node18: diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index f8372378a88..2e3de030e0c 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -8625,6 +8625,7 @@ "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "ORCA_E2E_FORWARD_APP_LOGS=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1 --repeat-each=10 --max-failures=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", @@ -8673,6 +8674,7 @@ "src/renderer/src/runtime/runtime-client-events.test.ts", "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-activation-inventory-fallback.test.ts", "src/shared/remote-runtime-shared-control-connection.test.ts", + "tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts", "tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts", "tests/e2e/paired-remote-terminal-stall-recovery.spec.ts", "tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts", @@ -8798,6 +8800,13 @@ "file": "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts", "assertions": ["recovers an ACK-starved stream from an isolated headless Orca host"] }, + { + "file": "tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts", + "assertions": [ + "a nonempty lossy initial snapshot paints exactly once on a paired client", + "the authoritative host cursor advances and live output resumes exactly once" + ] + }, { "file": "tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts", "assertions": [ @@ -16033,17 +16042,21 @@ "providers": ["paired-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["paired-runtime"], - "coverageNotes": "Loads the real host RPC methods, the real RpcDispatcher, and the real renderer terminal multiplexer from two builds (current working tree and the newest release tag) and drives them against each other over an in-process transport that reproduces production frame routing, including the host-side decode that silently drops unknown opcodes. Covers the terminal stream only; the session-tab sync channel, agent-session publications, file/Git RPCs, mobile E2EE framing, and the relay transport are uncovered.", + "coverageNotes": "Loads the real host RPC methods, real RpcDispatcher, real renderer terminal multiplexer, and codecs from the current working tree, exact pre-fix main, newest release tag, and the pinned reported 1.4.192-client/1.4.191-host commits. In-process transport reproduces production frame routing, including decoder-rejected opcodes. Covers the terminal stream only; session tabs, agent publications, file/Git RPCs, mobile E2EE framing, and relay transport are uncovered.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/12641", "https://github.com/stablyai/orca/pull/12655" ], "invariant": "A client and a server built from different releases must complete subscribe, input delivery to the process, hide/reveal buffer snapshot, transport drop, and resubscribe with no frame refused by the receiving build's decoder, the same negotiated capabilities, and the same published snapshot content — so a new optional field stays safe, a new opcode is only sent after negotiation, and a change in what the host publishes is visible before release.", - "oracle": "Run one fixed journey per pairing (old client/new server, new client/old server, and current/current as control) and assert the recorded step list, the exact named frame sequence, both subscribed events with their negotiated capabilities, the exact input texts the host wrote to the PTY before and after reconnect, the rendered snapshot and live-output content, and an empty set of decoder-rejected frames in either direction. Missing host runtime methods are reported by name so a harness gap can never be read as a wire incompatibility.", + "oracle": "Run one fixed journey per general pairing (old client/new server, new client/old server, and current/current) plus the pinned reported pair, exact pre-fix main control, and candidate controls. Assert exact frames, subscriptions, input, rendered snapshot/live content, and zero decoder rejects or missing host methods. The reported and pre-fix-main clients must render blank after discarding the nonempty lossy initial image; candidate pairings must paint it once, recover the first post-snapshot sequence gap, and resume contiguous live output once.", "commands": [ - "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts" + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts" + ], + "testFiles": [ + "tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts", + "tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts" ], - "testFiles": ["tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts"], "assertionRefs": [ { "file": "tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts", @@ -16054,6 +16067,16 @@ "expect(record.inputAtProcess).toEqual([JOURNEY_INPUTS.first, JOURNEY_INPUTS.second])", "expect(event.capabilities).toEqual({ outputPause: 1 })" ] + }, + { + "file": "tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts", + "assertions": [ + "the reported 1.4.192 client discards the reported 1.4.191 host's nonempty lossy snapshot", + "the exact pre-fix main client reproduces the same blank rendered terminal", + "the candidate client paints that old-host image exactly once and resumes live output", + "the candidate client paints the candidate-host image exactly once and resumes live output", + "both candidate pairings report zero decoder rejects and missing runtime methods" + ] } ], "evidenceRuns": [ @@ -16068,8 +16091,8 @@ } ], "runtimeBudget": { - "p95Seconds": 60, - "scope": "one baseline checkout extraction plus three in-process journeys" + "p95Seconds": 180, + "scope": "four historical checkout extractions plus seven in-process pairings" }, "flakeHistory": { "status": "not-started", @@ -16080,18 +16103,18 @@ "evidence": "Red proven separately for each rule by injecting the violation into the working tree and reverting it. Rule 2: adding opcode 17 and sending it ungated from the client turned new-client/old-server red with rejected rawOpcode 17 in the client-to-host direction, while old-client/new-server stayed green. Rule 3: making the host stop publishing the snapshot `source` field turned both new-server pairings red and left the old-server pairing green; trimming the published initial buffer removed the SnapshotChunk frame and failed the frame-sequence oracle. Rule 1: adding an optional `hiddenOutputReason` field to the snapshot frame kept all pairings green, and making the client require that field turned only new-client/old-server red." }, "performanceBudget": { - "required": false, - "evidence": "Test-only infrastructure; it adds no product code path. The extracted baseline tree is cached by resolved commit, so repeat runs skip extraction and each journey completes in roughly 35ms." + "required": true, + "evidence": "Product reconciliation adds one existing-target comparison per completed snapshot and no timer, polling, subscription, provider call, scan, or subprocess. The test-only host hook adds one environment read per initial subscribe. Extracted trees are cached by commit." }, "promotionCriteria": [ - "Extend the matrix beyond two version points, for example the previous two minor releases.", + "Extend the matrix beyond the newest stable release and pinned reported pair.", "Cover a second wire surface, starting with the session-tab sync channel that PR #12641 changed.", "Collect 100 consecutive CI passes on the dedicated cross-version-wire job.", "Run the job on Linux and Windows runners, not only macOS locally." ], "knownGaps": [ "Only the terminal stream is covered; session tabs, agent sessions, file/Git RPCs, mobile E2EE framing, and the relay transport are not.", - "Only two version points are compared, so a regression introduced and reverted between them is invisible.", + "The newest stable release and pinned reported pair do not expose every intermediate regression.", "The host runtime is a stub around a fake PTY, so real PTY, daemon, and SSH provider behavior is out of scope.", "The baseline is the newest release tag by default, so the compared pair changes when a new release is cut unless ORCA_CROSS_VERSION_BASELINE_REF pins it.", "tests/ is outside every tsconfig include, so the harness is linted and executed but not typechecked." diff --git a/src/main/runtime/rpc/methods/terminal-stream-extraction-characterization.test.ts b/src/main/runtime/rpc/methods/terminal-stream-extraction-characterization.test.ts index db05c6b3fe5..33731999dd4 100644 --- a/src/main/runtime/rpc/methods/terminal-stream-extraction-characterization.test.ts +++ b/src/main/runtime/rpc/methods/terminal-stream-extraction-characterization.test.ts @@ -2,13 +2,16 @@ import { describe, expect, it, vi } from 'vitest' import { TerminalStreamOpcode, decodeTerminalStreamJson, + decodeTerminalStreamText, encodeTerminalStreamFrame } from '../../../../shared/terminal-stream-protocol' import { sendSnapshotFrames } from './terminal/terminal-snapshot-publication' +import { publishMultiplexInitialSnapshot } from './terminal/terminal-multiplex-initial-snapshot' import { TerminalSourceRangeRegistry } from '../terminal-source-range-registry' import { initializeMultiplexStream } from './terminal/terminal-multiplex-stream-initialization' import type { TerminalMultiplexConnection } from './terminal/terminal-multiplex-connection' import type { OrcaRuntimeService } from '../../orca-runtime' +import type { TerminalMultiplexStream } from './terminal/terminal-stream-types' // These assertions protect the permanent binary publication seam while its state machine // lives in concrete terminal domain modules. @@ -103,6 +106,76 @@ describe('terminal stream extraction characterization', () => { ]).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]) }) + it('publishes the latest nonempty initial image when startup output overflows twice', async () => { + const frames: { opcode: TerminalStreamOpcode; payload: Uint8Array }[] = [] + const emit = vi.fn() + let readCount = 0 + const stream = { + streamId: 7, + terminal: 'terminal', + ptyId: 'pty', + isMobile: false, + ackOutputSourceRanges: false, + supportsOutputPause: false, + pendingOutput: [{ data: 'lost-live', bytes: 9 }], + pendingOutputBytes: 9, + pendingOutputOverflowed: true, + outputBatcher: { push: vi.fn(), flush: vi.fn() } + } as unknown as TerminalMultiplexStream + const runtime = { + readTerminal: vi.fn(async () => { + readCount += 1 + if (readCount === 2) { + stream.pendingOutputOverflowed = true + } + return { tail: ['fallback'] } + }), + serializeTerminalBuffer: vi.fn(async () => ({ + data: 'AUTHORITATIVE_INITIAL_MARKER', + cols: 80, + rows: 24, + seq: 41, + scrollbackRows: 0, + truncatedByByteBudget: false + })), + getTerminalSize: vi.fn(() => ({ cols: 80, rows: 24 })), + getMobileDisplayMode: vi.fn(() => 'fit'), + getLayout: vi.fn(() => null) + } as unknown as OrcaRuntimeService + const state = { + runtime, + streams: new Map([[stream.streamId, stream]]), + emit, + closed: false, + sendFrame: vi.fn((streamId, opcode, payload = new Uint8Array()) => { + expect(streamId).toBe(stream.streamId) + frames.push({ opcode, payload }) + return true + }) + } as unknown as TerminalMultiplexConnection + + await publishMultiplexInitialSnapshot( + state, + { streamId: stream.streamId, terminal: stream.terminal }, + stream + ) + + expect(emit).toHaveBeenCalledWith( + expect.objectContaining({ type: 'subscribed', truncated: true }) + ) + expect(frames.map(({ opcode }) => opcode)).toEqual([ + TerminalStreamOpcode.SnapshotStart, + TerminalStreamOpcode.SnapshotChunk, + TerminalStreamOpcode.SnapshotEnd + ]) + expect(decodeTerminalStreamJson(frames[0]!.payload)).toMatchObject({ + seq: 41, + truncated: true + }) + expect(decodeTerminalStreamText(frames[1]!.payload)).toBe('AUTHORITATIVE_INITIAL_MARKER') + expect(stream.outputBatcher.push).not.toHaveBeenCalled() + }) + it('lets a slot-handler registration throw escape before per-stream catch ownership begins', async () => { const streams = new Map() const runtime = { diff --git a/src/main/runtime/rpc/methods/terminal/terminal-multiplex-initial-snapshot.ts b/src/main/runtime/rpc/methods/terminal/terminal-multiplex-initial-snapshot.ts index 6f1c2f8fc47..a2c9fadfb8d 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-multiplex-initial-snapshot.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-multiplex-initial-snapshot.ts @@ -23,12 +23,14 @@ export async function publishMultiplexInitialSnapshot( const { runtime, streams, emit } = state const { ptyId } = stream const isMobile = stream.isMobile + const forcedInitialSnapshotTruncated = + process.env.ORCA_E2E_FORCE_REMOTE_TERMINAL_INITIAL_SNAPSHOT_TRUNCATED === '1' let read = await runtime.readTerminal(request.terminal) let serialized = await serializeBudgetedMobileSnapshot(runtime, ptyId, isMobile) if (state.closed || streams.get(request.streamId) !== stream) { return null } - let initialOutputOverflowed = false + let initialOutputOverflowed = forcedInitialSnapshotTruncated if (stream.pendingOutputOverflowed) { stream.pendingOutput.splice(0) stream.pendingOutputBytes = 0 diff --git a/src/renderer/src/components/sidebar/worktree-list-groups-section-label-disambiguation.test.ts b/src/renderer/src/components/sidebar/worktree-list-groups-section-label-disambiguation.test.ts index 43152aa9b9a..687ac4f86a4 100644 --- a/src/renderer/src/components/sidebar/worktree-list-groups-section-label-disambiguation.test.ts +++ b/src/renderer/src/components/sidebar/worktree-list-groups-section-label-disambiguation.test.ts @@ -29,9 +29,11 @@ const makeSetup = (checkout: Repo, projectId: string): ProjectHostSetup => ({ }) function buildHeaders(repos: Repo[], projects: Project[], setups: ProjectHostSetup[]) { - const worktrees = repos.map( - (entry): Worktree => ({ ...worktree, id: `wt-${entry.id}`, repoId: entry.id }) - ) + const worktrees = repos.map((entry): Worktree => ({ + ...worktree, + id: `wt-${entry.id}`, + repoId: entry.id + })) const rows = buildRows( 'repo', worktrees, diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-snapshot-replay.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-snapshot-replay.test.ts index 8f671b1a7e8..c6d1b8508f7 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-snapshot-replay.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-snapshot-replay.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { Terminal } from '@xterm/headless' import { TerminalStreamOpcode, decodeTerminalStreamJson, @@ -94,6 +95,51 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onConnect).toHaveBeenCalled() }) + it('paints a nonempty lossy initial snapshot once before resuming live output', async () => { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const terminal = new Terminal({ cols: 80, rows: 24 }) + let xtermWrites = 0 + const write = (data: string): void => { + xtermWrites += 1 + terminal.write(data) + } + const onReplayData = vi.fn(write) + const onData = vi.fn(write) + const onConnect = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' }) + + await transport.connect({ url: '', callbacks: { onReplayData, onData, onConnect } }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + const { streamId } = latestSubscribePayload() + emitSnapshotFrame( + streamId, + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ kind: 'scrollback', cols: 80, rows: 24, seq: 41, truncated: true }) + ) + emitSnapshotFrame( + streamId, + TerminalStreamOpcode.SnapshotChunk, + encodeTerminalStreamText('AUTHORITATIVE_INITIAL_MARKER') + ) + emitSnapshotFrame(streamId, TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + const liveOutput = 'LIVE_AFTER_INITIAL' + const liveSeq = 41 + liveOutput.length + emitOutput(streamId, liveOutput, liveSeq) + + expect(onReplayData).toHaveBeenCalledOnce() + expect(onReplayData).toHaveBeenCalledWith('AUTHORITATIVE_INITIAL_MARKER') + expect(onConnect).toHaveBeenCalledOnce() + expect(onData).toHaveBeenCalledWith(liveOutput, expect.objectContaining({ seq: liveSeq })) + await vi.waitFor(() => { + const rendered = terminal.buffer.active.getLine(0)?.translateToString(true) ?? '' + expect({ rendered, xtermWrites }).toEqual({ + rendered: 'AUTHORITATIVE_INITIAL_MARKERLIVE_AFTER_INITIAL', + xtermWrites: 2 + }) + }) + terminal.dispose() + }) + it('resolves explicit binary snapshot requests without replaying into xterm', async () => { const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') const onReplayData = vi.fn() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-fixtures.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-fixtures.ts index 939b3b2ae07..978d0428060 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-fixtures.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-fixtures.ts @@ -68,12 +68,12 @@ export function createTerminalStreamFixtures(bindings: { }) } - function emitOutput(streamId: number, data: string): void { + function emitOutput(streamId: number, data: string, seq = 1): void { bindings.getCallbacks()?.onBinary?.( encodeTerminalStreamFrame({ opcode: TerminalStreamOpcode.Output, streamId, - seq: 1, + seq, payload: encodeTerminalStreamText(data) }) ) diff --git a/src/renderer/src/runtime/remote-runtime-terminal-binary-snapshots.ts b/src/renderer/src/runtime/remote-runtime-terminal-binary-snapshots.ts index 208d6e99f48..c90c00fe00b 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-binary-snapshots.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-binary-snapshots.ts @@ -4,6 +4,7 @@ import { type TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' import { TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR } from '../../../shared/terminal-multiplex-flow-control' +import { recordE2eRemoteTerminalInitialSnapshotTruncated } from './remote-runtime-terminal-e2e-control' import { RemoteRuntimeTerminalResponseController } from './remote-runtime-terminal-response-controller' import { MAX_REMOTE_TERMINAL_SNAPSHOT_BYTES, @@ -57,7 +58,12 @@ export abstract class RemoteRuntimeTerminalBinarySnapshots extends RemoteRuntime const target = stream.snapshotTarget const info = stream.snapshotInfo const pendingRequest = stream.pendingSnapshotRequest - const snapshotApplied = !stream.snapshotOverflowed && info?.truncated !== true + if (target === 'initial' && info?.truncated === true) { + recordE2eRemoteTerminalInitialSnapshotTruncated() + } + // Initial truncation drops retained history, but the latest-screen image remains authoritative. + const snapshotApplied = + !stream.snapshotOverflowed && (target === 'initial' || info?.truncated !== true) const matchesPendingRequest = target === 'request' && pendingRequest && diff --git a/src/renderer/src/runtime/remote-runtime-terminal-e2e-control.ts b/src/renderer/src/runtime/remote-runtime-terminal-e2e-control.ts index e919ec12ea0..792606369d0 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-e2e-control.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-e2e-control.ts @@ -15,6 +15,7 @@ type E2eRemoteTerminalMultiplexAckGateSnapshot = { releasedAckChars: number streamSubscribeCount: number streamUnsubscribeCount: number + initialSnapshotTruncatedCount: number transportSubscribeCount: number transportUnsubscribeCount: number } @@ -48,6 +49,7 @@ let e2eDroppedOutputFrames = 0 let e2eReleasedRemoteAckChars = 0 let e2eStreamSubscribeCount = 0 let e2eStreamUnsubscribeCount = 0 +let e2eInitialSnapshotTruncatedCount = 0 let e2eTransportSubscribeCount = 0 let e2eTransportUnsubscribeCount = 0 @@ -84,11 +86,18 @@ function getE2eRemoteAckSnapshot( releasedAckChars: e2eReleasedRemoteAckChars, streamSubscribeCount: e2eStreamSubscribeCount, streamUnsubscribeCount: e2eStreamUnsubscribeCount, + initialSnapshotTruncatedCount: e2eInitialSnapshotTruncatedCount, transportSubscribeCount: e2eTransportSubscribeCount, transportUnsubscribeCount: e2eTransportUnsubscribeCount } } +export function recordE2eRemoteTerminalInitialSnapshotTruncated(): void { + if (e2eConfig.exposeStore) { + e2eInitialSnapshotTruncatedCount += 1 + } +} + export function recordE2eRemoteTransportSubscribe(): void { if (e2eConfig.exposeStore) { e2eTransportSubscribeCount += 1 @@ -204,6 +213,7 @@ export function resetRemoteRuntimeTerminalE2eState(): void { e2eReleasedRemoteAckChars = 0 e2eStreamSubscribeCount = 0 e2eStreamUnsubscribeCount = 0 + e2eInitialSnapshotTruncatedCount = 0 e2eTransportSubscribeCount = 0 e2eTransportUnsubscribeCount = 0 } diff --git a/tests/e2e/cross-version-wire/host-terminal-runtime-stub.ts b/tests/e2e/cross-version-wire/host-terminal-runtime-stub.ts index 528e1a85874..f353ec3a097 100644 --- a/tests/e2e/cross-version-wire/host-terminal-runtime-stub.ts +++ b/tests/e2e/cross-version-wire/host-terminal-runtime-stub.ts @@ -35,6 +35,7 @@ export function createHostTerminalRuntimeStub( cols?: number rows?: number initialBuffer?: string + overflowInitialSnapshots?: boolean } = {} ): HostTerminalRuntimeStub { const terminalHandle = options.terminalHandle ?? 'terminal-journey' @@ -93,15 +94,23 @@ export function createHostTerminalRuntimeStub( terminalOwner: 'shell' }> => { stub.serializeCount++ - return { + const snapshot = { data: stub.buffer, cols, rows, seq: outputSequence, - source: 'headless', - alternateScreen: false, - terminalOwner: 'shell' + source: 'headless' as const, + alternateScreen: false as const, + terminalOwner: 'shell' as const } + if (options.overflowInitialSnapshots && stub.serializeCount <= 2) { + const data = 'x'.repeat(300 * 1024) + outputSequence += data.length + for (const listener of Array.from(dataListeners)) { + listener(data, { seq: outputSequence, rawLength: data.length }) + } + } + return snapshot } const runtime: Record = { diff --git a/tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts b/tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts new file mode 100644 index 00000000000..98c510c22ae --- /dev/null +++ b/tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { Terminal } from '@xterm/headless' +import { createHostTerminalRuntimeStub } from './host-terminal-runtime-stub' +import { createTerminalWireLink } from './terminal-wire-link' +import { + loadTerminalWireBuild, + WORKING_TREE, + type TerminalWireBuild +} from './versioned-terminal-wire' + +const REPORTED_CLIENT_REF = '4cb013c0a9' +const REPORTED_HOST_REF = '4bb337741c' +const PRE_FIX_MAIN_REF = 'fd9125ea8c' +const MARKER = 'REPORTED_LOSSY_INITIAL_MARKER' +const RECOVERED_LIVE_MARKER = 'RECOVERED_LIVE_AFTER_INITIAL' +const CONTINUED_LIVE_MARKER = 'CONTINUED_LIVE_AFTER_RECOVERY' +const TIMEOUT_MS = 180_000 + +let candidate: TerminalWireBuild +let preFixMain: TerminalWireBuild +let reportedClient: TerminalWireBuild +let reportedHost: TerminalWireBuild + +beforeAll(async () => { + ;[candidate, preFixMain, reportedClient, reportedHost] = await Promise.all([ + loadTerminalWireBuild(WORKING_TREE), + loadTerminalWireBuild(PRE_FIX_MAIN_REF), + loadTerminalWireBuild(REPORTED_CLIENT_REF), + loadTerminalWireBuild(REPORTED_HOST_REF) + ]) +}, TIMEOUT_MS) + +afterEach(() => { + expect(typeof globalThis.window).toBe('undefined') +}) + +async function runLossyInitialSnapshotPair(args: { + clientBuild: TerminalWireBuild + hostBuild: TerminalWireBuild + exerciseLiveRecovery?: boolean +}): Promise<{ + frames: string[] + missingRuntimeMethods: string[] + rejected: unknown[] + rendered: string + snapshotStarts: Record[] + snapshots: string[] +}> { + const hostStub = createHostTerminalRuntimeStub({ + initialBuffer: MARKER, + overflowInitialSnapshots: true + }) + const link = createTerminalWireLink({ ...args, hostStub }) + const snapshots: string[] = [] + const terminalModel = new Terminal({ cols: 120, rows: 40 }) + let subscribed = 0 + try { + const terminal = await args.clientBuild.client + .getRemoteRuntimeTerminalMultiplexer('reported-lossy-initial') + .subscribeTerminal({ + terminal: hostStub.terminalHandle, + client: { id: 'reported-client', type: 'desktop' }, + callbacks: { + onData: (data) => terminalModel.write(data), + onSnapshot: (data) => { + snapshots.push(data) + terminalModel.write(data) + }, + onSubscribed: () => { + subscribed += 1 + } + } + }) + await vi.waitFor(() => expect(subscribed).toBe(1), { timeout: 10_000 }) + if (args.exerciseLiveRecovery) { + hostStub.emitOutput(RECOVERED_LIVE_MARKER) + await vi.waitFor( + () => expect(readTerminalText(terminalModel)).toContain(RECOVERED_LIVE_MARKER), + { timeout: 10_000 } + ) + hostStub.emitOutput(CONTINUED_LIVE_MARKER) + await vi.waitFor( + () => expect(readTerminalText(terminalModel)).toContain(CONTINUED_LIVE_MARKER), + { timeout: 10_000 } + ) + } + terminal.close() + const snapshotStartOpcode = Number(args.clientBuild.codec.TerminalStreamOpcode.SnapshotStart) + return { + frames: link.observed.map((frame) => { + const codec = + frame.direction === 'host-to-client' ? args.clientBuild.codec : args.hostBuild.codec + const name = codec.TerminalStreamOpcode[frame.opcode] + return `${frame.direction}:${typeof name === 'string' ? name : frame.opcode}` + }), + missingRuntimeMethods: hostStub.missingRuntimeMethods, + rejected: link.rejected, + rendered: readTerminalText(terminalModel), + snapshotStarts: link.observed + .filter( + (frame) => frame.direction === 'host-to-client' && frame.opcode === snapshotStartOpcode + ) + .map((frame) => frame.json ?? {}), + snapshots + } + } finally { + terminalModel.dispose() + await link.dispose() + } +} + +function readTerminalText(terminal: Terminal): string { + const lines: string[] = [] + for (let index = 0; index < terminal.buffer.active.length; index += 1) { + lines.push(terminal.buffer.active.getLine(index)?.translateToString(true) ?? '') + } + return lines.join('\n') +} + +describe('reported mixed-version lossy initial snapshot', () => { + it( + 'reconstructs the blank 1.4.192-client / 1.4.191-host boundary', + async () => { + const record = await runLossyInitialSnapshotPair({ + clientBuild: reportedClient, + hostBuild: reportedHost + }) + + expect(record.snapshotStarts).toEqual([ + expect.objectContaining({ truncated: true, seq: expect.any(Number) }) + ]) + expect(record.frames).toEqual([ + 'client-to-host:Subscribe', + 'host-to-client:SnapshotStart', + 'host-to-client:SnapshotChunk', + 'host-to-client:SnapshotEnd', + 'client-to-host:Unsubscribe' + ]) + expect(record.snapshots).toEqual([]) + expect(record.rendered.trim()).toBe('') + }, + TIMEOUT_MS + ) + + it( + 'reproduces the blank terminal with the exact pre-fix main client', + async () => { + const record = await runLossyInitialSnapshotPair({ + clientBuild: preFixMain, + hostBuild: reportedHost + }) + + expect(record.snapshotStarts).toEqual([ + expect.objectContaining({ truncated: true, seq: expect.any(Number) }) + ]) + expect(record.frames).toEqual([ + 'client-to-host:Subscribe', + 'host-to-client:SnapshotStart', + 'host-to-client:SnapshotChunk', + 'host-to-client:SnapshotEnd', + 'client-to-host:Unsubscribe' + ]) + expect(record.snapshots).toEqual([]) + expect(record.rendered.trim()).toBe('') + }, + TIMEOUT_MS + ) + + it( + 'paints the old host image exactly once with the candidate client', + async () => { + const record = await runLossyInitialSnapshotPair({ + clientBuild: candidate, + hostBuild: reportedHost, + exerciseLiveRecovery: true + }) + + expect(record.snapshotStarts).toEqual([ + expect.objectContaining({ truncated: true, seq: expect.any(Number) }), + expect.objectContaining({ truncated: false, seq: expect.any(Number) }) + ]) + expect(record.snapshots[0]).toBe(MARKER) + expect(record.rejected).toEqual([]) + expect(record.missingRuntimeMethods).toEqual([]) + expect(record.rendered.split(MARKER)).toHaveLength(2) + expect(record.rendered.split(RECOVERED_LIVE_MARKER)).toHaveLength(2) + expect(record.rendered.split(CONTINUED_LIVE_MARKER)).toHaveLength(2) + }, + TIMEOUT_MS + ) + + it( + 'paints the latest host image exactly once with the candidate client', + async () => { + const record = await runLossyInitialSnapshotPair({ + clientBuild: candidate, + hostBuild: candidate, + exerciseLiveRecovery: true + }) + + expect(record.snapshotStarts).toEqual([ + expect.objectContaining({ truncated: true, seq: expect.any(Number) }), + expect.objectContaining({ truncated: false, seq: expect.any(Number) }) + ]) + expect(record.snapshots[0]).toBe(MARKER) + expect(record.rejected).toEqual([]) + expect(record.missingRuntimeMethods).toEqual([]) + expect(record.rendered.split(MARKER)).toHaveLength(2) + expect(record.rendered.split(RECOVERED_LIVE_MARKER)).toHaveLength(2) + expect(record.rendered.split(CONTINUED_LIVE_MARKER)).toHaveLength(2) + }, + TIMEOUT_MS + ) +}) diff --git a/tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts b/tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts new file mode 100644 index 00000000000..589d23568cb --- /dev/null +++ b/tests/e2e/paired-remote-terminal-lossy-initial-snapshot.spec.ts @@ -0,0 +1,278 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalRead } from '../../src/shared/runtime-types' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient +} from './helpers/paired-electron-client' + +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-lossy-snapshot-')) +const fixturePath = path.join(scratch, 'lossy-snapshot-terminal.mjs') +writeFileSync( + fixturePath, + [ + 'import { readFileSync } from "node:fs"', + 'const marker = readFileSync(process.argv[2], "utf8")', + 'process.stdout.write(`${marker}\\r\\n`)', + 'process.stdin.setEncoding("utf8")', + 'process.stdin.on("data", (data) => process.stdout.write(`LIVE:${data.trim()}\\r\\n`))', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => rmSync(scratch, { recursive: true, force: true })) +test.use({ + orcaAppExtraEnv: { ORCA_E2E_FORCE_REMOTE_TERMINAL_INITIAL_SNAPSHOT_TRUNCATED: '1' } +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(marker: string): string { + const command = [process.execPath, fixturePath, marker] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +async function callRuntime( + page: Page, + environmentId: string, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId, method, params } + ) as Promise +} + +async function callLocalRuntime( + page: Page, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +test('paints a nonempty lossy initial snapshot on a paired Electron client @headful', async ({ + orcaPage +}, testInfo) => { + test.setTimeout(180_000) + const marker = `REMOTE_LOSSY_INITIAL_${Date.now()}` + const liveMarker = `REMOTE_LOSSY_LIVE_${Date.now()}` + const markerPath = path.join(scratch, 'marker-value.txt') + writeFileSync(markerPath, marker) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedElectronClient(offer, testInfo, 'lossy-initial-snapshot') + let terminal: string | null = null + try { + const worktreeId = await orcaPage.evaluate(() => window.__store?.getState().activeWorktreeId) + if (!worktreeId) { + throw new Error('Headed host has no active worktree') + } + await orcaPage.evaluate((id) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(id) + }, worktreeId) + await expect + .poll(() => orcaPage.evaluate(() => window.__store?.getState().activeWorktreeId)) + .toBe(worktreeId) + await expect + .poll( + () => + client.page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) ?? false, + worktreeId + ), + { timeout: 60_000 } + ) + .toBe(true) + const created = await callLocalRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(orcaPage, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(markerPath), + activate: true, + select: true, + navigation: 'host' + }) + terminal = created.tab.terminal + if (!terminal) { + throw new Error('Paired host did not publish the fixture terminal') + } + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await expect + .poll( + async () => { + const result = await callLocalRuntime<{ terminal: RuntimeTerminalRead }>( + orcaPage, + 'terminal.read', + { terminal, screen: true } + ) + return result.terminal.tail.join('\n').includes(marker) + }, + { timeout: 30_000 } + ) + .toBe(true) + const { terminal: hostEvidence } = await callLocalRuntime<{ terminal: RuntimeTerminalRead }>( + orcaPage, + 'terminal.read', + { terminal, screen: true } + ) + console.log( + `[lossy-initial] ${JSON.stringify({ hostLatestCursor: hostEvidence?.latestCursor, hostNextCursor: hostEvidence?.nextCursor, marker })}` + ) + + await expect + .poll( + () => + client.page.evaluate( + ({ tabId, worktreeId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === tabId + ), + { tabId: webTabId, worktreeId } + ), + { timeout: 60_000 } + ) + .toBe(true) + await client.page.evaluate( + ({ tabId, worktreeId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + state?.setActiveTab(tabId) + state?.setActiveTabType('terminal') + }, + { tabId: webTabId, worktreeId } + ) + const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await expect(tab).toHaveAttribute('data-active', 'true') + await expect + .poll( + () => + client.page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return { + mounted: Boolean(pane), + markerCount: + (pane?.serializeAddon?.serialize?.() ?? '').split('REMOTE_LOSSY_INITIAL_').length - + 1 + } + }, webTabId), + { timeout: 30_000 } + ) + .toEqual({ mounted: true, markerCount: 1 }) + + const beforeLiveCursor = Number(hostEvidence?.latestCursor) + const sent = await callRuntime<{ send: { accepted: boolean } }>( + client.page, + client.environmentId, + 'terminal.send', + { + terminal, + text: liveMarker, + enter: true, + client: { id: 'paired-lossy-initial-e2e', type: 'desktop' } + } + ) + expect(sent.send.accepted).toBe(true) + await expect + .poll(() => + client.page.evaluate( + () => + ( + window as Window & { + __remoteTerminalMultiplexAckGate?: { + snapshot: () => { initialSnapshotTruncatedCount?: number } + } + } + ).__remoteTerminalMultiplexAckGate?.snapshot().initialSnapshotTruncatedCount + ) + ) + .toBeGreaterThanOrEqual(1) + await expect + .poll( + () => + client.page.evaluate( + ({ initialMarker, liveMarker, tabId }) => { + const manager = window.__paneManagers?.get(tabId) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + const content = pane?.serializeAddon?.serialize?.() ?? '' + return { + initialMarkerCount: content.split(initialMarker).length - 1, + liveMarkerCount: content.split(`LIVE:${liveMarker}`).length - 1 + } + }, + { initialMarker: marker, liveMarker, tabId: webTabId } + ), + { timeout: 30_000 } + ) + .toEqual({ initialMarkerCount: 1, liveMarkerCount: 1 }) + const clientWindow = await client.app.browserWindow(client.page) + await clientWindow.evaluate((window) => { + window.show() + window.focus() + }) + await expect.poll(() => clientWindow.evaluate((window) => window.isVisible())).toBe(true) + await client.page.evaluate( + () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))) + ) + await testInfo.attach('paired-client-lossy-initial-restored', { + body: await client.page.screenshot({ animations: 'disabled' }), + contentType: 'image/png' + }) + await expect + .poll( + async () => { + const result = await callLocalRuntime<{ terminal: RuntimeTerminalRead }>( + orcaPage, + 'terminal.read', + { terminal, screen: true } + ) + return Number(result.terminal.latestCursor) + }, + { timeout: 30_000 } + ) + .toBeGreaterThan(beforeLiveCursor) + } finally { + if (terminal) { + await callRuntime(client.page, client.environmentId, 'terminal.closeTab', { terminal }).catch( + () => undefined + ) + } + await client.dispose() + } +})