From 468f5b77b5e2c6c745f124d70fd2f249d3737283 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:58:40 -0700 Subject: [PATCH] test(e2e): fix two stale E2E specs failing on main (runtime host seeding, alt-screen snapshot) (#10614) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(e2e): register a real runtime host and publish the alt-screen frame as its snapshot Two long-running scheduled-E2E failures on main were stale test setup, not product defects. `onboarding.spec.ts:420` seeded the Active Server by faking a runtime environment in the renderer store and writing `activeRuntimeEnvironmentId` through the generic `settings:set` IPC. Since #10011 that setter strips the key, and the dedicated `settings:set-active-runtime-environment-preference` handler resolves the id against the main-process environment store — CI logged `RuntimeEnvironmentStoreError: Unknown environment: env-e2e` from `runtimeEnvironments:subscribe`/`:call` alongside the assertion failure. Register the host for real via `runtimeEnvironments:addFromPairingCode` (offline; no live server) and write the preference through its own channel. `terminal-tab-switch-visual-restore.spec.ts:604` wrote alt-screen frames straight into the renderer's xterm, so those bytes never transited the PTY and main's model could not contain them. On cycle 0 the freshly spawned shell still has queued startup output, so hiding the pane makes main's hidden-delivery gate drop bytes and latch a reveal restore, which repaints main's snapshot over the fabricated frame; later cycles run against an idle shell and survive. Arm the existing `setHiddenSnapshotOverride` seam (already used by sibling tests in this file) with the same frame so the live-write and restore paths render identically, and keep the `markerPresent` assertion. Co-authored-by: Orca * test(e2e): keep the alt-screen restore path observable Numbering the snapshot frame one higher than the live-written frame keeps the marker assertion path-agnostic while leaving the frame number on screen as the signal for which path painted. An unrecognised frame now fails, and the per-cycle path is recorded rather than asserted because which cycles latch a restore is load-dependent. Frame authoring and readback move to a helper module; the additions crossed the spec's max-lines cap. Co-authored-by: Orca * Escape regex metacharacters in alt-screen marker pattern Marker is treated as a literal string, so escape regex metacharacters to prevent them from being interpreted as regex syntax. --------- Co-authored-by: Orca --- tests/e2e/helpers/alt-screen-frame.ts | 76 +++++++++++++++ tests/e2e/onboarding.spec.ts | 64 +++++++------ ...terminal-tab-switch-visual-restore.spec.ts | 96 +++++++++---------- 3 files changed, 157 insertions(+), 79 deletions(-) create mode 100644 tests/e2e/helpers/alt-screen-frame.ts diff --git a/tests/e2e/helpers/alt-screen-frame.ts b/tests/e2e/helpers/alt-screen-frame.ts new file mode 100644 index 00000000000..6292b0c84e1 --- /dev/null +++ b/tests/e2e/helpers/alt-screen-frame.ts @@ -0,0 +1,76 @@ +import type { Page } from '@stablyai/playwright-test' + +// Boxed alt-screen TUI frame: enters the alternate buffer, clears it, and paints +// a marker line carrying a zero-padded frame number. +export function buildAltScreenFrame(marker: string, frame: number): string { + const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}` + return [ + '\x1b[?2026h', + '\x1b[?1049h', + '\x1b[2J\x1b[H', + '\x1b[?25l', + `╭────────────────────────────────────────────────────────────────────╮`, + `│ ${marker} frame ${String(frame).padStart(3, '0')} ${progress} │`, + `│ Dimension │ Rating │`, + `╰────────────────────────────────────────────────────────────────────╯`, + '\x1b[?2026l' + ].join('\r\n') +} + +// Why: the live-write and the reveal restore paint the same layout, so the frame +// number is the only thing on screen that says which of the two landed last. +export async function readRenderedAltScreenFrame( + page: Page, + tabId: string, + marker: string +): Promise { + return page.evaluate( + ({ tabId, marker }) => { + const manager = window.__paneManagers?.get(tabId) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] + if (!pane) { + throw new Error(`No terminal pane for tab ${tabId}`) + } + // marker is a literal, so escape it rather than letting `[`/`.`/`+` act as regex syntax. + const pattern = new RegExp(`${marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} frame (\\d{3})`) + const buffer = pane.terminal.buffer.active + for (let row = 0; row < pane.terminal.rows; row += 1) { + const line = buffer.getLine(buffer.viewportY + row)?.translateToString(true) ?? '' + const match = pattern.exec(line) + if (match) { + return Number(match[1]) + } + } + return null + }, + { tabId, marker } + ) +} + +export function describeAltScreenRenderPath( + renderedFrame: number | null, + liveFrame: number, + restoreFrame: number +): string { + if (renderedFrame === restoreFrame) { + return 'reveal restore' + } + if (renderedFrame === liveFrame) { + return 'live write (no restore)' + } + return renderedFrame === null ? 'no marker' : `unexpected frame ${renderedFrame}` +} + +export async function writeToPaneTerminal(page: Page, tabId: string, data: string): Promise { + await page.evaluate( + ({ tabId, data }) => { + const manager = window.__paneManagers?.get(tabId) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] + if (!pane) { + throw new Error(`No terminal pane for tab ${tabId}`) + } + return new Promise((resolve) => pane.terminal.write(data, resolve)) + }, + { tabId, data } + ) +} diff --git a/tests/e2e/onboarding.spec.ts b/tests/e2e/onboarding.spec.ts index 86ed4b9cec4..0fb5ad870c1 100644 --- a/tests/e2e/onboarding.spec.ts +++ b/tests/e2e/onboarding.spec.ts @@ -12,6 +12,7 @@ import { waitForSessionReady } from './helpers/store' import type { Page } from '@stablyai/playwright-test' import type { GlobalSettings, TuiAgent } from '../../src/shared/types' import { ONBOARDING_FINAL_STEP } from '../../src/shared/constants' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../src/shared/pairing' type OnboardingState = { closedAt: number | null @@ -421,43 +422,38 @@ test.describe('Onboarding flow', () => { await expect(orcaPage.getByRole('heading', { name: /Pick your default agent/i })).toBeVisible({ timeout: 15_000 }) - await orcaPage.evaluate(async () => { + // Why: since #10011 `settings:set` strips activeRuntimeEnvironmentId — the + // durable Active Server preference is only writable through its dedicated + // handler, which resolves the id against the main-process environment + // store. So the host has to be registered for real, not faked in the + // renderer. Pairing is offline (no live server needed). + const pairingCode = encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + scope: 'runtime', + endpoint: 'wss://e2e.invalid/ws', + deviceToken: 'e2e-device-token', + publicKeyB64: 'ZTJlLXB1YmxpYy1rZXk' + }) + const environmentId = await orcaPage.evaluate(async (code) => { const store = window.__store if (!store) { throw new Error('window.__store is not available') } + const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({ + name: 'E2E Server', + pairingCode: code + }) // Why: after #5071 the server-path add step gates on the registered // runtime-environment list (store.runtimeEnvironments), not just the - // activeRuntimeEnvironmentId setting. Seed a redacted environment so the - // host option exists and the "on host" add UI renders. - const now = Date.now() - store.getState().setRuntimeEnvironments([ - { - id: 'env-e2e', - name: 'E2E Server', - createdAt: now, - updatedAt: now, - lastUsedAt: null, - runtimeId: null, - source: 'manual', - endpoints: [ - { - id: 'ws-env-e2e', - kind: 'websocket', - label: 'WebSocket', - endpoint: 'wss://e2e.invalid/ws' - } - ], - preferredEndpointId: 'ws-env-e2e' - } - ]) + // activeRuntimeEnvironmentId setting. + store.getState().setRuntimeEnvironments(await window.api.runtimeEnvironments.list()) // Why: a runtime host is only auto-selectable (health 'available') when it // has a live, protocol-compatible status; without one it reads // 'disconnected' and the Add Project dialog falls back to Local Mac. // runtimeProtocolVersion 3 clears MIN_COMPATIBLE_RUNTIME_SERVER_VERSION. - store.getState().setRuntimeEnvironmentStatus('env-e2e', { + store.getState().setRuntimeEnvironmentStatus(environment.id, { status: { - runtimeId: 'env-e2e-runtime', + runtimeId: `${environment.id}-runtime`, rendererGraphEpoch: 0, graphStatus: 'ready', authoritativeWindowId: null, @@ -466,15 +462,23 @@ test.describe('Onboarding flow', () => { runtimeProtocolVersion: 3, minCompatibleRuntimeClientVersion: 1 }, - checkedAt: now + checkedAt: Date.now() }) - await store.getState().updateSettings({ activeRuntimeEnvironmentId: 'env-e2e' }) - }) + // Why: the store's switchRuntimeEnvironment probes reachability, which a + // synthetic host can't satisfy — write the preference directly and push + // the returned settings in rather than refetching (fetchSettings would + // kick off a status hydrate that clobbers the seeded 'available' health). + const settings = await window.api.settings.setActiveRuntimeEnvironmentPreference({ + environmentId: environment.id + }) + store.setState({ settings }) + return environment.id + }, pairingCode) await expect .poll(async () => (await getSettings(orcaPage)).activeRuntimeEnvironmentId, { timeout: 5_000 }) - .toBe('env-e2e') + .toBe(environmentId) await onboardingFooterButton(orcaPage, SKIP_TO_PROJECT_SETUP_BUTTON).click() diff --git a/tests/e2e/terminal-tab-switch-visual-restore.spec.ts b/tests/e2e/terminal-tab-switch-visual-restore.spec.ts index 36258bd3b96..4f0d2a5df11 100644 --- a/tests/e2e/terminal-tab-switch-visual-restore.spec.ts +++ b/tests/e2e/terminal-tab-switch-visual-restore.spec.ts @@ -1,5 +1,11 @@ import type { Page, TestInfo } from '@stablyai/playwright-test' import { test, expect } from './helpers/orca-app' +import { + buildAltScreenFrame, + describeAltScreenRenderPath, + readRenderedAltScreenFrame, + writeToPaneTerminal +} from './helpers/alt-screen-frame' import { runNodeScriptInTerminal } from './helpers/run-node-script-in-terminal' import { ensureTerminalVisible, @@ -611,67 +617,50 @@ test.describe('Terminal tab switch visual restore', () => { const { firstTabId, secondTabId } = await ensureTwoTerminalTabs(orcaPage) await forceWebglOnActiveTab(orcaPage) + await waitForPanePtyIdOnTab(orcaPage, firstTabId) const runId = `${Date.now()}` const finalMarker = `${TAB_SWITCH_MARKER_PREFIX}_${runId}_ALT_24` - await orcaPage.evaluate( - ({ tabId, finalMarker }) => { - const manager = window.__paneManagers?.get(tabId) - const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] - if (!pane) { - throw new Error(`No terminal pane for tab ${tabId}`) - } - const frames = Array.from({ length: 25 }, (_, frame) => { - const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}` - return [ - '\x1b[?2026h', - '\x1b[?1049h', - '\x1b[2J\x1b[H', - '\x1b[?25l', - `╭────────────────────────────────────────────────────────────────────╮`, - `│ ${finalMarker} frame ${String(frame).padStart(3, '0')} ${progress} │`, - `│ Dimension │ Rating │`, - `╰────────────────────────────────────────────────────────────────────╯`, - '\x1b[?2026l' - ].join('\r\n') - }).join('') - return new Promise((resolve) => pane.terminal.write(frames, resolve)) - }, - { tabId: firstTabId, finalMarker } + await writeToPaneTerminal( + orcaPage, + firstTabId, + Array.from({ length: 25 }, (_, frame) => buildAltScreenFrame(finalMarker, frame)).join('') ) const corruptionReports: string[] = [] + const renderPaths: string[] = [] for (let cycle = 0; cycle < 6; cycle += 1) { + const liveFrame = cycle * 4 + const restoreFrame = liveFrame + 1 + const redraw = buildAltScreenFrame(finalMarker, liveFrame) + // Why: this frame never transits the PTY, so a reveal restore would + // repaint main's model over it. Publish an equivalent frame as the + // snapshot so either path leaves a valid screen — numbered one higher so + // the readback still reports which one painted. Identity is re-read per + // cycle because a reattach would re-key the override. + const { ptyId, cols, rows } = await readPaneIdentityOnTab(orcaPage, firstTabId) + await setHiddenSnapshotOverride(orcaPage, ptyId, { + data: buildAltScreenFrame(finalMarker, restoreFrame), + cols, + rows + }) await activateTerminalTab(orcaPage, secondTabId) - await orcaPage.evaluate( - ({ tabId, finalMarker, cycle }) => { - const manager = window.__paneManagers?.get(tabId) - const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] - if (!pane) { - throw new Error(`No terminal pane for tab ${tabId}`) - } - const frame = cycle * 4 - const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}` - const redraw = [ - '\x1b[?2026h', - '\x1b[?1049h', - '\x1b[2J\x1b[H', - '\x1b[?25l', - `╭────────────────────────────────────────────────────────────────────╮`, - `│ ${finalMarker} frame ${String(frame).padStart(3, '0')} ${progress} │`, - `│ Dimension │ Rating │`, - `╰────────────────────────────────────────────────────────────────────╯`, - '\x1b[?2026l' - ].join('\r\n') - return new Promise((resolve) => pane.terminal.write(redraw, resolve)) - }, - { tabId: firstTabId, finalMarker, cycle } - ) + await writeToPaneTerminal(orcaPage, firstTabId, redraw) await activateTerminalTab(orcaPage, firstTabId) const geometry = await readTabTerminalGeometry(orcaPage, firstTabId, `${runId}_ALT`) - const issue = geometryLooksCorrupted(geometry) + const renderedFrame = await readRenderedAltScreenFrame(orcaPage, firstTabId, finalMarker) + renderPaths.push( + `cycle ${cycle}: ${describeAltScreenRenderPath(renderedFrame, liveFrame, restoreFrame)}` + ) + // Why: whichever path won must have painted its own frame. Anything else + // on screen means the restore replayed stale content. + const staleFrame = + renderedFrame !== null && renderedFrame !== liveFrame && renderedFrame !== restoreFrame + ? `alt-screen shows frame ${renderedFrame}, expected ${liveFrame} (live write) or ${restoreFrame} (reveal restore)` + : null + const issue = geometryLooksCorrupted(geometry) ?? staleFrame if (issue || !geometry.markerPresent) { corruptionReports.push( `cycle ${cycle}: ${issue ?? 'marker missing after alt-screen redraw'}` @@ -685,6 +674,15 @@ test.describe('Terminal tab switch visual restore', () => { } } + // Why: which cycles latched a restore is load-dependent, so it is recorded + // rather than asserted — without it a "both paths agree" run is opaque. + // Logged as well because the list reporter omits annotations. + testInfo.annotations.push({ + type: 'alt-screen-render-path', + description: renderPaths.join(', ') + }) + console.log('[tab-switch-repro] alt-screen render path:', renderPaths.join(', ')) + expect( corruptionReports, corruptionReports.length > 0