test(e2e): fix two stale E2E specs failing on main (runtime host seeding, alt-screen snapshot) (#10614)

* test(e2e): register a real runtime host and publish the alt-screen frame as its snapshot Two long-running scheduled-E2E failures on main were stale test setup, not product defects. `onboarding.spec.ts:420` seeded the Active Server by faking a runtime environment in the renderer store and writing `activeRuntimeEnvironmentId` through the generic `settings:set` IPC. Since #10011 that setter strips the key, and the dedicated `settings:set-active-runtime-environment-preference` handler resolves the id against the main-process environment store — CI logged `RuntimeEnvironmentStoreError: Unknown environment: env-e2e` from `runtimeEnvironments:subscribe`/`:call` alongside the assertion failure. Register the host for real via `runtimeEnvironments:addFromPairingCode` (offline; no live server) and write the preference through its own channel. `terminal-tab-switch-visual-restore.spec.ts:604` wrote alt-screen frames straight into the renderer's xterm, so those bytes never transited the PTY and main's model could not contain them. On cycle 0 the freshly spawned shell still has queued startup output, so hiding the pane makes main's hidden-delivery gate drop bytes and latch a reveal restore, which repaints main's snapshot over the fabricated frame; later cycles run against an idle shell and survive. Arm the existing `setHiddenSnapshotOverride` seam (already used by sibling tests in this file) with the same frame so the live-write and restore paths render identically, and keep the `markerPresent` assertion. Co-authored-by: Orca <help@stably.ai> * test(e2e): keep the alt-screen restore path observable Numbering the snapshot frame one higher than the live-written frame keeps the marker assertion path-agnostic while leaving the frame number on screen as the signal for which path painted. An unrecognised frame now fails, and the per-cycle path is recorded rather than asserted because which cycles latch a restore is load-dependent. Frame authoring and readback move to a helper module; the additions crossed the spec's max-lines cap. Co-authored-by: Orca <help@stably.ai> * Escape regex metacharacters in alt-screen marker pattern Marker is treated as a literal string, so escape regex metacharacters to prevent them from being interpreted as regex syntax. --------- Co-authored-by: Orca <help@stably.ai>
2026-09-22 00:02:31 +00:00 · 2026-07-25 15:58:40 -07:00
parent eb545aaa59
commit 468f5b77b5
3 changed files with 157 additions and 79 deletions
@@ -0,0 +1,76 @@
+import type { Page } from '@stablyai/playwright-test'
+
+// Boxed alt-screen TUI frame: enters the alternate buffer, clears it, and paints
+// a marker line carrying a zero-padded frame number.
+export function buildAltScreenFrame(marker: string, frame: number): string {
+  const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}`
+  return [
+    '\x1b[?2026h',
+    '\x1b[?1049h',
+    '\x1b[2J\x1b[H',
+    '\x1b[?25l',
+    `╭────────────────────────────────────────────────────────────────────╮`,
+    `│ ${marker} frame ${String(frame).padStart(3, '0')} ${progress}                     │`,
+    `│ Dimension              │ Rating                                      │`,
+    `╰────────────────────────────────────────────────────────────────────╯`,
+    '\x1b[?2026l'
+  ].join('\r\n')
+}
+
+// Why: the live-write and the reveal restore paint the same layout, so the frame
+// number is the only thing on screen that says which of the two landed last.
+export async function readRenderedAltScreenFrame(
+  page: Page,
+  tabId: string,
+  marker: string
+): Promise<number | null> {
+  return page.evaluate(
+    ({ tabId, marker }) => {
+      const manager = window.__paneManagers?.get(tabId)
+      const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0]
+      if (!pane) {
+        throw new Error(`No terminal pane for tab ${tabId}`)
+      }
+      // marker is a literal, so escape it rather than letting `[`/`.`/`+` act as regex syntax.
+      const pattern = new RegExp(`${marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} frame (\\d{3})`)
+      const buffer = pane.terminal.buffer.active
+      for (let row = 0; row < pane.terminal.rows; row += 1) {
+        const line = buffer.getLine(buffer.viewportY + row)?.translateToString(true) ?? ''
+        const match = pattern.exec(line)
+        if (match) {
+          return Number(match[1])
+        }
+      }
+      return null
+    },
+    { tabId, marker }
+  )
+}
+
+export function describeAltScreenRenderPath(
+  renderedFrame: number | null,
+  liveFrame: number,
+  restoreFrame: number
+): string {
+  if (renderedFrame === restoreFrame) {
+    return 'reveal restore'
+  }
+  if (renderedFrame === liveFrame) {
+    return 'live write (no restore)'
+  }
+  return renderedFrame === null ? 'no marker' : `unexpected frame ${renderedFrame}`
+}
+
+export async function writeToPaneTerminal(page: Page, tabId: string, data: string): Promise<void> {
+  await page.evaluate(
+    ({ tabId, data }) => {
+      const manager = window.__paneManagers?.get(tabId)
+      const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0]
+      if (!pane) {
+        throw new Error(`No terminal pane for tab ${tabId}`)
+      }
+      return new Promise<void>((resolve) => pane.terminal.write(data, resolve))
+    },
+    { tabId, data }
+  )
+}
@@ -12,6 +12,7 @@ import { waitForSessionReady } from './helpers/store'
 import type { Page } from '@stablyai/playwright-test'
 import type { GlobalSettings, TuiAgent } from '../../src/shared/types'
 import { ONBOARDING_FINAL_STEP } from '../../src/shared/constants'
+import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../src/shared/pairing'

 type OnboardingState = {
  closedAt: number | null
@@ -421,43 +422,38 @@ test.describe('Onboarding flow', () => {
    await expect(orcaPage.getByRole('heading', { name: /Pick your default agent/i })).toBeVisible({
      timeout: 15_000
    })
-    await orcaPage.evaluate(async () => {
+    // Why: since #10011 `settings:set` strips activeRuntimeEnvironmentId — the
+    // durable Active Server preference is only writable through its dedicated
+    // handler, which resolves the id against the main-process environment
+    // store. So the host has to be registered for real, not faked in the
+    // renderer. Pairing is offline (no live server needed).
+    const pairingCode = encodePairingOffer({
+      v: PAIRING_OFFER_VERSION,
+      scope: 'runtime',
+      endpoint: 'wss://e2e.invalid/ws',
+      deviceToken: 'e2e-device-token',
+      publicKeyB64: 'ZTJlLXB1YmxpYy1rZXk'
+    })
+    const environmentId = await orcaPage.evaluate(async (code) => {
      const store = window.__store
      if (!store) {
        throw new Error('window.__store is not available')
      }
+      const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({
+        name: 'E2E Server',
+        pairingCode: code
+      })
      // Why: after #5071 the server-path add step gates on the registered
      // runtime-environment list (store.runtimeEnvironments), not just the
-      // activeRuntimeEnvironmentId setting. Seed a redacted environment so the
-      // host option exists and the "on host" add UI renders.
-      const now = Date.now()
-      store.getState().setRuntimeEnvironments([
-        {
-          id: 'env-e2e',
-          name: 'E2E Server',
-          createdAt: now,
-          updatedAt: now,
-          lastUsedAt: null,
-          runtimeId: null,
-          source: 'manual',
-          endpoints: [
-            {
-              id: 'ws-env-e2e',
-              kind: 'websocket',
-              label: 'WebSocket',
-              endpoint: 'wss://e2e.invalid/ws'
-            }
-          ],
-          preferredEndpointId: 'ws-env-e2e'
-        }
-      ])
+      // activeRuntimeEnvironmentId setting.
+      store.getState().setRuntimeEnvironments(await window.api.runtimeEnvironments.list())
      // Why: a runtime host is only auto-selectable (health 'available') when it
      // has a live, protocol-compatible status; without one it reads
      // 'disconnected' and the Add Project dialog falls back to Local Mac.
      // runtimeProtocolVersion 3 clears MIN_COMPATIBLE_RUNTIME_SERVER_VERSION.
-      store.getState().setRuntimeEnvironmentStatus('env-e2e', {
+      store.getState().setRuntimeEnvironmentStatus(environment.id, {
        status: {
-          runtimeId: 'env-e2e-runtime',
+          runtimeId: `${environment.id}-runtime`,
          rendererGraphEpoch: 0,
          graphStatus: 'ready',
          authoritativeWindowId: null,
@@ -466,15 +462,23 @@ test.describe('Onboarding flow', () => {
          runtimeProtocolVersion: 3,
          minCompatibleRuntimeClientVersion: 1
        },
-        checkedAt: now
+        checkedAt: Date.now()
      })
-      await store.getState().updateSettings({ activeRuntimeEnvironmentId: 'env-e2e' })
-    })
+      // Why: the store's switchRuntimeEnvironment probes reachability, which a
+      // synthetic host can't satisfy — write the preference directly and push
+      // the returned settings in rather than refetching (fetchSettings would
+      // kick off a status hydrate that clobbers the seeded 'available' health).
+      const settings = await window.api.settings.setActiveRuntimeEnvironmentPreference({
+        environmentId: environment.id
+      })
+      store.setState({ settings })
+      return environment.id
+    }, pairingCode)
    await expect
      .poll(async () => (await getSettings(orcaPage)).activeRuntimeEnvironmentId, {
        timeout: 5_000
      })
-      .toBe('env-e2e')
+      .toBe(environmentId)

    await onboardingFooterButton(orcaPage, SKIP_TO_PROJECT_SETUP_BUTTON).click()

@@ -1,5 +1,11 @@
 import type { Page, TestInfo } from '@stablyai/playwright-test'
 import { test, expect } from './helpers/orca-app'
+import {
+  buildAltScreenFrame,
+  describeAltScreenRenderPath,
+  readRenderedAltScreenFrame,
+  writeToPaneTerminal
+} from './helpers/alt-screen-frame'
 import { runNodeScriptInTerminal } from './helpers/run-node-script-in-terminal'
 import {
  ensureTerminalVisible,
@@ -611,67 +617,50 @@ test.describe('Terminal tab switch visual restore', () => {

    const { firstTabId, secondTabId } = await ensureTwoTerminalTabs(orcaPage)
    await forceWebglOnActiveTab(orcaPage)
+    await waitForPanePtyIdOnTab(orcaPage, firstTabId)

    const runId = `${Date.now()}`
    const finalMarker = `${TAB_SWITCH_MARKER_PREFIX}_${runId}_ALT_24`

-    await orcaPage.evaluate(
-      ({ tabId, finalMarker }) => {
-        const manager = window.__paneManagers?.get(tabId)
-        const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0]
-        if (!pane) {
-          throw new Error(`No terminal pane for tab ${tabId}`)
-        }
-        const frames = Array.from({ length: 25 }, (_, frame) => {
-          const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}`
-          return [
-            '\x1b[?2026h',
-            '\x1b[?1049h',
-            '\x1b[2J\x1b[H',
-            '\x1b[?25l',
-            `╭────────────────────────────────────────────────────────────────────╮`,
-            `│ ${finalMarker} frame ${String(frame).padStart(3, '0')} ${progress}                     │`,
-            `│ Dimension              │ Rating                                      │`,
-            `╰────────────────────────────────────────────────────────────────────╯`,
-            '\x1b[?2026l'
-          ].join('\r\n')
-        }).join('')
-        return new Promise<void>((resolve) => pane.terminal.write(frames, resolve))
-      },
-      { tabId: firstTabId, finalMarker }
+    await writeToPaneTerminal(
+      orcaPage,
+      firstTabId,
+      Array.from({ length: 25 }, (_, frame) => buildAltScreenFrame(finalMarker, frame)).join('')
    )

    const corruptionReports: string[] = []
+    const renderPaths: string[] = []
    for (let cycle = 0; cycle < 6; cycle += 1) {
+      const liveFrame = cycle * 4
+      const restoreFrame = liveFrame + 1
+      const redraw = buildAltScreenFrame(finalMarker, liveFrame)
+      // Why: this frame never transits the PTY, so a reveal restore would
+      // repaint main's model over it. Publish an equivalent frame as the
+      // snapshot so either path leaves a valid screen — numbered one higher so
+      // the readback still reports which one painted. Identity is re-read per
+      // cycle because a reattach would re-key the override.
+      const { ptyId, cols, rows } = await readPaneIdentityOnTab(orcaPage, firstTabId)
+      await setHiddenSnapshotOverride(orcaPage, ptyId, {
+        data: buildAltScreenFrame(finalMarker, restoreFrame),
+        cols,
+        rows
+      })
      await activateTerminalTab(orcaPage, secondTabId)
-      await orcaPage.evaluate(
-        ({ tabId, finalMarker, cycle }) => {
-          const manager = window.__paneManagers?.get(tabId)
-          const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0]
-          if (!pane) {
-            throw new Error(`No terminal pane for tab ${tabId}`)
-          }
-          const frame = cycle * 4
-          const progress = `${'█'.repeat((frame % 8) + 1)}${'░'.repeat(8 - ((frame % 8) + 1))}`
-          const redraw = [
-            '\x1b[?2026h',
-            '\x1b[?1049h',
-            '\x1b[2J\x1b[H',
-            '\x1b[?25l',
-            `╭────────────────────────────────────────────────────────────────────╮`,
-            `│ ${finalMarker} frame ${String(frame).padStart(3, '0')} ${progress}                     │`,
-            `│ Dimension              │ Rating                                      │`,
-            `╰────────────────────────────────────────────────────────────────────╯`,
-            '\x1b[?2026l'
-          ].join('\r\n')
-          return new Promise<void>((resolve) => pane.terminal.write(redraw, resolve))
-        },
-        { tabId: firstTabId, finalMarker, cycle }
-      )
+      await writeToPaneTerminal(orcaPage, firstTabId, redraw)
      await activateTerminalTab(orcaPage, firstTabId)

      const geometry = await readTabTerminalGeometry(orcaPage, firstTabId, `${runId}_ALT`)
-      const issue = geometryLooksCorrupted(geometry)
+      const renderedFrame = await readRenderedAltScreenFrame(orcaPage, firstTabId, finalMarker)
+      renderPaths.push(
+        `cycle ${cycle}: ${describeAltScreenRenderPath(renderedFrame, liveFrame, restoreFrame)}`
+      )
+      // Why: whichever path won must have painted its own frame. Anything else
+      // on screen means the restore replayed stale content.
+      const staleFrame =
+        renderedFrame !== null && renderedFrame !== liveFrame && renderedFrame !== restoreFrame
+          ? `alt-screen shows frame ${renderedFrame}, expected ${liveFrame} (live write) or ${restoreFrame} (reveal restore)`
+          : null
+      const issue = geometryLooksCorrupted(geometry) ?? staleFrame
      if (issue || !geometry.markerPresent) {
        corruptionReports.push(
          `cycle ${cycle}: ${issue ?? 'marker missing after alt-screen redraw'}`
@@ -685,6 +674,15 @@ test.describe('Terminal tab switch visual restore', () => {
      }
    }

+    // Why: which cycles latched a restore is load-dependent, so it is recorded
+    // rather than asserted — without it a "both paths agree" run is opaque.
+    // Logged as well because the list reporter omits annotations.
+    testInfo.annotations.push({
+      type: 'alt-screen-render-path',
+      description: renderPaths.join(', ')
+    })
+    console.log('[tab-switch-repro] alt-screen render path:', renderPaths.join(', '))
+
    expect(
      corruptionReports,
      corruptionReports.length > 0