From 4ff96df2b527b74e2d1001d4ef7c5073743ce099 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:05:29 -0700 Subject: [PATCH] Auto e2e tests autofix scheduled ci 1h run 32 20260902T0700 (#18227) * Fix flaky e2e tests with improved locators and synchronization Add explicit waits, use more robust element selectors, and simplify test setup to reduce race conditions. Replace file-based fixtures with programmatic browser creation, use parent-scoped locators for menu interactions, and poll for stable state before assertions. * Add E2E failure triage report for run 33564563164 - Reconciles 14 failed tests against job logs and trace artifacts - Categorizes failures: 8 product bugs, 2 flaky tests, 4 test updates - Documents test-maintenance fixes and diagnostic findings - Files 8 Linear issues with owners and fresh recurrence evidence - Provides next actions for product owners and repository maintenance * rm artifact notes * Refactor browser creation E2E test to use UI interactions - Click through menu instead of manipulating internal store state - Use Playwright's locator and toBeVisible() assertion patterns * Record E2E browser creation pageId before barrier check Move createdPageId assignment before the barrier arm/fire checks. This ensures the pageId is recorded unconditionally when tracking is enabled, allowing tests to distinguish between creations rejected before the host attempt vs those that failed after creation. * Remove browser page reclamation assertion from restart test Simplifies test by removing page ID tracking and poll checking if pages persist after paired runtime restart. --- .../web-runtime-browser-creation-e2e-fault.ts | 7 +- .../issue-12656-terminal-link-tooltip.spec.ts | 9 ++- ...er-creation-reconciliation-failure.spec.ts | 78 +++++++++---------- .../source-control-large-file-count.spec.ts | 4 +- tests/e2e/tabs.spec.ts | 4 +- ...tree-active-delete-scroll-position.spec.ts | 22 +----- 6 files changed, 58 insertions(+), 66 deletions(-) diff --git a/src/renderer/src/runtime/web-runtime-browser-creation-e2e-fault.ts b/src/renderer/src/runtime/web-runtime-browser-creation-e2e-fault.ts index db26cfa3d67..44a422a5d6a 100644 --- a/src/renderer/src/runtime/web-runtime-browser-creation-e2e-fault.ts +++ b/src/renderer/src/runtime/web-runtime-browser-creation-e2e-fault.ts @@ -179,10 +179,15 @@ export function throwIfE2eWebRuntimeBrowserCapabilityUnavailable(): void { } export async function pauseAfterE2eWebRuntimeBrowserCreate(remotePageId: string): Promise { - if (!e2eConfig.exposeStore || !armed || !createdPageBarrier) { + if (!e2eConfig.exposeStore) { return } + // Recorded before the arm check so a journey that never arms the barrier can still prove no host + // page was created — a null id is only evidence if a real create would have set one. createdPageId = remotePageId + if (!armed || !createdPageBarrier) { + return + } await createdPageBarrier } diff --git a/tests/e2e/issue-12656-terminal-link-tooltip.spec.ts b/tests/e2e/issue-12656-terminal-link-tooltip.spec.ts index 06fc725ee36..879c8305266 100644 --- a/tests/e2e/issue-12656-terminal-link-tooltip.spec.ts +++ b/tests/e2e/issue-12656-terminal-link-tooltip.spec.ts @@ -147,8 +147,13 @@ test.describe('Issue #12656 terminal link tooltip', () => { expect(Math.abs(idle.paneBottom - idle.terminalBottom)).toBeLessThanOrEqual(1) await expect .poll(async () => { - await moveToLink(orcaPage, probe) - return readTooltipState(orcaPage, probe.tabId) + const currentProbe = await locateUrl(orcaPage, url) + if (!currentProbe) { + return { display: 'none', text: '' } + } + probe = currentProbe + await moveToLink(orcaPage, currentProbe) + return readTooltipState(orcaPage, currentProbe.tabId) }) .toMatchObject({ display: '', text: expect.stringContaining(url) }) diff --git a/tests/e2e/paired-browser-creation-reconciliation-failure.spec.ts b/tests/e2e/paired-browser-creation-reconciliation-failure.spec.ts index 27bbfff03fb..be280fcff13 100644 --- a/tests/e2e/paired-browser-creation-reconciliation-failure.spec.ts +++ b/tests/e2e/paired-browser-creation-reconciliation-failure.spec.ts @@ -1,10 +1,7 @@ -import { writeFileSync } from 'node:fs' -import path from 'node:path' import type { Page, TestInfo } from '@stablyai/playwright-test' import { RuntimeClient } from '../../src/cli/runtime/client' import { expect, test } from './helpers/orca-app' import { readHostBrowserPageIds, readHostTabs } from './helpers/host-session-tabs' -import { openFileExplorer } from './helpers/file-explorer' import { launchHeadlessPairedRuntimeHost, type HeadlessPairedRuntimeHost @@ -17,8 +14,6 @@ import { } from './helpers/paired-electron-client' import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' -const FIXTURE_NAME = 'paired-browser-reconcile-failure.html' - type FaultSnapshot = { armed: boolean capabilityRejectionArmed: boolean @@ -36,6 +31,22 @@ type FaultWindow = Window & { } } +// Drives the real create menu so the failure surfaces through handleNewBrowserTab's toast. +async function startBrowserCreate(page: Page): Promise { + await page.evaluate(() => window.__store?.getState().setBrowserDefaultUrl('about:blank')) + await page.getByRole('button', { name: 'New tab' }).first().click() + const newBrowserTab = page.getByRole('menuitem', { name: /New Browser Tab/i }) + await expect(newBrowserTab).toBeVisible({ timeout: 30_000 }) + await newBrowserTab.click() +} + +async function readStableHostTabs(hostClient: RuntimeClient, repoPath: string) { + const { publicationEpoch, snapshotVersion, ...state } = await readHostTabs(hostClient, repoPath) + expect(publicationEpoch).not.toBe('') + expect(snapshotVersion).toBeGreaterThan(0) + return state +} + type ClientTabState = { browserTabIds: string[] browserWorkspaceIds: string[] @@ -115,14 +126,7 @@ async function runReconciliationFailureJourney(args: { }) .toMatchObject({ terminalTabIds: expect.arrayContaining([expect.any(String)]) }) - await openFileExplorer(page) - const fixtureRow = page.locator('[data-file-explorer-row]').filter({ hasText: FIXTURE_NAME }) - await expect(fixtureRow).toBeVisible({ timeout: 30_000 }) - await fixtureRow.click() - const openPreviewToSide = page.getByRole('button', { name: 'Open Preview to the Side' }) - await expect(openPreviewToSide).toBeVisible({ timeout: 30_000 }) const baselineClient = await readClientTabs(page, worktreeId) - expect(baselineClient.editorTabIds).not.toHaveLength(0) expect(baselineClient.terminalTabIds).not.toHaveLength(0) const baselineHostBrowserIds = await readHostBrowserPageIds(args.hostClient, args.repoPath) @@ -133,7 +137,7 @@ async function runReconciliationFailureJourney(args: { } fault.arm() }) - await openPreviewToSide.click() + await startBrowserCreate(page) const faultSnapshot = await expect .poll( @@ -155,9 +159,8 @@ async function runReconciliationFailureJourney(args: { } expect(await readHostBrowserPageIds(args.hostClient, args.repoPath)).toContain(createdPageId) - // Why: the tab is staged on click, so while the create is held the user already sees it — - // exactly one of it, in the new split. The rollback assertions after release are what prove - // the optimism is unwound rather than stranded. + // The managed-browser action stages one tab in the active group while the host create is held. + // The rollback assertions prove that optimism is unwound rather than stranded. const heldClient = await readClientTabs(page, worktreeId) const addedSince = (baseline: string[], held: string[]): string[] => { expect(held).toEqual(expect.arrayContaining(baseline)) @@ -169,7 +172,7 @@ async function runReconciliationFailureJourney(args: { ).toHaveLength(1) expect(heldClient.editorTabIds).toEqual(baselineClient.editorTabIds) expect(heldClient.terminalTabIds).toEqual(baselineClient.terminalTabIds) - expect(addedSince(baselineClient.groupIds, heldClient.groupIds)).toHaveLength(1) + expect(heldClient.groupIds).toEqual(baselineClient.groupIds) await page.screenshot({ path: args.testInfo.outputPath(`${args.topology}-browser-reconciliation-held.png`), @@ -181,9 +184,9 @@ async function runReconciliationFailureJourney(args: { ) ).toBe(true) - await expect(page.getByText('Unable to open this file in Orca Browser.')).toBeVisible({ - timeout: 30_000 - }) + await expect( + page.getByText('The paired runtime could not create a managed browser tab.') + ).toBeVisible({ timeout: 30_000 }) await expect .poll(() => readHostBrowserPageIds(args.hostClient, args.repoPath), { timeout: 30_000, @@ -261,14 +264,8 @@ async function runCapabilityFailureJourney(args: { }) .toMatchObject({ terminalTabIds: expect.arrayContaining([expect.any(String)]) }) - await openFileExplorer(page) - const fixtureRow = page.locator('[data-file-explorer-row]').filter({ hasText: FIXTURE_NAME }) - await expect(fixtureRow).toBeVisible({ timeout: 30_000 }) - await fixtureRow.click() - const openPreviewToSide = page.getByRole('button', { name: 'Open Preview to the Side' }) - await expect(openPreviewToSide).toBeVisible({ timeout: 30_000 }) const baselineClient = await readClientTabs(page, worktreeId) - const baselineHost = await readHostTabs(args.hostClient, args.repoPath) + const baselineHost = await readStableHostTabs(args.hostClient, args.repoPath) await page.evaluate(() => { const fault = (window as FaultWindow).__webRuntimeBrowserCreationFault @@ -277,18 +274,25 @@ async function runCapabilityFailureJourney(args: { } fault.armCapabilityRejection() }) - await openPreviewToSide.click() + await startBrowserCreate(page) - await expect(page.getByText('Unable to open this file in Orca Browser.')).toBeVisible({ + await expect(page.getByText(/E2E forced browser capability rejection/)).toBeVisible({ timeout: 30_000 }) + // Why: baseline equality alone also holds for a create that was rolled back. A null page id is + // what separates rejecting before the host create from undoing one afterwards. + expect( + await page.evaluate( + () => (window as FaultWindow).__webRuntimeBrowserCreationFault?.snapshot() ?? null + ) + ).toMatchObject({ createdPageId: null }) await expect .poll(() => readClientTabs(page, worktreeId), { timeout: 30_000, message: 'client split state did not settle after capability rejection' }) .toEqual(baselineClient) - expect(await readHostTabs(args.hostClient, args.repoPath)).toEqual(baselineHost) + expect(await readStableHostTabs(args.hostClient, args.repoPath)).toEqual(baselineHost) await page.screenshot({ path: args.testInfo.outputPath(`${args.topology}-browser-capability-rejected.png`), fullPage: true @@ -305,10 +309,6 @@ test('rolls back a headed-host browser when client reconciliation times out @hea testRepoPath }, testInfo) => { test.setTimeout(300_000) - writeFileSync( - path.join(testRepoPath, FIXTURE_NAME), - '

browser reconciliation fault

\n' - ) await waitForSessionReady(orcaPage) await waitForActiveWorktree(orcaPage) await ensureTerminalVisible(orcaPage) @@ -326,16 +326,12 @@ test('rolls back a headed-host browser when client reconciliation times out @hea }) }) -test('cleans up a headed-host preview when capability rejects after preflight @headful', async ({ +test('cleans up a headed-host browser when capability rejects before create @headful', async ({ electronApp, orcaPage, testRepoPath }, testInfo) => { test.setTimeout(300_000) - writeFileSync( - path.join(testRepoPath, FIXTURE_NAME), - '

browser capability fault

\n' - ) await waitForSessionReady(orcaPage) await waitForActiveWorktree(orcaPage) await ensureTerminalVisible(orcaPage) @@ -355,10 +351,6 @@ test('cleans up a headed-host preview when capability rejects after preflight @h test('keeps browser failure cleanup on a headless host', async ({ testRepoPath }, testInfo) => { test.setTimeout(300_000) - writeFileSync( - path.join(testRepoPath, FIXTURE_NAME), - '

headless capability fault

\n' - ) const host: HeadlessPairedRuntimeHost = await launchHeadlessPairedRuntimeHost() try { await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) diff --git a/tests/e2e/source-control-large-file-count.spec.ts b/tests/e2e/source-control-large-file-count.spec.ts index fb98eb9a123..28a5e7758f2 100644 --- a/tests/e2e/source-control-large-file-count.spec.ts +++ b/tests/e2e/source-control-large-file-count.spec.ts @@ -438,7 +438,9 @@ test.describe('Source Control large file count (#8013)', () => { // explicit recovery path after the underlying change count drops. removeLargeFileCountUntrackedTree(fixture.repoPath) await expect(tooManyChangesBanner).toBeVisible() - await orcaPage.getByRole('button', { name: 'Retry' }).click() + const retryButton = tooManyChangesBanner.locator('..').getByRole('button', { name: 'Retry' }) + await expect(retryButton).toBeVisible() + await retryButton.click() await expect(tooManyChangesBanner).not.toBeVisible() await expect .poll(() => diff --git a/tests/e2e/tabs.spec.ts b/tests/e2e/tabs.spec.ts index b89e47d4079..2faabafc1cc 100644 --- a/tests/e2e/tabs.spec.ts +++ b/tests/e2e/tabs.spec.ts @@ -29,7 +29,8 @@ import { getActiveTabType, getWorktreeTabs, getTabBarOrder, - ensureTerminalVisible + ensureTerminalVisible, + waitForStartupWorktreeRefresh } from './helpers/store' const SORTABLE_TAB = '[data-testid="sortable-tab"]' @@ -69,6 +70,7 @@ async function getFocusedTerminalTabId(page: Page): Promise { test.describe('Tabs', () => { test.beforeEach(async ({ orcaPage }) => { await waitForSessionReady(orcaPage) + await waitForStartupWorktreeRefresh(orcaPage) await waitForActiveWorktree(orcaPage) await ensureTerminalVisible(orcaPage) }) diff --git a/tests/e2e/worktree-active-delete-scroll-position.spec.ts b/tests/e2e/worktree-active-delete-scroll-position.spec.ts index 1b9d4bc3283..5e2f15f5b75 100644 --- a/tests/e2e/worktree-active-delete-scroll-position.spec.ts +++ b/tests/e2e/worktree-active-delete-scroll-position.spec.ts @@ -237,24 +237,10 @@ test('deleting the active scrolled worktree preserves position and closes the ro `[data-worktree-sidebar] [data-worktree-id=${JSON.stringify(belowId)}]` ) await pauseForVisualProof(orcaPage) - await target.evaluate((element) => { - const scope = element.querySelector( - '[data-worktree-context-menu-scope="worktree"]' - ) - if (!scope) { - throw new Error('Worktree context-menu scope is unavailable') - } - scope.dispatchEvent( - new MouseEvent('contextmenu', { - bubbles: true, - button: 2, - cancelable: true, - clientX: scope.getBoundingClientRect().left + 10, - clientY: scope.getBoundingClientRect().top + 10 - }) - ) - }) - const deleteItem = orcaPage.getByRole('menuitem', { name: 'Delete', exact: true }) + const contextMenuScope = target.locator('[data-worktree-context-menu-scope="worktree"]') + await expect(contextMenuScope).toBeVisible() + await contextMenuScope.click({ button: 'right' }) + const deleteItem = orcaPage.getByRole('menuitem', { name: /^Delete(?:\s|$)/ }) await expect(deleteItem).toBeVisible() await expect(deleteItem).toBeInViewport() await pauseForVisualProof(orcaPage)