From 0b99371f612a1842d636fff18d393bfb1728b2ed Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:16:54 +0200 Subject: [PATCH] ci: fail when opening, searching or switching folders renders more than budgeted --- .github/workflows/perf-guard.yml | 44 +++++++ package.json | 1 + scripts/perf-guard/budget.json | 22 ++++ scripts/perf-guard/perf-guard.spec.ts | 152 ++++++++++++++++++++++++ scripts/perf-guard/playwright.config.ts | 53 +++++++++ 5 files changed, 272 insertions(+) create mode 100644 .github/workflows/perf-guard.yml create mode 100644 scripts/perf-guard/budget.json create mode 100644 scripts/perf-guard/perf-guard.spec.ts create mode 100644 scripts/perf-guard/playwright.config.ts diff --git a/.github/workflows/perf-guard.yml b/.github/workflows/perf-guard.yml new file mode 100644 index 000000000..35ecb88ee --- /dev/null +++ b/.github/workflows/perf-guard.yml @@ -0,0 +1,44 @@ +name: "CI: Perf guard" + +# Render passes, component renders and JMAP requests per open / search / +# folder switch, against a production build with the dev mock's +# benchmark-sized corpus (scripts/perf-guard). Counts, not timings: they do +# not drift with runner load, so a failure means the code does more work. + +on: + push: + pull_request: + +jobs: + perf-guard: + name: Perf guard + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browser + run: npx playwright install --with-deps chromium + + # The same bundler the Docker image uses, with the dev mock compiled in. + - name: Build + env: + DEV_MOCK_JMAP: "true" + NEXT_TELEMETRY_DISABLED: "1" + GIT_COMMIT: ${{ github.sha }} + run: npx next build --webpack + + - name: Run perf guard + env: + CI: "true" + run: npm run test:perf-guard diff --git a/package.json b/package.json index 3475aa8fd..671de889f 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "lite:serve": "node scripts/lite/serve.mjs out 4173", "test:lite-smoke": "playwright test -c scripts/lite/smoke/playwright.config.ts", "test:lite-smoke:stalwart": "playwright test -c scripts/lite/smoke/playwright.stalwart.config.ts", + "test:perf-guard": "playwright test -c scripts/perf-guard/playwright.config.ts", "start": "next start", "lint": "eslint .", "lint:fix": "eslint . --fix", diff --git a/scripts/perf-guard/budget.json b/scripts/perf-guard/budget.json new file mode 100644 index 000000000..c64b631df --- /dev/null +++ b/scripts/perf-guard/budget.json @@ -0,0 +1,22 @@ +{ + "open": { + "commits": 8, + "renders": 428, + "requests": 1 + }, + "search": { + "commits": 9, + "renders": 410, + "requests": 2 + }, + "folder (first visit)": { + "commits": 8, + "renders": 488, + "requests": 2 + }, + "folder (revisit)": { + "commits": 3, + "renders": 275, + "requests": 1 + } +} diff --git a/scripts/perf-guard/perf-guard.spec.ts b/scripts/perf-guard/perf-guard.spec.ts new file mode 100644 index 000000000..fcb6b9bdd --- /dev/null +++ b/scripts/perf-guard/perf-guard.spec.ts @@ -0,0 +1,152 @@ +import { test, expect, type Page } from "@playwright/test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +/** + * Counts what each hot-path action costs: React commits (render passes), + * component renders, and JMAP requests by method. See playwright.config.ts. + * + * Each action runs a few times and the budget is the worst run, so one + * unlucky interleaving does not decide the result, but a change that adds a + * render pass or a request to every open does. + */ + +type Cost = { commits: number; renders: number; requests: string[] }; +type Budget = Record; + +const UPDATE = process.env.PERF_GUARD_UPDATE === "1"; +// Renders vary a little with timing (a response landing before or after a +// deferred write); commits and requests may not grow at all. +const RENDER_SLACK = 1.1; + +async function instrument(page: Page, requests: string[]) { + await page.addInitScript(() => { + // Never mark read: an Email/set per open is not the cost being measured. + try { + const raw = localStorage.getItem("settings-storage"); + const parsed = raw ? JSON.parse(raw) : { state: {}, version: 1 }; + parsed.state = { ...(parsed.state ?? {}), markAsReadDelay: -1 }; + localStorage.setItem("settings-storage", JSON.stringify(parsed)); + } catch { /* checked after login */ } + const stats = { commits: 0, renders: 0 }; + (window as unknown as { __perfGuard: typeof stats }).__perfGuard = stats; + // Function, class, forwardRef, memo and simple-memo components. + const COUNTED = new Set([0, 1, 11, 14, 15]); + type Fiber = { tag: number; flags: number; alternate: Fiber | null; child: Fiber | null; sibling: Fiber | null }; + const walk = (fiber: Fiber | null) => { + for (let f = fiber; f; f = f.sibling) { + const prev = f.alternate; + // PerformedWork (1): the component's render function ran this commit. + if (COUNTED.has(f.tag) && (prev === null || (f.flags & 1) === 1)) stats.renders++; + // A subtree whose child list was not rebuilt bailed out entirely. + if (f.child && (prev === null || f.child !== prev.child)) walk(f.child); + } + }; + const renderers = new Map(); + (window as unknown as { __REACT_DEVTOOLS_GLOBAL_HOOK__: unknown }).__REACT_DEVTOOLS_GLOBAL_HOOK__ = { + supportsFiber: true, + renderers, + isDisabled: false, + inject(renderer: unknown) { const id = renderers.size + 1; renderers.set(id, renderer); return id; }, + onCommitFiberRoot(_id: number, root: { current: Fiber }) { stats.commits++; try { walk(root.current.child); } catch { /* keep counting */ } }, + onCommitFiberUnmount() {}, + onPostCommitFiberRoot() {}, + onScheduleFiberRoot() {}, + checkDCE() {}, + }; + }); + page.on("request", (req) => { + if (req.method() !== "POST" || !req.url().includes("/api/dev-jmap")) return; + try { + const calls = (JSON.parse(req.postData() || "{}").methodCalls || []) as [string][]; + if (calls.length) requests.push(calls.map((c) => c[0]).join("+")); + } catch { /* not a JMAP call */ } + }); +} + +const LIST = '[data-tour="email-list"]'; +const VIEWER = '[data-tour="email-viewer"]'; + +test("hot paths stay within their render and request budget", async ({ page }, testInfo) => { + // testDir is this directory (playwright.config.ts); works as CJS and ESM. + const BUDGET_FILE = join(testInfo.project.testDir, "budget.json"); + const requests: string[] = []; + await instrument(page, requests); + await page.goto("/"); + await page.getByRole("button", { name: /^sign in$/i }).first().click(); + await page.waitForFunction((sel) => /#\d+/.test(document.querySelector(sel)?.textContent ?? ""), LIST, { timeout: 60_000 }); + expect(await page.evaluate(() => JSON.parse(localStorage.getItem("settings-storage") || "{}")?.state?.markAsReadDelay)).toBe(-1); + await page.waitForTimeout(2000); + + const settle = () => page.waitForTimeout(700); + const measure = async (action: () => Promise): Promise => { + await settle(); + requests.length = 0; + const before = await page.evaluate(() => ({ ...(window as unknown as { __perfGuard: { commits: number; renders: number } }).__perfGuard })); + await action(); + await settle(); + const after = await page.evaluate(() => ({ ...(window as unknown as { __perfGuard: { commits: number; renders: number } }).__perfGuard })); + return { commits: after.commits - before.commits, renders: after.renders - before.renders, requests: requests.slice() }; + }; + + const open = async (index: number) => { + const row = page.locator(`${LIST} [data-index="${index}"]`).first(); + const id = (await row.innerText()).match(/#\d+/)?.[0]; + await row.click(); + await page.waitForFunction(({ sel, id }) => { + const v = document.querySelector(sel); + const frame = v?.querySelector("iframe") as HTMLIFrameElement | null; + const text = (v?.textContent ?? "") + (frame?.contentDocument?.body?.textContent ?? ""); + return !!id && text.includes(id) && /Synthetic seed message|Lorem/.test(text); + }, { sel: VIEWER, id }, { timeout: 20_000 }); + }; + const search = async (query: string) => { + const input = page.locator('input[type="search"]').first(); + await input.fill(query); + await input.press("Enter"); + await page.locator('button[aria-label="Clear search"]').first().waitFor(); + await page.waitForFunction((sel) => !document.querySelector(`${sel} .animate-spin`), LIST); + }; + const clearSearch = async () => { + await page.locator('button[aria-label="Clear search"]').first().click(); + await page.waitForFunction((sel) => !document.querySelector('button[aria-label="Clear search"]') && !document.querySelector(`${sel} .animate-spin`), LIST); + }; + const folder = async (name: string) => { + await page.getByRole("button", { name, exact: false }).first().click(); + await page.waitForFunction((sel) => !document.querySelector(`${sel} .animate-spin`) && /#\d+/.test(document.querySelector(sel)?.textContent ?? ""), LIST); + }; + + const runs: Record = { open: [], search: [], "folder (first visit)": [], "folder (revisit)": [] }; + for (const i of [1, 2, 3]) runs.open.push(await measure(() => open(i))); + for (const q of ["incident", "review"]) { + runs.search.push(await measure(() => search(q))); + await clearSearch(); + } + runs["folder (first visit)"].push(await measure(() => folder("Sent"))); + runs["folder (first visit)"].push(await measure(() => folder("Drafts"))); + runs["folder (revisit)"].push(await measure(() => folder("Inbox"))); + runs["folder (revisit)"].push(await measure(() => folder("Sent"))); + + const worst: Budget = {}; + for (const [action, costs] of Object.entries(runs)) { + worst[action] = { + commits: Math.max(...costs.map((c) => c.commits)), + renders: Math.max(...costs.map((c) => c.renders)), + requests: Math.max(...costs.map((c) => c.requests.length)), + }; + console.log(`${action}: ${JSON.stringify(worst[action])} requests=${JSON.stringify(costs.map((c) => c.requests))}`); + } + + if (UPDATE) { + writeFileSync(BUDGET_FILE, JSON.stringify(worst, null, 2) + "\n"); + return; + } + const budget = JSON.parse(readFileSync(BUDGET_FILE, "utf8")) as Budget; + for (const [action, cost] of Object.entries(worst)) { + const allowed = budget[action]; + expect(allowed, `no budget for "${action}"; run with PERF_GUARD_UPDATE=1`).toBeTruthy(); + expect(cost.commits, `${action}: render passes (commits)`).toBeLessThanOrEqual(allowed.commits); + expect(cost.requests, `${action}: JMAP requests`).toBeLessThanOrEqual(allowed.requests); + expect(cost.renders, `${action}: component renders`).toBeLessThanOrEqual(Math.ceil(allowed.renders * RENDER_SLACK)); + } +}); diff --git a/scripts/perf-guard/playwright.config.ts b/scripts/perf-guard/playwright.config.ts new file mode 100644 index 000000000..7e480e5aa --- /dev/null +++ b/scripts/perf-guard/playwright.config.ts @@ -0,0 +1,53 @@ +import { defineConfig } from "@playwright/test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Render and request budget for the mail view's hot paths (open a message, +// search, switch folder). Runs against a production build made with +// DEV_MOCK_JMAP=true (`DEV_MOCK_JMAP=true npx next build --webpack`), served +// with the dev mock's benchmark-sized corpus. It counts React commits, +// component renders and JMAP requests per action, which are stable where +// timings are not, and fails when one goes over budget.json. +// +// DEV_MOCK_JMAP=true npx next build --webpack +// npm run test:perf-guard # check +// PERF_GUARD_UPDATE=1 npm run test:perf-guard # rewrite budget.json +const port = Number(process.env.PERF_GUARD_PORT || 3150); +// A server that is already running (e.g. a container built with +// DEV_MOCK_JMAP=true and started with DEV_MOCK_JMAP_BULK=1000): nothing is +// started then. +const externalBaseUrl = (process.env.PERF_GUARD_BASE_URL ?? "").replace(/\/+$/, ""); +const adminDir = mkdtempSync(join(tmpdir(), "perf-guard-")); + +export default defineConfig({ + testDir: ".", + testMatch: /perf-guard\.spec\.ts$/, + timeout: 180_000, + // A count over budget is a result, not a flake. + retries: 0, + workers: 1, + reporter: process.env.CI ? [["list"]] : "list", + use: { + baseURL: externalBaseUrl || `http://localhost:${port}`, + locale: "en-US", + viewport: { width: 1440, height: 900 }, + }, + projects: [{ name: "chromium", use: { browserName: "chromium" } }], + webServer: externalBaseUrl ? undefined : { + command: `npx next start -p ${port}`, + port, + reuseExistingServer: !process.env.CI, + cwd: process.cwd(), + timeout: 120_000, + env: { + DEV_MOCK_JMAP: "true", + DEV_MOCK_JMAP_BULK: "1000", + JMAP_SERVER_URL: `http://localhost:${port}/api/dev-jmap`, + SESSION_SECRET: "perf-guard-session-secret-0123456789abcdef", + ADMIN_CONFIG_DIR: join(adminDir, "config"), + ADMIN_STATE_DIR: join(adminDir, "state"), + NEXT_TELEMETRY_DISABLED: "1", + }, + }, +});