diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0325745a..6a9f6401d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: admin: ${{ steps.filter.outputs.admin }} site: ${{ steps.filter.outputs.site }} forms: ${{ steps.filter.outputs.forms }} + qa: ${{ steps.filter.outputs.qa }} make: ${{ steps.filter.outputs.make }} ios: ${{ steps.filter.outputs.ios }} installer: ${{ steps.filter.outputs.installer }} @@ -80,6 +81,8 @@ jobs: - 'web/src/components/app/forms/designCore.ts' - 'web/src/components/app/forms/form-theme.css' - 'scripts/check-forms-mirror.sh' + qa: + - 'qa/**' make: - 'integrations/make/**' ios: @@ -186,6 +189,29 @@ jobs: - name: Build run: pnpm build + # The proof harness records against a live local stack, so CI only checks that it compiles and its flows load. + qa-ci: + name: QA harness CI + needs: changes + if: needs.changes.outputs.qa == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: qa + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-pnpm + with: + working-directory: qa + node-version: "24" + + - name: Typecheck + run: pnpm typecheck + + - name: Flows load + run: pnpm exec playwright test --list + admin-ci: name: Admin CI needs: changes @@ -490,7 +516,7 @@ jobs: ci-status: name: CI Status runs-on: ubuntu-latest - needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] + needs: [changes, migrations-ci, go-ci, web-ci, qa-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] if: always() steps: - name: Check CI status diff --git a/AGENTS.md b/AGENTS.md index 75fb8ffbd..74cff93c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ CI is strict. `go build ./...` succeeding is not enough — `golangci-lint` runs Other CI-touching rules: -- the frontend trees (`admin/`, `web/`, `site/`) each have their own CI jobs; run `pnpm typecheck` in any tree you touched and `pnpm lint` when the rules are non-trivial +- the frontend trees (`admin/`, `web/`, `site/`, `qa/`) each have their own CI jobs; run `pnpm typecheck` in any tree you touched and `pnpm lint` when the rules are non-trivial - never push without first re-running the relevant `*build*` / `*typecheck*` / `*lint*` step on the affected tree - a `make lint` (or `gofmt -l`) failure is always a real CI failure; do not push hoping it will pass @@ -173,10 +173,24 @@ is `make cli-check` (and `make cli-sha` after any edit). Do not: - do not run `go build ./...`, `pnpm build`, or docker image builds as a "did it work" check. They are slow and are not what CI gates on. `go run` (via the make dev targets) already compiles; `make fmt` + `make lint` + `pnpm typecheck` are the real signals. -- do not write or run Python/Playwright (or any browser-automation) scripts to test the app. Manual, in-browser verification is the user's job against the native dev stack (`make infra` + `make backend` + `make web`). Do not add screenshot/e2e test harnesses to this repo. +- do not write Python or ad-hoc browser scripts, and do not drive the app step by step from screenshots or accessibility snapshots. Browser automation lives in `qa/` and only records proof for a PR (below); it is not a test gate, and manual verification against the native dev stack stays the user's job. - do not run the Go test suite as a default gate unless the task is specifically about those tests. - do not push hoping CI passes; a `gofmt -l` / `make lint` / `pnpm typecheck` failure is always a real CI failure. +### Visual proof on pull requests + +Record proof only for a UI change a reviewer should actually see; most changes need none. `qa/` is the harness and `qa/README.md` is its playbook; read it before the first recording. + +- record for: a new page, dialog, drawer or multi-step flow; a redesigned or re-laid-out screen; a changed interaction (new controls, new states, a different path through a task); a visible bug fix where the before was visibly broken; or when the user asks for it +- skip for: anything without a visible change (backend, API, migrations, workers, tests, CI, docs, refactors, dependency bumps) and small visual tweaks (copy, a label, an icon, spacing or colour nudges), which the PR text describes instead. When unsure, skip +- a follow-up commit is re-recorded only when it changes what the proof shows + +- write a scripted flow in `qa/flows/` from the code you changed, run `pnpm proof ` in `qa/`, and read only pass or the failing step. Never screenshot your way around the page; `pnpm aria ` prints the accessibility tree when you need a selector +- record against this worktree's own stack (`pnpm stack up`, in the smallest of `lite`, `full`, `sandbox` the flow needs), never another session's and never anything but localhost: the repository is public and so is everything attached to its PRs +- publish with `pnpm share` once the PR exists. The first publish goes into the PR description; every later one is a comment showing before/after of only the stills that changed. Record follow-ups with `pnpm proof:fresh` so the comparison starts from fresh seed data. Media reaches GitHub only through `gh --attach`, never an image host, a commit or another repository +- keep shot names stable across commits, because follow-up comments diff stills by name +- the machine is shared: record from the main agent, never from fanned-out subagents (a machine-wide lock runs one recording at a time, so they would only queue while each holds a context), and `pnpm stack down` once the proof is published + ## Security And Compliance Invariants Warmbly's Google OAuth client is assessed against **ADA CASA v2.1.1 at Assurance Level 1**, which maps to OWASP ASVS 4.0.3. The evidence pack is a claim about the code on `main`: a change that breaks one of the invariants below does not just introduce a bug, it makes a submitted statement untrue and puts the OAuth client's verification at risk. Treat them as constraints on every change, not as a checklist run before an audit. @@ -361,6 +375,7 @@ API keys with the `REALTIME_SUBSCRIBE` permission (bit 11) can connect to the sa - `realtime/`: websocket fanout service - `web/`: in-product frontend (dashboard). Customer-facing only: it holds no platform-admin screens, and operator tooling must not be added back here - `admin/`: platform admin panel (:5174), the single operator surface. Workers, users, orgs, warmup, campaigns, analytics, audit. Every route sits behind `RequireAdmin` and the backend's `RequireAdminPermission` gates +- `qa/`: the proof harness (scripted Playwright flows recording 1080p walkthroughs and stills for pull requests, published with `gh --attach`). Its own package; see `qa/README.md` - `site/`: public marketing site (Astro 5 + Tailwind v4). `site/public/install.sh` is the self-host installer served at warmbly.com/install.sh and `site/public/cli.sh` is the CLI installer served at warmbly.com/cli.sh (with `cli.ps1` for Windows), each with its checksum next to it; see the rules above before touching either - `deploy/`: production deploy manifests, infrastructure, and runtime config. `deploy/split-cloud/` is the three-provider shape (control plane on a container host, bus + cache + fleet on machines you own, database + root key + object store in a cloud region), documented at `docs/content/docs/development/split-deployment.mdx` - `docs/`: documentation site (docs.warmbly.com); product guides, API reference, and self-hosting/engineering docs under `content/docs/development/` diff --git a/qa/.gitignore b/qa/.gitignore new file mode 100644 index 000000000..4a9aebd68 --- /dev/null +++ b/qa/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +# Recordings, stills, saved sessions and the per-worktree stack state. Never committed. +.artifacts/ diff --git a/qa/README.md b/qa/README.md new file mode 100644 index 000000000..0bf9a1cd5 --- /dev/null +++ b/qa/README.md @@ -0,0 +1,243 @@ +# Proof harness + +Recorded proof for pull requests. A flow is a short scripted Playwright walk +through the dashboard. Running it produces a 1920x1080 H.264 walkthrough and +named full-resolution stills, and `pnpm share` puts them on the PR: + +- **first publish**: a Proof section appended to the PR description, stills + inline, walkthrough videos last +- **every publish after that**: a comment with before/after pairs for only the + stills that changed since the previous publish, the commits in between, and + the videos of the flows that changed. Nothing changed means nothing is posted + +It is not a test gate and CI does not run it (CI only typechecks it and checks +the flows load). It exists so a reviewer can see a change without checking it +out. + +## When to record + +Only for a UI change worth watching. Most PRs need no proof. + +- **record**: a new page, dialog, drawer or multi-step flow; a redesigned or + re-laid-out screen; a changed interaction (new controls, new states, a + different path through a task); a visible bug fix where the before was + visibly broken; or when the user asks for it +- **skip**: anything without a visible change (backend, API, migrations, + workers, tests, CI, docs, refactors, dependency bumps) and small visual + tweaks (copy, a label, an icon, spacing or colour nudges). Describe those in + the PR text. When unsure, skip +- **follow-ups**: re-record only when the commit changes what the proof shows + +## Why scripted + +The model never looks at the page while the browser runs. You write the flow +from the code you just changed (you know the routes, labels and fields), run +it, and read one line: pass, or the failing step. A 15-step flow costs a few +thousand tokens to write and nothing to re-run. Driving a browser step by step +from screenshots or snapshots costs that much on every step of every run, and +leaves dead air in the video while the model thinks. + +## Be frugal with the machine + +Several agents share this machine, and every stack and browser holds real +memory. The harness enforces the expensive parts; the rest is on you. + +- **one recording at a time, machine-wide.** `pnpm proof` (and `pnpm aria`) + take a lock in the system temp directory; a second one waits for the first + instead of starting its own Chromium. Never fan recording out to subagents: + they would only queue behind each other while each holds a context +- **one stack per worktree**, in the smallest mode the flow needs. `lite` is + enough for anything that only reads or edits records +- **stop it when you are done**: `pnpm stack down` after `pnpm share`. A stack + nobody records against stops itself after 60 minutes (`QA_STACK_IDLE_MIN`), + but do not lean on that +- the browser and the encoder never run at once: frames are spooled to disk + during the run and encoded after the browser exits, one flow at a time, with + bounded encoder threads + +Measured on this repo's dashboard: + +| | memory | +| --- | --- | +| `lite` stack (backend, dashboard, Redis, NATS, Mailpit) | ~300 MB idle, ~500 MB once used | +| `full` stack (+ consumer, worker, realtime) | ~560 MB idle | +| `sandbox` stack (+ tracking, Dovecot, simulator) | ~760 MB idle | +| Chromium during a recording (proportional set size) | ~800 MB | +| ffmpeg encoding, after Chromium has exited | ~580 MB | + +## One-time setup + +```bash +make infra # repo root: the shared postgres (each stack gets its own database in it) +cd qa && pnpm bootstrap # deps, Playwright's Chromium, the small stack images, then `pnpm check` +``` + +`pnpm check` verifies node 22.18+, ffmpeg with libx264, Chromium (and that it +launches), gh 2.99+ and signed in, docker, the shared postgres, the images each +mode needs, and this worktree's stack. Each failure prints its fix. + +## The loop + +```bash +pnpm stack up # this worktree's own stack (lite unless told otherwise) +pnpm proof contacts # run the flows whose file or title matches +git push && gh pr create ... # the PR has to exist first +pnpm share # publish the last run to this branch's PR +pnpm stack down # free the memory +``` + +After a follow-up commit: push, record the flows your change touched again, +and `pnpm share`. It posts a comment, never a second description section. +Record follow-ups with `pnpm proof:fresh `, which re-seeds the stack +(about 15 seconds) before recording, so the comparison shows what the commit +changed and not what background jobs did to the data in the meantime. + +`pnpm share --dry-run` prints the body and the gh command without uploading +anything, and works before the PR exists. `--no-video` posts stills only, +`--force` comments even when no still changed, `--replace-body` rewrites the +description's Proof section instead of commenting (for a re-record nobody has +reviewed yet), `--allow-failed` publishes a run with failing flows. + +`share` refuses a run with failing flows and a run recorded at a different +commit than HEAD, and warns when the tree was dirty or HEAD is not pushed. + +## The stack + +`pnpm stack up [lite|full|sandbox]` runs this worktree's Warmbly natively +against the shared postgres, with everything else private to it: its own +database (`warmbly_qa_`), Redis, NATS and Mailpit, on ports picked +once per worktree and remembered in `.artifacts/stack/ports`. Nothing crosses +between stacks: no cached profile, no worker event, no login code. The Go +services run from binaries built once per `up` (`go build`, cached), not from +one `go run` toolchain per service. Linux and WSL. + +| Mode | Runs | Seed and account | +| --- | --- | --- | +| `lite` | backend, dashboard | rich seed (`SEED_RICH` + `SEED_FULL`), `dev@warmbly.com` | +| `full` | lite + consumer, worker, realtime | same seed; sends, syncs and live updates work | +| `sandbox` | full + tracking, Dovecot, simulator | Sunrise Labs (`sandbox@warmbly.test`): live mailboxes, campaign mail, opens, clicks, replies | + +`lite` and `full` share a seed and switch freely. The sandbox is a different +organization, so moving to or from it needs `pnpm stack reset-data `; +`up` refuses rather than mix them. In the sandbox the simulator keeps changing +data while you record, so its stills differ from run to run by design. + +Other commands: `status` (what runs, ports, memory), `logs [name]`, `restart` +(rebuilds the binaries; the dashboard hot-reloads on its own), `reset-data +[mode]` (drop the database, fresh Redis, reseed), `ls` (every QA stack on the +machine), `down`. + +Realtime and tracking run from the local `ghcr.io/warmbly/warmbly/*:prod` +images. `up` warns when one was built before the last commit to its source, +with the command that rebuilds it. + +Sign-in happens once per stack through the API and the login code in the +stack's Mailpit; the first-run wizard is completed through its own endpoint, +and the session is saved under `.artifacts/auth`. A flow that lands on the +sign-in page, onboarding or the workspace picker fails and says why. + +Recording against a stack the harness did not start is possible but explicit: +`QA_WEB_URL` and `QA_API_URL` (and `QA_MAILPIT_URL`, `QA_EMAIL`, +`QA_PASSWORD`). Without either, `pnpm proof` refuses rather than record +whatever happens to answer on the default ports, which is usually another +session's stack. + +## Writing a flow + +Flows live in `flows/.flow.ts`, one `test` per walkthrough. Extend the +area's file when your change lands in it; add one when it does not exist. + +```ts +import { expect, test } from "../lib/proof.ts"; + +test("search contacts and open a contact's details", async ({ page, proof }) => { + await page.goto("/app/contacts"); + await expect(page.getByRole("table")).toBeVisible(); + await proof.chapter("Contacts", "Search the list, then open one contact"); + + await page.getByRole("textbox", { name: /Search by name/ }).pressSequentially("beth", { delay: 60 }); + await expect(page.getByRole("row", { name: /Beth Chen/ })).toBeVisible(); + await proof.shot("search-results", { caption: "Search narrowed to one contact" }); +}); +``` + +- `proof.chapter(title, description?)` shows a title card in the video +- `proof.shot(name, { caption?, target?, ignore? })` saves a still. The name is + kebab-case and is the key follow-up comments diff by, so keep it stable + across commits. `target` shoots one element. `ignore` takes locators whose + content changes on its own (relative times, live counters): the still is + published untouched, and those regions are skipped when it is compared with + the previous publish +- `proof.dwell(ms?)` holds a result on screen long enough to read +- `test.use({ seed: "sandbox" })` for a flow written against the Sunrise Labs + data. A flow is skipped, with the command that fixes it, when the stack holds + the other seed +- `test.use({ signedIn: false })` for a flow that starts on the sign-in pages + +Conventions: + +- `expect(...)` the content before every `chapter` and `shot`, so neither lands + on a loading state +- locate by role and accessible name (`getByRole`, `getByLabel`, `getByText`); + no CSS classes +- `pressSequentially(text, { delay: 60 })` where typing should be visible, + `fill` where it should not +- no `waitForTimeout` except through `dwell` +- keep a walkthrough under about 30 seconds; split longer stories into tests +- the title reads as a sentence about what the reviewer sees; it heads the PR + section + +When you need a selector you cannot read off the code, print the page's +accessibility tree with the saved session instead of opening a browser: + +```bash +pnpm aria /app/contacts # whole page +pnpm aria /app/contacts main # one region, fewer tokens +``` + +Do not read the video or screenshot your way around the page. Read a still +only when the change is a visual judgment (spacing, colour, layout) that the +assertions cannot make. + +## When a flow fails + +The list reporter prints the failing step and its locator. Failures keep a +Playwright trace (DOM snapshots, network, console) and the reporter prints its +path; open it with `pnpm exec playwright show-trace ` when the error +line is not enough. A failed flow's frames are dropped instead of encoded +(`QA_VIDEO_ON_FAIL=1` keeps them). + +## Recording settings + +1920x1080 at 30 fps, H.264 CRF 18 (`slow` preset), yuv420p so every browser's +player shows it. A clip over 10 MB is re-encoded smaller. Frames come from +Playwright's screencast at full size and are encoded here, because Playwright's +built-in `video` option is VP8 at 1 Mbit/s and smears text. The video starts at +the first painted frame, not on the blank page before the app renders. The +animated cursor, click ripples and action labels are Playwright's +`screencast.showActions`. Trace screenshots stay off: they would start a +smaller screencast first, and the recorder refuses frames below 1080p. + +| Variable | Default | | +| --- | --- | --- | +| `QA_FPS` | `30` | frame rate | +| `QA_CRF` | `18` | quality, lower is better and bigger | +| `QA_PRESET` | `slow` | x264 preset | +| `QA_ENCODE_THREADS` | `4` | encoder threads, which also bound its memory | +| `QA_SLOWMO` | `250` | ms between actions, so a viewer can follow | +| `QA_MAX_VIDEO_MB` | `10` | re-encode above this | +| `QA_VIDEO_ON_FAIL` | | `1` encodes failed flows too | +| `QA_STACK_IDLE_MIN` | `60` | stop an unused stack after this long; `0` never | +| `QA_LOCK_WAIT_MIN` | `30` | how long a recording waits for another to finish | +| `QA_WEB_URL`, `QA_API_URL`, `QA_MAILPIT_URL` | this worktree's stack | record a stack the harness did not start | +| `QA_EMAIL`, `QA_PASSWORD` | the mode's seeded account | account to sign in as | +| `QA_ALLOW_HOST` | | extra hostnames allowed besides localhost | + +## Safety + +The repository is public and so is everything attached to its pull requests. +The harness refuses to record anything but localhost (or a host named in +`QA_ALLOW_HOST`), so recordings only ever show seed data. Never point it at +production or a customer workspace, and never attach media to a PR any other +way: no image hosts, no commits of screenshots, no other repositories. +Everything the harness writes stays under the ignored `.artifacts/`. diff --git a/qa/flows/contacts.flow.ts b/qa/flows/contacts.flow.ts new file mode 100644 index 000000000..c6b637061 --- /dev/null +++ b/qa/flows/contacts.flow.ts @@ -0,0 +1,19 @@ +import { expect, test } from "../lib/proof.ts"; + +test("search contacts and open a contact's details", async ({ page, proof }) => { + await page.goto("/app/contacts"); + await expect(page.getByRole("table")).toBeVisible(); + await proof.chapter("Contacts", "Search the list, then open one contact"); + + await page.getByRole("textbox", { name: /Search by name/ }).pressSequentially("beth", { delay: 60 }); + const row = page.getByRole("row", { name: /Beth Chen/ }); + await expect(row).toBeVisible(); + await expect(page.getByRole("row", { name: /Carlos Diaz/ })).toBeHidden(); + await proof.shot("search-results", { caption: "Search narrowed to one contact" }); + + await row.click(); + await expect(page.getByText("beth.chen@initech.test").last()).toBeVisible(); + await proof.dwell(); + // "Checked ... 4m ago" drifts between runs; ignored so it never reads as a change. + await proof.shot("contact-details", { caption: "Contact details drawer", ignore: [page.getByText(/\d+[smhd] ago/)] }); +}); diff --git a/qa/flows/dashboard.flow.ts b/qa/flows/dashboard.flow.ts new file mode 100644 index 000000000..85af15d1e --- /dev/null +++ b/qa/flows/dashboard.flow.ts @@ -0,0 +1,8 @@ +import { expect, test } from "../lib/proof.ts"; + +test("dashboard opens on the mailboxes page", async ({ page, proof }) => { + await page.goto("/app/emails"); + await expect(page.getByRole("row", { name: /dev\.outbound@warmbly\.test/ })).toBeVisible(); + await proof.chapter("Mailboxes", "The signed-in dashboard landing page"); + await proof.shot("mailboxes", { caption: "Mailboxes list" }); +}); diff --git a/qa/flows/inbox.flow.ts b/qa/flows/inbox.flow.ts new file mode 100644 index 000000000..20140a76e --- /dev/null +++ b/qa/flows/inbox.flow.ts @@ -0,0 +1,18 @@ +import { expect, test } from "../lib/proof.ts"; + +test.use({ seed: "sandbox" }); + +test("open a reply in the unified inbox", async ({ page, proof }) => { + await page.goto("/app/unibox"); + const thread = page.getByRole("button", { name: /^Hana Jules .*Wayne Enterprises/ }); + await expect(thread).toBeVisible(); + // Relative times ("9h", "1d") move on their own; ignored so a re-run tomorrow is not a change. + const times = page.getByText(/^\d+[mhdw]$/); + await proof.chapter("Unified inbox", "Replies from every mailbox in one list"); + await proof.shot("inbox-list", { caption: "Inbox with replies across mailboxes", ignore: [times] }); + + await thread.click(); + await expect(page.getByText("Thursday at 2pm works for a demo").last()).toBeVisible(); + await proof.dwell(); + await proof.shot("conversation", { caption: "The reply opened in the conversation pane", ignore: [times] }); +}); diff --git a/qa/lib/auth.ts b/qa/lib/auth.ts new file mode 100644 index 000000000..c27c809c7 --- /dev/null +++ b/qa/lib/auth.ts @@ -0,0 +1,134 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { authFile, env } from "./env.ts"; + +type Tokens = { + access_token: string; + access_token_expires_at: string; + refresh_token: string; + refresh_token_expires_at: string; +}; + +type StorageState = { + cookies: []; + origins: { origin: string; localStorage: { name: string; value: string }[] }[]; +}; + +const TOKEN_KEYS = ["access_token", "access_token_expires_at", "refresh_token", "refresh_token_expires_at"] as const; + +// Reuses the saved session while it still works: every fresh login sends a code email, and those are budgeted. +export async function ensureSession(): Promise { + const file = authFile(); + const saved = readTokens(file); + if (saved && (await sessionWorks(saved))) { + await finishOnboarding(saved.access_token); + return file; + } + + const tokens = await login(); + await finishOnboarding(tokens.access_token); + mkdirSync(dirname(file), { recursive: true }); + const state: StorageState = { + cookies: [], + origins: [ + { + origin: env.webURL, + // The dashboard reads `auth_token` (one JSON object) and keeps the four flat keys alongside it. + localStorage: [ + { name: "auth_token", value: JSON.stringify(Object.fromEntries(TOKEN_KEYS.map((k) => [k, tokens[k]]))) }, + ...TOKEN_KEYS.map((k) => ({ name: k, value: tokens[k] })), + ], + }, + ], + }; + writeFileSync(file, JSON.stringify(state, null, 2)); + return file; +} + +function readTokens(file: string): Tokens | undefined { + if (!existsSync(file)) return undefined; + const state = JSON.parse(readFileSync(file, "utf8")) as StorageState; + const items = state.origins.find((o) => o.origin === env.webURL)?.localStorage ?? []; + const get = (k: string) => items.find((i) => i.name === k)?.value ?? ""; + const tokens = Object.fromEntries(TOKEN_KEYS.map((k) => [k, get(k)])) as Tokens; + if (!tokens.refresh_token || Date.parse(tokens.refresh_token_expires_at) < Date.now() + 3600_000) return undefined; + return tokens; +} + +async function sessionWorks(tokens: Tokens): Promise { + if (Date.parse(tokens.access_token_expires_at) > Date.now() + 60_000) { + const res = await fetch(`${env.apiURL}/v1/auth/me`, { headers: { Authorization: `Bearer ${tokens.access_token}` } }); + return res.ok; + } + // An expired access token is fine: the dashboard refreshes it on its first request. + return true; +} + +async function call(method: string, path: string, body?: object, token?: string): Promise { + const res = await fetch(`${env.apiURL}/v1${path}`, { + method, + headers: { "Content-Type": "application/json", ...(token ? { Authorization: `Bearer ${token}` } : {}) }, + body: body ? JSON.stringify(body) : undefined, + }); + const json = (await res.json().catch(() => ({}))) as T & { error?: string; code?: string }; + if (!res.ok) throw new Error(`${method} ${path} answered ${res.status}: ${json.code ?? ""} ${json.error ?? ""}`.trim()); + return json; +} + +function post(path: string, body: object): Promise { + return call("POST", path, body); +} + +// Seeded accounts have not been through the first-run wizard, and the dashboard sends them there +// before anything else. Answer it once through the same endpoint the wizard uses. +async function finishOnboarding(token: string): Promise { + if (!token) return; + const me = await call<{ first_name?: string; last_name?: string; onboarding_completed_at?: string | null }>("GET", "/auth/me", undefined, token).catch(() => undefined); + if (!me || me.onboarding_completed_at) return; + await call("PATCH", "/auth/me/onboarding", { + first_name: me.first_name || "Dev", + last_name: me.last_name || "User", + referral_source: "other", + }, token); +} + +async function login(): Promise { + const startedAt = Date.now(); + const start = await post<{ session?: string; code_required: boolean; token?: Tokens; two_fa_required?: boolean }>( + "/auth/login", + { email: env.email, password: env.password, turnstile: env.turnstile }, + ); + if (start.token) return start.token; + if (start.two_fa_required) throw new Error(`${env.email} has 2FA enabled; use an account without it for QA`); + if (!start.session) throw new Error("login answered with neither a token nor a code session"); + + const code = await loginCode(startedAt); + const confirm = await post & { two_fa_required?: boolean }>("/auth/login/confirm", { + session: start.session, + code, + turnstile: env.turnstile, + }); + if (confirm.two_fa_required) throw new Error(`${env.email} has 2FA enabled; use an account without it for QA`); + if (!confirm.access_token) throw new Error("login confirm returned no token"); + return confirm as Tokens; +} + +type MailpitSummary = { ID: string; Created: string }; + +async function loginCode(since: number): Promise { + const query = encodeURIComponent(`to:"${env.email}" subject:"Your Login Code"`); + for (let i = 0; i < 40; i++) { + const res = await fetch(`${env.mailpitURL}/api/v1/search?query=${query}&limit=5`); + if (res.ok) { + const { messages } = (await res.json()) as { messages: MailpitSummary[] }; + const fresh = messages.find((m) => Date.parse(m.Created) >= since - 2000); + if (fresh) { + const msg = (await (await fetch(`${env.mailpitURL}/api/v1/message/${fresh.ID}`)).json()) as { Text: string }; + const code = msg.Text.match(/\b(\d{6})\b/)?.[1]; + if (code) return code; + } + } + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`no login code for ${env.email} reached Mailpit at ${env.mailpitURL}`); +} diff --git a/qa/lib/env.ts b/qa/lib/env.ts new file mode 100644 index 000000000..d4333f3a1 --- /dev/null +++ b/qa/lib/env.ts @@ -0,0 +1,64 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const QA_DIR = join(dirname(fileURLToPath(import.meta.url)), ".."); +export const ARTIFACTS = join(QA_DIR, ".artifacts"); +export const RUN_DIR = join(ARTIFACTS, "run"); +export const AUTH_DIR = join(ARTIFACTS, "auth"); +export const STACK_DIR = join(ARTIFACTS, "stack"); + +export const VIDEO = { + width: 1920, + height: 1080, + fps: Number(process.env.QA_FPS ?? 30), + crf: Number(process.env.QA_CRF ?? 18), + preset: process.env.QA_PRESET ?? "slow", +}; + +type StackEnv = { webURL?: string; apiURL?: string; mailpitURL?: string; email?: string; password?: string; mode?: string }; + +function stackEnv(): StackEnv { + const file = join(STACK_DIR, "env.json"); + if (!existsSync(file)) return {}; + return JSON.parse(readFileSync(file, "utf8")) as StackEnv; +} + +const stack = stackEnv(); + +export const env = { + webURL: trimSlash(process.env.QA_WEB_URL ?? stack.webURL ?? "http://localhost:5173"), + apiURL: trimSlash(process.env.QA_API_URL ?? stack.apiURL ?? "http://localhost:8080"), + mailpitURL: trimSlash(process.env.QA_MAILPIT_URL ?? stack.mailpitURL ?? "http://localhost:18025"), + email: process.env.QA_EMAIL ?? stack.email ?? "dev@warmbly.com", + password: process.env.QA_PASSWORD ?? stack.password ?? "password123", + // "external": QA_WEB_URL names a stack the harness did not start. "none": nothing to record. + mode: process.env.QA_WEB_URL ? "external" : (stack.mode ?? "none"), + turnstile: "warmbly-local-turnstile-bypass", +}; + +function trimSlash(url: string): string { + return url.replace(/\/+$/, ""); +} + +// The repo is public and recordings are uploaded to it, so only a local stack with seed data may be recorded. +export function assertLocal(url: string): void { + const host = new URL(url).hostname; + const allowed = (process.env.QA_ALLOW_HOST ?? "").split(",").filter(Boolean); + const local = ["localhost", "127.0.0.1", "[::1]", "::1"].includes(host) || host.endsWith(".localhost"); + if (!local && !allowed.includes(host)) { + throw new Error( + `refusing to record ${url}: recordings are published to a public repo, so only a local dev stack is allowed ` + + `(set QA_ALLOW_HOST=${host} if this host is a dev machine running seed data)`, + ); + } +} + +// The stack's idle watchdog stops it after a stretch with no recording; every use resets the clock. +export function touchStack(): void { + if (existsSync(STACK_DIR)) writeFileSync(join(STACK_DIR, "last-used"), new Date().toISOString()); +} + +export function authFile(): string { + return join(AUTH_DIR, `${new URL(env.webURL).host.replace(/[^a-z0-9]+/gi, "_")}.json`); +} diff --git a/qa/lib/global-setup.ts b/qa/lib/global-setup.ts new file mode 100644 index 000000000..6a786f709 --- /dev/null +++ b/qa/lib/global-setup.ts @@ -0,0 +1,75 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { ensureSession } from "./auth.ts"; +import { ARTIFACTS, assertLocal, env, QA_DIR, RUN_DIR, touchStack } from "./env.ts"; +import { acquireRecordingLock, releaseRecordingLock } from "./lock.ts"; + +export type RunInfo = { sha: string; branch: string; dirty: boolean; webURL: string; startedAt: string }; + +async function reachable(url: string): Promise { + try { + return (await fetch(url, { signal: AbortSignal.timeout(5000) })).ok; + } catch { + return false; + } +} + +function git(...args: string[]): string { + return execFileSync("git", args, { cwd: QA_DIR, encoding: "utf8" }).trim(); +} + +// Keeps only this run's traces: earlier runs' output directories belong to processes that have exited. +function pruneOldResults(): void { + const dir = join(ARTIFACTS, "test-results"); + if (!existsSync(dir)) return; + for (const name of readdirSync(dir)) { + const pid = Number(name.replace(/^run-/, "")); + if (pid === process.pid) continue; + let alive = false; + try { + process.kill(pid, 0); + alive = true; + } catch { + // Exited. + } + if (!alive) rmSync(join(dir, name), { recursive: true, force: true }); + } +} + +export default async function globalSetup(): Promise { + if (env.mode === "none") { + throw new Error("this worktree has no stack running. Start one with `pnpm stack up` (in qa/), or set QA_WEB_URL and QA_API_URL to a local stack you started yourself."); + } + assertLocal(env.webURL); + assertLocal(env.apiURL); + + const down = []; + if (!(await reachable(env.webURL))) down.push(`dashboard ${env.webURL}`); + if (!(await reachable(`${env.apiURL}/health`))) down.push(`backend ${env.apiURL}`); + if (!(await reachable(`${env.mailpitURL}/api/v1/info`))) down.push(`mailpit ${env.mailpitURL}`); + if (down.length) { + throw new Error(`not reachable: ${down.join(", ")}. Start this worktree's stack with \`pnpm stack up\` (in qa/), or run \`pnpm doctor\`.`); + } + + touchStack(); + await acquireRecordingLock(QA_DIR); + try { + rmSync(RUN_DIR, { recursive: true, force: true }); + mkdirSync(RUN_DIR, { recursive: true }); + pruneOldResults(); + await ensureSession(); + } catch (err) { + releaseRecordingLock(); + throw err; + } + + const run: RunInfo = { + sha: git("rev-parse", "HEAD"), + branch: git("rev-parse", "--abbrev-ref", "HEAD"), + dirty: git("status", "--porcelain").length > 0, + webURL: env.webURL, + startedAt: new Date().toISOString(), + }; + writeFileSync(join(RUN_DIR, "run.json"), JSON.stringify(run, null, 2)); +} diff --git a/qa/lib/global-teardown.ts b/qa/lib/global-teardown.ts new file mode 100644 index 000000000..11390f866 --- /dev/null +++ b/qa/lib/global-teardown.ts @@ -0,0 +1,28 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { RUN_DIR } from "./env.ts"; +import { releaseRecordingLock } from "./lock.ts"; +import type { FlowMeta } from "./proof.ts"; +import { encodeFrames } from "./recorder.ts"; + +// Runs after the workers, and their browser, have exited: encodes each flow's frames one at a time. +export default async function globalTeardown(): Promise { + try { + if (!existsSync(RUN_DIR)) return; + for (const entry of readdirSync(RUN_DIR, { withFileTypes: true })) { + const metaFile = join(RUN_DIR, entry.name, "meta.json"); + if (!entry.isDirectory() || !existsSync(metaFile)) continue; + const meta = JSON.parse(readFileSync(metaFile, "utf8")) as FlowMeta; + if (!meta.frames) continue; + const out = join(RUN_DIR, entry.name, "video.mp4"); + const started = Date.now(); + await encodeFrames(meta.frames, out); + meta.video = out; + meta.frames = undefined; + writeFileSync(metaFile, JSON.stringify(meta, null, 2)); + console.log(`encoded ${entry.name}/video.mp4 in ${((Date.now() - started) / 1000).toFixed(1)}s`); + } + } finally { + releaseRecordingLock(); + } +} diff --git a/qa/lib/lock.ts b/qa/lib/lock.ts new file mode 100644 index 000000000..c2501b025 --- /dev/null +++ b/qa/lib/lock.ts @@ -0,0 +1,67 @@ +import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// One recording at a time on this machine, whichever worktree or agent starts it: +// each holds a Chromium and an encoder, and parallel ones only starve each other. +const LOCK = join(tmpdir(), "warmbly-qa-recording.lock"); +const WAIT_MIN = Number(process.env.QA_LOCK_WAIT_MIN ?? 30); + +type Holder = { pid: number; label: string; since: string }; + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function holder(): Holder | undefined { + try { + return JSON.parse(readFileSync(LOCK, "utf8")) as Holder; + } catch { + return undefined; + } +} + +export async function acquireRecordingLock(label: string): Promise { + const deadline = Date.now() + WAIT_MIN * 60_000; + let announced = false; + for (;;) { + try { + const fd = openSync(LOCK, "wx"); + writeSync(fd, JSON.stringify({ pid: process.pid, label, since: new Date().toISOString() } satisfies Holder)); + closeSync(fd); + return; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; + } + const h = holder(); + if (!h || !alive(h.pid)) { + try { + unlinkSync(LOCK); + } catch { + // Another waiter cleared it first. + } + continue; + } + if (h.pid === process.pid) return; + if (Date.now() > deadline) throw new Error(`gave up after ${WAIT_MIN} min waiting for the recording in ${h.label} (pid ${h.pid})`); + if (!announced) { + console.log(`waiting: another recording is running (${h.label}, pid ${h.pid}, since ${h.since})`); + announced = true; + } + await new Promise((r) => setTimeout(r, 2000)); + } +} + +export function releaseRecordingLock(): void { + if (holder()?.pid !== process.pid) return; + try { + unlinkSync(LOCK); + } catch { + // Already gone. + } +} diff --git a/qa/lib/proof.ts b/qa/lib/proof.ts new file mode 100644 index 000000000..42c246b11 --- /dev/null +++ b/qa/lib/proof.ts @@ -0,0 +1,148 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test as base, expect, type Locator, type Page, type TestInfo } from "@playwright/test"; +import { env, RUN_DIR } from "./env.ts"; +import { Recorder } from "./recorder.ts"; + +export type Box = { x: number; y: number; width: number; height: number }; +// `ignore` holds regions, in the still's own pixels, that comparisons skip. +export type Shot = { name: string; file: string; caption: string; ignore?: Box[] }; + +export type FlowMeta = { + slug: string; + title: string; + file: string; + status: TestInfo["status"]; + // Spooled frames waiting for the end-of-run encode; `video` replaces it once encoded. + frames?: string; + video?: string; + shots: Shot[]; +}; + +// Pages a signed-in flow must never land on: each means the session or the account setup is wrong. +const GATES: [RegExp, string][] = [ + [/^\/auth\//, "the sign-in page: the saved session was refused. Delete qa/.artifacts/auth and run again"], + [/^\/onboarding/, "onboarding: the account has not finished setup. Delete qa/.artifacts/auth so global setup completes it"], + [/^\/select-org/, "the workspace picker: the account belongs to several workspaces; open the one the flow needs first"], +]; + +// Dev-only chrome that would otherwise sit in every frame (the React Query devtools button). +const HIDE_DEV_CHROME = `.tsqd-parent-container, .tsqd-open-btn-container { display: none !important; }`; + +export class Proof { + readonly shots: Shot[] = []; + readonly dir: string; + private readonly page: Page; + + constructor(page: Page, dir: string) { + this.page = page; + this.dir = dir; + } + + // A title card in the video, so a reviewer knows what the next few seconds show. + async chapter(title: string, description?: string): Promise { + await this.page.screencast.showChapter(title, { description, duration: 1600 }); + } + + // A named full-resolution still. Keep names stable: follow-up comments diff stills by name. + // `ignore` names content that changes on its own (relative times, live counters): the still is + // published as is, and those regions are skipped when it is compared with the previous publish. + async shot(name: string, opts: { caption?: string; target?: Locator; ignore?: Locator[] } = {}): Promise { + if (!/^[a-z0-9][a-z0-9-]*$/.test(name)) throw new Error(`shot name "${name}" must be kebab-case`); + if (this.shots.some((s) => s.name === name)) throw new Error(`shot "${name}" taken twice in one flow`); + const file = join(this.dir, "shots", `${name}.png`); + const origin = opts.target ? await opts.target.boundingBox() : { x: 0, y: 0 }; + const ignore: Box[] = []; + for (const locator of opts.ignore ?? []) { + for (const el of await locator.all()) { + const b = await el.boundingBox(); + if (b && origin) ignore.push({ x: Math.floor(b.x - origin.x), y: Math.floor(b.y - origin.y), width: Math.ceil(b.width) + 1, height: Math.ceil(b.height) + 1 }); + } + } + const options = { path: file, animations: "disabled" as const, caret: "hide" as const }; + await this.page.screencast.hideOverlays(); + try { + if (opts.target) await opts.target.screenshot(options); + else await this.page.screenshot(options); + } finally { + await this.page.screencast.showOverlays(); + } + this.shots.push({ name, file, caption: opts.caption ?? name, ignore: ignore.length ? ignore : undefined }); + } + + // Lets a result sit on screen long enough to be read in the video. + async dwell(ms = 1200): Promise { + await this.page.waitForTimeout(ms); + } +} + +export function slugOf(info: TestInfo): string { + return info.titlePath + .slice(1) + .join(" ") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, ""); +} + +export type Seed = "rich" | "sandbox"; + +// The seed each stack mode loads; a stack the harness did not start ("external") is trusted as is. +const SEED_OF_MODE: Record = { lite: "rich", full: "rich", sandbox: "sandbox" }; + +export const test = base.extend<{ proof: Proof; signedIn: boolean; seed: Seed }>({ + // `test.use({ signedIn: false })` for a flow that starts on the sign-in pages. + signedIn: [true, { option: true }], + // `test.use({ seed: "sandbox" })` for a flow written against the Sunrise Labs sandbox data. + seed: ["rich", { option: true }], + storageState: async ({ signedIn, storageState }, use) => { + await use(signedIn ? storageState : { cookies: [], origins: [] }); + }, + proof: [ + async ({ page, signedIn, seed }, use, info) => { + const loaded = SEED_OF_MODE[env.mode]; + const fix = seed === "sandbox" ? "pnpm stack reset-data sandbox" : "pnpm stack reset-data lite"; + if (loaded !== undefined && loaded !== seed) { + const why = `written for the ${seed} seed, but the stack has the ${loaded} seed (${fix})`; + console.log(`skipped "${info.title}": ${why}`); + info.skip(true, why); + } + const slug = slugOf(info); + const dir = join(RUN_DIR, slug); + mkdirSync(join(dir, "shots"), { recursive: true }); + await page.addInitScript((css) => { + const add = () => document.head.appendChild(Object.assign(document.createElement("style"), { textContent: css })); + if (document.head) add(); + else document.addEventListener("DOMContentLoaded", add); + }, HIDE_DEV_CHROME); + + const frames = join(dir, "frames"); + const recorder = new Recorder(page, frames); + await recorder.start(); + await page.screencast.showActions({ cursor: "pointer", duration: 700, fontSize: 20, position: "top-right" }); + const proof = new Proof(page, dir); + + await use(proof); + + const captured = await recorder.stop(); + // A failed flow's video is never published, so its frames are not worth encoding unless asked for. + const keep = captured && (info.status === "passed" || process.env.QA_VIDEO_ON_FAIL === "1"); + if (!keep) rmSync(frames, { recursive: true, force: true }); + const meta: FlowMeta = { + slug, + title: info.title, + file: info.file, + status: info.status, + frames: keep ? frames : undefined, + shots: proof.shots, + }; + writeFileSync(join(dir, "meta.json"), JSON.stringify(meta, null, 2)); + const path = new URL(page.url()).pathname; + const gate = signedIn ? GATES.find(([re]) => re.test(path)) : undefined; + if (gate) throw new Error(`the dashboard sent this flow to ${gate[1]}.`); + }, + { auto: true }, + ], +}); + +export { expect }; diff --git a/qa/lib/recorder.ts b/qa/lib/recorder.ts new file mode 100644 index 000000000..fa25637d4 --- /dev/null +++ b/qa/lib/recorder.ts @@ -0,0 +1,125 @@ +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Page } from "@playwright/test"; +import { VIDEO } from "./env.ts"; + +// GitHub refuses larger attachments on some plans; above this the clip is re-encoded smaller. +const MAX_BYTES = Number(process.env.QA_MAX_VIDEO_MB ?? 10) * 1024 * 1024; +// x264 holds frames per thread, so this bounds the encoder's memory as much as its CPU. +const THREADS = String(process.env.QA_ENCODE_THREADS ?? 4); + +type Frame = { file: string; ts: number }; + +// Spools Playwright's screencast to disk as full-size JPEGs while the flow runs. +// Encoding waits until the browser is closed (`encodeFrames`), so the two never hold memory at once. +export class Recorder { + private readonly page: Page; + private readonly dir: string; + private readonly frames: Frame[] = []; + private lowRes = ""; + + constructor(page: Page, dir: string) { + this.page = page; + this.dir = dir; + } + + async start(): Promise { + mkdirSync(this.dir, { recursive: true }); + await this.page.screencast.start({ + size: { width: VIDEO.width, height: VIDEO.height }, + quality: 95, + onFrame: ({ data, timestamp }) => this.onFrame(data, timestamp), + }); + } + + // Frames arrive only when the page repaints; their timestamps carry the pauses in between. + private async onFrame(data: Buffer, timestamp: number): Promise { + if (!this.frames.length && !(await this.painted())) return; + if (!this.frames.length) { + const size = jpegSize(data); + if (size && size.width < VIDEO.width) this.lowRes = `${size.width}x${size.height}`; + } + const file = `f${String(this.frames.length).padStart(6, "0")}.jpg`; + this.frames.push({ file, ts: timestamp }); + await writeFile(join(this.dir, file), data); + } + + // The video starts at the first frame with visible text, not on the blank page before the app renders. + private async painted(): Promise { + if (this.page.url() === "about:blank") return false; + return this.page + .evaluate( + () => + performance.getEntriesByType("paint").some((e) => e.name === "first-contentful-paint") && + (document.body?.innerText ?? "").trim().length > 0, + ) + .catch(() => false); + } + + // Writes the frame list ffmpeg reads later. Returns false when nothing was captured. + async stop(tailMs = 800): Promise { + await this.page.screencast.stop().catch(() => {}); + if (this.lowRes) { + throw new Error(`screencast frames arrived at ${this.lowRes}: another screencast (trace screenshots?) started first and set the size`); + } + if (!this.frames.length) return false; + const end = Date.now() + tailMs; + const lines = ["ffconcat version 1.0"]; + this.frames.forEach((f, i) => { + const next = this.frames[i + 1]?.ts ?? end; + lines.push(`file '${f.file}'`, `duration ${Math.max(0.001, (next - f.ts) / 1000).toFixed(4)}`); + }); + // The concat demuxer ignores the last entry's duration unless the file is listed once more. + lines.push(`file '${this.frames[this.frames.length - 1].file}'`); + writeFileSync(join(this.dir, "frames.ffconcat"), `${lines.join("\n")}\n`); + return true; + } +} + +// Encodes a spooled flow to 1080p H.264, then removes the frames. +export async function encodeFrames(framesDir: string, out: string): Promise { + const list = join(framesDir, "frames.ffconcat"); + if (!existsSync(list)) throw new Error(`no frame list in ${framesDir}`); + const { width, height, fps, crf, preset } = VIDEO; + const vf = [ + `fps=${fps}`, + `scale=${width}:${height}:force_original_aspect_ratio=decrease:flags=lanczos:in_range=pc:out_range=tv`, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:white`, + "format=yuv420p", + ].join(","); + await ffmpeg(["-f", "concat", "-safe", "0", "-i", list, "-vf", vf, ...x264(crf, preset), out]); + for (let next = crf + 5; statSync(out).size > MAX_BYTES && next <= 38; next += 5) { + const tmp = out.replace(/\.mp4$/, `.crf${next}.mp4`); + await ffmpeg(["-i", out, ...x264(next, preset), tmp]); + await rename(tmp, out); + } + rmSync(framesDir, { recursive: true, force: true }); +} + +function x264(crf: number, preset: string): string[] { + return [ + ...["-c:v", "libx264", "-preset", preset, "-crf", String(crf), "-threads", THREADS], + ...["-x264-params", "rc-lookahead=20", "-pix_fmt", "yuv420p", "-color_range", "tv", "-movflags", "+faststart"], + ]; +} + +async function ffmpeg(args: string[]): Promise { + let stderr = ""; + const child = spawn("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { stdio: ["ignore", "ignore", "pipe"] }); + child.stderr.on("data", (d: Buffer) => (stderr += d.toString())); + const code = await new Promise((resolve) => child.on("close", resolve)); + if (code !== 0) throw new Error(`ffmpeg exited ${code}: ${stderr.trim()}`); +} + +// Reads the frame size from the JPEG's start-of-frame marker. +function jpegSize(buf: Buffer): { width: number; height: number } | undefined { + for (let i = 2; i + 9 < buf.length; ) { + if (buf[i] !== 0xff) return undefined; + const marker = buf[i + 1]; + if (marker >= 0xc0 && marker <= 0xc2) return { height: buf.readUInt16BE(i + 5), width: buf.readUInt16BE(i + 7) }; + i += 2 + buf.readUInt16BE(i + 2); + } + return undefined; +} diff --git a/qa/package.json b/qa/package.json new file mode 100644 index 000000000..c395a7a6a --- /dev/null +++ b/qa/package.json @@ -0,0 +1,27 @@ +{ + "name": "warmbly-qa", + "private": true, + "description": "Recorded proof for pull requests: scripted Playwright flows, 1080p H.264 walkthroughs and stills, published with gh", + "type": "module", + "engines": { + "node": ">=22.18" + }, + "scripts": { + "bootstrap": "sh scripts/bootstrap.sh", + "check": "node scripts/doctor.ts", + "stack": "bash scripts/stack.sh", + "proof": "playwright test", + "proof:fresh": "bash scripts/stack.sh reset-data && playwright test", + "aria": "node scripts/aria.ts", + "share": "node scripts/publish.ts", + "typecheck": "tsc --noEmit -p ." + }, + "devDependencies": { + "@playwright/test": "1.63.0", + "@types/node": "^26.6.4", + "@types/pngjs": "^6.0.5", + "pixelmatch": "^7.2.0", + "pngjs": "^7.0.0", + "typescript": "~5.9.3" + } +} diff --git a/qa/playwright.config.ts b/qa/playwright.config.ts new file mode 100644 index 000000000..8affc1c7f --- /dev/null +++ b/qa/playwright.config.ts @@ -0,0 +1,41 @@ +import { defineConfig } from "@playwright/test"; +import { authFile, env, VIDEO } from "./lib/env.ts"; + +// Playwright empties its output directory when a run starts, so a run waiting on the recording lock +// would delete the traces of the one recording. Each run gets its own; workers inherit the id. +process.env.QA_RUN_ID ??= String(process.pid); + +export default defineConfig({ + testDir: "./flows", + testMatch: "**/*.flow.ts", + outputDir: `./.artifacts/test-results/run-${process.env.QA_RUN_ID}`, + globalSetup: "./lib/global-setup.ts", + // Encodes the run's videos after the browser has exited, then releases the machine-wide recording lock. + globalTeardown: "./lib/global-teardown.ts", + // One flow at a time: a 1080p encode per parallel worker starves the browser and the video stutters. + workers: 1, + fullyParallel: false, + retries: 0, + timeout: 180_000, + expect: { timeout: 15_000 }, + reporter: [["list"]], + use: { + browserName: "chromium", + // Full Chromium in new headless mode renders exactly like the desktop browser. + channel: "chromium", + baseURL: env.webURL, + storageState: authFile(), + viewport: { width: VIDEO.width, height: VIDEO.height }, + deviceScaleFactor: 1, + colorScheme: "light", + locale: "en-US", + actionTimeout: 15_000, + navigationTimeout: 30_000, + video: "off", + screenshot: "off", + // Trace screenshots would start a small screencast first, and the recorder would share its size. + trace: { mode: "retain-on-failure", screenshots: false, snapshots: true }, + // Human pacing, so a reviewer can follow each click in the video. + launchOptions: { slowMo: Number(process.env.QA_SLOWMO ?? 250) }, + }, +}); diff --git a/qa/pnpm-lock.yaml b/qa/pnpm-lock.yaml new file mode 100644 index 000000000..935edb9eb --- /dev/null +++ b/qa/pnpm-lock.yaml @@ -0,0 +1,97 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@playwright/test': + specifier: 1.63.0 + version: 1.63.0 + '@types/node': + specifier: ^26.6.4 + version: 26.6.4 + '@types/pngjs': + specifier: ^6.0.5 + version: 6.0.5 + pixelmatch: + specifier: ^7.2.0 + version: 7.2.0 + pngjs: + specifier: ^7.0.0 + version: 7.0.0 + typescript: + specifier: ~5.9.3 + version: 5.9.3 + +packages: + + '@playwright/test@1.63.0': + resolution: {integrity: sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==} + engines: {node: '>=20'} + hasBin: true + + '@types/node@26.6.4': + resolution: {integrity: sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg==} + + '@types/pngjs@6.0.5': + resolution: {integrity: sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==} + + pixelmatch@7.2.0: + resolution: {integrity: sha512-xhcb4yHu9sM/G7foGzoLtXYcC0zHEaOXXjRKhGup0fw78Nf2Tkiapv4EQyMzrbcmQPsllAI7DbFY2UT7PlI9Pg==} + hasBin: true + + playwright-core@1.63.0: + resolution: {integrity: sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==} + engines: {node: '>=20'} + hasBin: true + + playwright@1.63.0: + resolution: {integrity: sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==} + engines: {node: '>=20'} + hasBin: true + + pngjs@7.0.0: + resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} + engines: {node: '>=14.19.0'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@8.9.0: + resolution: {integrity: sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==} + +snapshots: + + '@playwright/test@1.63.0': + dependencies: + playwright: 1.63.0 + + '@types/node@26.6.4': + dependencies: + undici-types: 8.9.0 + + '@types/pngjs@6.0.5': + dependencies: + '@types/node': 26.6.4 + + pixelmatch@7.2.0: + dependencies: + pngjs: 7.0.0 + + playwright-core@1.63.0: {} + + playwright@1.63.0: + dependencies: + playwright-core: 1.63.0 + + pngjs@7.0.0: {} + + typescript@5.9.3: {} + + undici-types@8.9.0: {} diff --git a/qa/scripts/aria.ts b/qa/scripts/aria.ts new file mode 100644 index 000000000..b67f6bc8b --- /dev/null +++ b/qa/scripts/aria.ts @@ -0,0 +1,27 @@ +// Prints a page's accessibility tree with the saved session: the cheap way to find selectors for a flow. +// node scripts/aria.ts /app/contacts [css-selector] +import { chromium } from "@playwright/test"; +import { existsSync } from "node:fs"; +import { assertLocal, authFile, env, QA_DIR, touchStack } from "../lib/env.ts"; +import { acquireRecordingLock, releaseRecordingLock } from "../lib/lock.ts"; + +const [path = "/app/emails", selector = "body"] = process.argv.slice(2); +if (env.mode === "none") { + console.error("this worktree has no stack running; start one with `pnpm stack up`"); + process.exit(1); +} +assertLocal(env.webURL); +if (!existsSync(authFile())) { + console.error("no saved session yet; run any flow once (`pnpm proof`) so global setup signs in"); + process.exit(1); +} +touchStack(); +await acquireRecordingLock(`${QA_DIR} (aria)`); +const browser = await chromium.launch({ channel: "chromium" }); +const context = await browser.newContext({ storageState: authFile(), viewport: { width: 1920, height: 1080 } }); +const page = await context.newPage(); +await page.goto(new URL(path, env.webURL).toString()); +await page.waitForLoadState("networkidle"); +console.log(await page.locator(selector).first().ariaSnapshot()); +await browser.close(); +releaseRecordingLock(); diff --git a/qa/scripts/bootstrap.sh b/qa/scripts/bootstrap.sh new file mode 100644 index 000000000..9ffda8bcc --- /dev/null +++ b/qa/scripts/bootstrap.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Installs the harness: dependencies, Playwright's Chromium and the small images every stack runs. Then checks it all. +set -eu +cd "$(dirname "$0")/.." +pnpm install --frozen-lockfile +pnpm exec playwright install chromium +for image in redis:7-alpine nats:2.10-alpine axllent/mailpit:latest; do + docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image" +done +exec node scripts/doctor.ts diff --git a/qa/scripts/doctor.ts b/qa/scripts/doctor.ts new file mode 100644 index 000000000..9241450e0 --- /dev/null +++ b/qa/scripts/doctor.ts @@ -0,0 +1,97 @@ +// Checks everything a recording needs and prints the fix for whatever is missing. +// node scripts/doctor.ts +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { chromium } from "@playwright/test"; +import { authFile, env } from "../lib/env.ts"; + +let failed = false; + +function report(ok: boolean, what: string, fix?: string): void { + console.log(`${ok ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m"} ${what}`); + if (!ok) { + failed = true; + if (fix) console.log(` fix: ${fix}`); + } +} + +function run(cmd: string, args: string[]): { ok: boolean; out: string } { + const res = spawnSync(cmd, args, { encoding: "utf8" }); + return { ok: res.status === 0, out: `${res.stdout ?? ""}${res.stderr ?? ""}` }; +} + +function atLeast(version: string, min: number[]): boolean { + const parts = version.split(".").map(Number); + for (let i = 0; i < min.length; i++) { + if ((parts[i] ?? 0) !== min[i]) return (parts[i] ?? 0) > min[i]; + } + return true; +} + +async function reachable(url: string): Promise { + try { + return (await fetch(url, { signal: AbortSignal.timeout(4000) })).ok; + } catch { + return false; + } +} + +const node = process.versions.node; +report(atLeast(node, [22, 18]), `node ${node} (22.18+ runs the TypeScript scripts directly)`, "install Node 22.18 or newer"); + +const ffmpeg = run("ffmpeg", ["-hide_banner", "-encoders"]); +report(ffmpeg.ok, "ffmpeg on PATH", "install ffmpeg (pacman -S ffmpeg / apt install ffmpeg / brew install ffmpeg)"); +if (ffmpeg.ok) report(/libx264/.test(ffmpeg.out), "ffmpeg has libx264", "install an ffmpeg build with libx264 (the distro package has it)"); + +const exe = chromium.executablePath(); +report(existsSync(exe), "Playwright Chromium installed", "pnpm exec playwright install chromium"); +if (existsSync(exe)) { + try { + const browser = await chromium.launch({ channel: "chromium" }); + await browser.close(); + report(true, "Chromium launches"); + } catch (err) { + report(false, `Chromium launches (${String(err).split("\n")[0]})`, "sudo pnpm exec playwright install-deps chromium"); + } +} + +const gh = run("gh", ["--version"]); +const ghVersion = gh.out.match(/gh version (\d+\.\d+\.\d+)/)?.[1] ?? ""; +report(gh.ok && atLeast(ghVersion, [2, 99]), `gh ${ghVersion || "missing"} (2.99+ uploads media with --attach)`, "install or upgrade the GitHub CLI: https://cli.github.com"); +if (gh.ok) report(run("gh", ["auth", "status"]).ok, "gh is signed in", "gh auth login"); + +console.log(""); +const docker = run("docker", ["info", "--format", "{{.ServerVersion}}"]); +report(docker.ok, "docker is running", "start the docker daemon"); +if (docker.ok) { + const names = run("docker", ["ps", "--format", "{{.Names}}"]).out; + report(/^warmbly-postgres-1$/m.test(names), "shared postgres (warmbly-postgres-1)", "make infra (from the repo root)"); + const images: [string, string][] = [ + ["redis:7-alpine", "every mode"], + ["nats:2.10-alpine", "every mode"], + ["axllent/mailpit:latest", "every mode"], + ["ghcr.io/warmbly/warmbly/realtime:prod", "full and sandbox"], + ["dovecot/dovecot:latest", "sandbox"], + ["ghcr.io/warmbly/warmbly/tracking:prod", "sandbox"], + ]; + for (const [image, modes] of images) { + const present = run("docker", ["image", "inspect", image]).ok; + const required = modes === "every mode" || (modes.includes(env.mode) && env.mode !== "external"); + if (present) report(true, `image ${image}`); + else if (required) report(false, `image ${image} (${modes})`, `docker pull ${image}`); + else console.log(` - image ${image} not pulled (needed for ${modes}): docker pull ${image}`); + } +} + +console.log(""); +if (env.mode === "none") { + report(false, "this worktree's stack is running", "pnpm stack up (lite; `full` or `sandbox` for more)"); +} else { + console.log(` stack: ${env.mode === "external" ? "QA_WEB_URL" : `mode ${env.mode}`}, signs in as ${env.email}`); + report(await reachable(env.webURL), `dashboard ${env.webURL}`, "pnpm stack up"); + report(await reachable(`${env.apiURL}/health`), `backend ${env.apiURL}`, "pnpm stack up"); + report(await reachable(`${env.mailpitURL}/api/v1/info`), `mailpit ${env.mailpitURL} (login codes)`, "pnpm stack up"); + console.log(` session: ${existsSync(authFile()) ? "saved" : "none yet; the first `pnpm proof` signs in"}`); +} + +process.exit(failed ? 1 : 0); diff --git a/qa/scripts/publish.ts b/qa/scripts/publish.ts new file mode 100644 index 000000000..830ceb79f --- /dev/null +++ b/qa/scripts/publish.ts @@ -0,0 +1,262 @@ +// Publishes the last recorded run to the current branch's pull request. +// First publish: a Proof section appended to the PR description (stills inline, walkthrough videos last). +// Every later publish: a comment with before/after pairs for the stills that changed since the previous one. +// +// node scripts/publish.ts publish +// node scripts/publish.ts --dry-run write the body and print the gh command, upload nothing +// node scripts/publish.ts --no-video stills only +// node scripts/publish.ts --force comment even when no still changed +// node scripts/publish.ts --replace-body rewrite the PR description's Proof section instead of commenting +// node scripts/publish.ts --allow-failed publish a run with failing flows +import { execFileSync, spawnSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join, relative } from "node:path"; +import pixelmatch from "pixelmatch"; +import pngjs from "pngjs"; +import { ARTIFACTS, assertLocal, QA_DIR, RUN_DIR } from "../lib/env.ts"; +import type { RunInfo } from "../lib/global-setup.ts"; +import type { Box, FlowMeta, Shot } from "../lib/proof.ts"; + +const { PNG } = pngjs; +const MARK = ""; +const MARK_END = ""; +const MAX_ATTACHMENTS = 50; + +const flags = new Set(process.argv.slice(2)); +for (const f of flags) { + if (!["--dry-run", "--no-video", "--force", "--replace-body", "--allow-failed"].includes(f)) die(`unknown flag ${f}`); +} + +function die(msg: string): never { + console.error(`publish: ${msg}`); + process.exit(1); +} + +function git(...args: string[]): string { + return execFileSync("git", args, { cwd: QA_DIR, encoding: "utf8" }).trim(); +} + +function short(sha: string): string { + return sha.slice(0, 9); +} + +type Baseline = { sha: string; shots: Record; ignore?: Record }; +type Pr = { number: number; body: string; url: string }; +type Change = { flow: FlowMeta; shot: Shot; key: string; kind: "new" | "changed"; before?: string }; + +function loadRun(): { run: RunInfo; flows: FlowMeta[] } { + const runFile = join(RUN_DIR, "run.json"); + if (!existsSync(runFile)) die("no recorded run; record one with `pnpm proof` first"); + const run = JSON.parse(readFileSync(runFile, "utf8")) as RunInfo; + const flows = readdirSync(RUN_DIR, { withFileTypes: true }) + .filter((d) => d.isDirectory() && existsSync(join(RUN_DIR, d.name, "meta.json"))) + .map((d) => JSON.parse(readFileSync(join(RUN_DIR, d.name, "meta.json"), "utf8")) as FlowMeta) + .sort((a, b) => a.slug.localeCompare(b.slug)); + if (!flows.length) die("the last run recorded no flows"); + return { run, flows }; +} + +function currentPr(): Pr { + const res = spawnSync("gh", ["pr", "view", "--json", "number,body,url"], { cwd: QA_DIR, encoding: "utf8" }); + // A dry run previews the first publish before the PR exists. + if (res.status !== 0 && flags.has("--dry-run")) return { number: 0, body: "", url: "(no pull request yet)" }; + if (res.status !== 0) die(`no pull request for this branch yet. Open it with \`gh pr create\`, then publish.\n${res.stderr.trim()}`); + return JSON.parse(res.stdout) as Pr; +} + +// Blanks the regions a flow marked as changing on its own, in both images, before comparing. +function blank(img: InstanceType, boxes: Box[]): void { + for (const b of boxes) { + for (let y = Math.max(0, b.y); y < Math.min(img.height, b.y + b.height); y++) { + img.data.fill(0, (y * img.width + Math.max(0, b.x)) * 4, (y * img.width + Math.min(img.width, b.x + b.width)) * 4); + } + } +} + +function differs(a: string, b: string, ignore: Box[]): boolean { + const x = PNG.sync.read(readFileSync(a)); + const y = PNG.sync.read(readFileSync(b)); + if (x.width !== y.width || x.height !== y.height) return true; + blank(x, ignore); + blank(y, ignore); + const changed = pixelmatch(x.data, y.data, undefined, x.width, x.height, { threshold: 0.1 }); + return changed > Math.max(40, x.width * x.height * 0.0002); +} + +function keyOf(flow: FlowMeta, shot: Shot): string { + return `${flow.slug}__${shot.name}`; +} + +// Attachments are referenced in the body by the exact path passed to --attach, which gh rewrites to the upload. +class Attachments { + readonly dir = join(ARTIFACTS, "publish"); + readonly paths: string[] = []; + + constructor() { + rmSync(this.dir, { recursive: true, force: true }); + mkdirSync(this.dir, { recursive: true }); + } + + add(src: string, name: string): string { + const dest = join(this.dir, name); + copyFileSync(src, dest); + const ref = `./${relative(QA_DIR, dest)}`; + this.paths.push(ref); + return ref; + } +} + +function sentence(title: string): string { + return title.charAt(0).toUpperCase() + title.slice(1); +} + +function image(alt: string, ref: string): string { + return `![${alt.replace(/[[\]]/g, "")}](${ref})`; +} + +function initialSection(run: RunInfo, flows: FlowMeta[], files: Attachments, videos: FlowMeta[]): string { + const lines = [ + MARK, + "## Proof", + "", + `Recorded at \`${short(run.sha)}\` against a local stack with seed data, 1920x1080.`, + "", + ]; + for (const flow of flows) { + lines.push(`### ${sentence(flow.title)}`, ""); + for (const shot of flow.shots) lines.push(`**${shot.caption}**`, "", image(shot.caption, files.add(shot.file, `${flow.slug}-${shot.name}.png`)), ""); + } + if (videos.length) { + lines.push("### Walkthrough", "", "Videos below, in this order:", ""); + videos.forEach((f, i) => lines.push(`${i + 1}. ${sentence(f.title)}`)); + lines.push(""); + } + lines.push(MARK_END); + return lines.join("\n"); +} + +function updateComment(run: RunInfo, base: Baseline | undefined, changes: Change[], unchanged: number, videos: FlowMeta[], files: Attachments): string { + const lines: string[] = [""]; + if (base) { + lines.push(`### Proof update: \`${short(base.sha)}..${short(run.sha)}\``, ""); + try { + const subjects = git("log", "--format=%s", "--max-count=20", `${base.sha}..${run.sha}`).split("\n").filter(Boolean); + for (const s of subjects) lines.push(`- ${s}`); + if (subjects.length) lines.push(""); + } catch { + // A rebase can drop the old commit; the range header still says what moved. + } + } else { + lines.push(`### Proof update at \`${short(run.sha)}\``, "", "No earlier publish is recorded on this machine, so every screen is shown.", ""); + } + + let flowTitle = ""; + for (const c of changes) { + if (c.flow.title !== flowTitle) { + flowTitle = c.flow.title; + lines.push(`#### ${sentence(flowTitle)}`, ""); + } + const after = files.add(c.shot.file, `${c.flow.slug}-${c.shot.name}.png`); + if (c.kind === "changed" && c.before) { + const before = files.add(c.before, `before-${c.flow.slug}-${c.shot.name}.png`); + lines.push(`**${c.shot.caption}** (changed)`, "", "| Before | After |", "| --- | --- |"); + lines.push(`| ${image(`${c.shot.caption}, before`, before)} | ${image(`${c.shot.caption}, after`, after)} |`, ""); + } else { + lines.push(`**${c.shot.caption}** (new)`, "", image(c.shot.caption, after), ""); + } + } + if (unchanged) lines.push(`Unchanged: ${unchanged} screen${unchanged === 1 ? "" : "s"}.`, ""); + if (videos.length) { + lines.push("Walkthrough videos below, in this order:", ""); + videos.forEach((f, i) => lines.push(`${i + 1}. ${sentence(f.title)}`)); + } + return lines.join("\n").trimEnd(); +} + +function saveBaseline(file: string, run: RunInfo, flows: FlowMeta[]): void { + const dir = join(file, "..", "shots"); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + const shots: Record = {}; + const ignore: Record = {}; + for (const flow of flows) { + for (const shot of flow.shots) { + const key = keyOf(flow, shot); + shots[key] = join(dir, `${key}.png`); + copyFileSync(shot.file, shots[key]); + if (shot.ignore) ignore[key] = shot.ignore; + } + } + writeFileSync(file, JSON.stringify({ sha: run.sha, shots, ignore } satisfies Baseline, null, 2)); +} + +function main(): void { + const { run, flows } = loadRun(); + assertLocal(run.webURL); + + const failed = flows.filter((f) => f.status !== "passed"); + if (failed.length && !flags.has("--allow-failed")) { + die(`flows failed, fix them before publishing: ${failed.map((f) => f.title).join(", ")}`); + } + const unencoded = flows.filter((f) => f.frames); + if (unencoded.length) die(`the run ended before its videos were encoded (${unencoded.map((f) => f.title).join(", ")}); record again`); + const head = git("rev-parse", "HEAD"); + if (run.sha !== head) die(`the recording is from ${short(run.sha)} but HEAD is ${short(head)}; record again`); + if (run.dirty) console.warn("publish: warning: the recording was made with uncommitted changes in the tree"); + if (!git("branch", "-r", "--contains", head)) console.warn("publish: warning: HEAD is not pushed yet, so reviewers cannot see this commit"); + + const pr = currentPr(); + const baselineFile = join(ARTIFACTS, "published", `pr-${pr.number}`, "state.json"); + const base = existsSync(baselineFile) ? (JSON.parse(readFileSync(baselineFile, "utf8")) as Baseline) : undefined; + const files = new Attachments(); + const withVideo = (list: FlowMeta[]) => (flags.has("--no-video") ? [] : list.filter((f) => f.video && existsSync(f.video))); + const bodyFile = join(files.dir, "body.md"); + let args: string[]; + + const firstPublish = !pr.body?.includes(MARK); + if (firstPublish || flags.has("--replace-body")) { + const videos = withVideo(flows); + const kept = (pr.body ?? "").split(MARK)[0].trimEnd(); + writeFileSync(bodyFile, `${kept ? `${kept}\n\n` : ""}${initialSection(run, flows, files, videos)}\n`); + for (const f of videos) files.add(f.video!, `${f.slug}.mp4`); + args = ["pr", "edit", String(pr.number), "--body-file", bodyFile]; + } else { + const changes: Change[] = []; + let unchanged = 0; + for (const flow of flows) { + for (const shot of flow.shots) { + const key = keyOf(flow, shot); + const before = base?.shots[key]; + if (!before || !existsSync(before)) changes.push({ flow, shot, key, kind: "new" }); + else if (differs(before, shot.file, [...(base?.ignore?.[key] ?? []), ...(shot.ignore ?? [])])) changes.push({ flow, shot, key, kind: "changed", before }); + else unchanged++; + } + } + if (!changes.length && !flags.has("--force")) { + console.log(`publish: no screen changed since ${base ? short(base.sha) : "the last publish"}; nothing posted (--force posts anyway)`); + return; + } + const changedFlows = new Set(changes.map((c) => c.flow.slug)); + const videos = withVideo(flows.filter((f) => changedFlows.has(f.slug) || flags.has("--force"))); + writeFileSync(bodyFile, `${updateComment(run, base, changes, unchanged, videos, files)}\n`); + for (const f of videos) files.add(f.video!, `${f.slug}.mp4`); + args = ["pr", "comment", String(pr.number), "--body-file", bodyFile]; + } + + if (files.paths.length > MAX_ATTACHMENTS) { + die(`${files.paths.length} attachments is over gh's limit of ${MAX_ATTACHMENTS}; record fewer flows or pass --no-video`); + } + for (const p of files.paths) args.push("--attach", p); + + if (flags.has("--dry-run")) { + console.log(readFileSync(bodyFile, "utf8")); + console.log(`\n(dry run) gh ${args.join(" ")}`); + return; + } + const res = spawnSync("gh", args, { cwd: QA_DIR, stdio: "inherit" }); + if (res.status !== 0) die("gh failed; nothing was recorded as published"); + saveBaseline(baselineFile, run, flows); + console.log(`publish: ${firstPublish || flags.has("--replace-body") ? "description updated" : "comment posted"} on ${pr.url}`); +} + +main(); diff --git a/qa/scripts/stack.sh b/qa/scripts/stack.sh new file mode 100755 index 000000000..fddd8d796 --- /dev/null +++ b/qa/scripts/stack.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# This worktree's own Warmbly stack, isolated from every other stack on the machine: +# its own database, Redis, NATS and Mailpit, so no cache, event or login code crosses over. +# +# stack.sh up [lite|full|sandbox] start (the mode is remembered; lite on first run) +# stack.sh down stop everything; the data stays +# stack.sh restart rebuild the Go binaries and start again +# stack.sh status what runs, where, and how much memory it holds +# stack.sh logs [name] follow the logs (backend, web, consumer, worker, simulator) +# stack.sh reset-data [mode] drop the database and start again with fresh seed data +# stack.sh ls every QA stack on this machine +# +# Modes: +# lite backend + dashboard on the rich seed (SEED_RICH + SEED_FULL), dev@warmbly.com +# full lite + consumer, worker and realtime: sends, syncs and live updates work +# sandbox full + tracking, Dovecot and the simulator on the Sunrise Labs sandbox seed +# (sandbox@warmbly.test): live mailboxes, opens, clicks and replies +# +# The stack stops itself after QA_STACK_IDLE_MIN minutes (default 60) with no recording; 0 keeps it up. +set -euo pipefail + +QA=$(cd "$(dirname "$0")/.." && pwd) +REPO=$(cd "$QA/.." && pwd) +RUN=$QA/.artifacts/stack +BIN=$RUN/bin +mkdir -p "$RUN" + +slug=$(basename "$REPO" | tr -c '[:alnum:]\n' '_' | tr '[:upper:]' '[:lower:]') +DB=${QA_DB:-warmbly_qa_$slug} +LABEL=warmbly-qa=$slug +IDLE_MIN=${QA_STACK_IDLE_MIN:-60} +REALTIME_IMAGE=${QA_REALTIME_IMAGE:-ghcr.io/warmbly/warmbly/realtime:prod} +TRACKING_IMAGE=${QA_TRACKING_IMAGE:-ghcr.io/warmbly/warmbly/tracking:prod} +AUTH_SECRET=local-dev-auth-secret-minimum-32-characters-long +INTERNAL_TOKEN=local-dev-internal-token +MODE=lite + +say() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33mwarning:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +alive() { [ -s "$RUN/$1.pid" ] && kill -0 "$(cat "$RUN/$1.pid")" 2>/dev/null; } +psql_() { docker exec warmbly-postgres-1 psql -U warmbly "$@"; } +cname() { echo "wqa-$slug-$1"; } +NET=wqa-$slug +running() { [ "$(docker inspect -f '{{.State.Running}}' "$(cname "$1")" 2>/dev/null)" = true ]; } + +listening() { + if command -v ss >/dev/null; then ss -ltn | awk '{print $4}' | grep -qE ":$1\$" + else lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1; fi +} + +has() { # does the current mode include this tier + case "$1" in + full) [ "$MODE" = full ] || [ "$MODE" = sandbox ] ;; + sandbox) [ "$MODE" = sandbox ] ;; + esac +} + +# ── ports ──────────────────────────────────────────────────────────────── +PORT_KEYS="API WEB REDIS NATS SMTP MAILPIT RT TRACK IMAPS VENDOR" +base_of() { + case "$1" in + API) echo 18180 ;; WEB) echo 15280 ;; REDIS) echo 16480 ;; NATS) echo 14322 ;; + SMTP) echo 11125 ;; MAILPIT) echo 18125 ;; RT) echo 14100 ;; TRACK) echo 13100 ;; + IMAPS) echo 10994 ;; VENDOR) echo 18199 ;; + esac +} + +assigned() { # port already given to another key of this stack + local k + for k in $PORT_KEYS; do [ "${!k:-}" = "$1" ] && return 0; done + return 1 +} + +pick() { # first port at or above $1 that is free and not assigned here + local p=$1 + while listening "$p" || assigned "$p"; do p=$((p + 1)); done + echo "$p" +} + +save_ports() { + local k + : >"$RUN/ports" + for k in $PORT_KEYS; do echo "$k=${!k}" >>"$RUN/ports"; done +} + +load_ports() { + local k changed="" + # shellcheck disable=SC1091 + [ -s "$RUN/ports" ] && . "$RUN/ports" + for k in $PORT_KEYS; do + if [ -z "${!k:-}" ]; then printf -v "$k" '%s' "$(pick "$(base_of "$k")")"; changed=1; fi + done + [ -n "$changed" ] && save_ports + return 0 +} + +# A port another process took while the stack was down is reassigned rather than fought over. +heal_ports() { + local k new + for k in $PORT_KEYS; do + if listening "${!k}"; then + printf -v "$k" '%s' "" + new=$(pick "$(base_of "$k")") + warn "a port for $k was taken by another process; using $new" + printf -v "$k" '%s' "$new" + fi + done + save_ports +} + +db_exists() { psql_ -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$DB'" 2>/dev/null | grep -q 1; } +db_seeded() { db_exists && [ "$(psql_ -d "$DB" -tAc "SELECT count(*) FROM users" 2>/dev/null || echo 0)" != 0 ]; } + +load_mode() { + local saved + saved=$(cat "$RUN/mode" 2>/dev/null || true) + MODE=${1:-${saved:-lite}} + case "$MODE" in lite | full | sandbox) ;; *) die "unknown mode $MODE (lite, full, sandbox)" ;; esac + # lite and full share a seed; the sandbox seed is a different organization and account. + if [ -n "$saved" ] && [ "$saved" != "$MODE" ] && { [ "$saved" = sandbox ] || [ "$MODE" = sandbox ]; } && db_seeded; then + die "$DB holds the $saved seed; $MODE needs a fresh one: stack.sh reset-data $MODE" + fi +} + +# ── commands derived from the Makefile, so the env stays in step with `make dev` ─ +rewrite() { + local mock="" + has sandbox && mock="http://127.0.0.1:$VENDOR" + sed \ + -e "s#/warmbly_dev?#/$DB?#g" \ + -e "s#API_HOST=0.0.0.0:8080#API_HOST=0.0.0.0:$API#" \ + -e "s#localhost:8080#localhost:$API#g" \ + -e "s#APP_URL=http://localhost:5173#APP_URL=http://localhost:$WEB#" \ + -e "s#CORS_ALLOW_ORIGINS= #CORS_ALLOW_ORIGINS=http://localhost:$WEB #" \ + -e "s#redis://localhost:16379#redis://localhost:$REDIS#g" \ + -e "s#nats://localhost:4222#nats://localhost:$NATS#g" \ + -e "s#SMTP_PORT=11025#SMTP_PORT=$SMTP#g" \ + -e "s#ws://localhost:4000#ws://localhost:$RT#g" \ + -e "s#TRACKING_DOMAIN=localhost:3000#TRACKING_DOMAIN=localhost:$TRACK#g" \ + -e "s#BLOB_FS_ROOT=/tmp/warmbly-blobs#BLOB_FS_ROOT=$RUN/blobs#g" \ + -e "s#MAILVENDOR_SANDBOX_URL=[^ ]*#MAILVENDOR_SANDBOX_URL=$mock#" \ + -e "s#go run ./cmd/\\([a-z]*\\)#$BIN/\\1#g" +} + +make_cmd() { (cd "$REPO" && make -s -n "$1") | rewrite; } + +# The sandbox reads its own endpoints from the environment (internal/sandbox/config.go). +sandbox_env() { + echo "MAILPIT_URL=http://localhost:$MAILPIT TRACKING_URL=http://localhost:$TRACK DOVECOT_IMAP_ADDR=localhost:$IMAPS" \ + "SANDBOX_SMTP_PORT=$SMTP SANDBOX_IMAP_PORT=$IMAPS SANDBOX_VENDOR_ADDR=127.0.0.1:$VENDOR" +} + +start() { # name, command + rm -f "$RUN/$1.pid" + (cd "$REPO" && setsid -f bash -c "echo \$\$ >'$RUN/$1.pid'; $2" >"$RUN/$1.log" 2>&1 /dev/null 2>&1 && return 0 + sleep 1 + done + die "$1 was not ready within $3s (logs: stack.sh logs $1, or docker logs $(cname "$1"))" +} + +stop_proc() { + local pid + if alive "$1"; then + pid=$(cat "$RUN/$1.pid") + kill -TERM -- "-$pid" 2>/dev/null || true + for _ in $(seq 1 20); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done + kill -KILL -- "-$pid" 2>/dev/null || true + fi + rm -f "$RUN/$1.pid" +} + +container() { # name, docker run args... + local name=$1 + shift + running "$name" && return 0 + docker rm -f "$(cname "$name")" >/dev/null 2>&1 || true + docker run -d --name "$(cname "$name")" --network "$NET" --label "$LABEL" \ + --add-host host.docker.internal:host-gateway "$@" >/dev/null +} + +# Warns when the image was built before the last commit to its source, so a recording never shows stale code silently. +check_image() { # image, source dir + local built changed + built=$(docker inspect -f '{{.Created}}' "$1" 2>/dev/null) || die "image $1 is missing: cd $REPO && docker compose -p warmbly build ${2%/}" + changed=$(cd "$REPO" && git log -1 --format=%cI -- "$2") + if [ -n "$changed" ] && [ "$(date -d "$changed" +%s)" -gt "$(date -d "$built" +%s)" ]; then + warn "$1 was built before the last change to $2; rebuild: cd $REPO && docker compose -p warmbly build ${2%/}" + fi +} + +build_bins() { + local cmds=(./cmd/backend ./cmd/seed) + has full && cmds+=(./cmd/consumer ./cmd/worker) + has sandbox && cmds+=(./cmd/sandbox) + say "building ${cmds[*]##*/} (one go build, cached between runs)" + mkdir -p "$BIN" + (cd "$REPO" && go build -o "$BIN/" "${cmds[@]}") || die "go build failed" +} + +account() { if has sandbox; then echo sandbox@warmbly.test; else echo dev@warmbly.com; fi; } + +write_env() { + cat >"$RUN/env.json" <"$RUN/last-used" + start watchdog "while sleep 60; do + last=\$(stat -c %Y '$RUN/last-used' 2>/dev/null || echo 0) + if [ \$(( \$(date +%s) - last )) -ge $((IDLE_MIN * 60)) ]; then + echo \"unused for $IDLE_MIN min, stopping\"; rm -f '$RUN/watchdog.pid'; exec bash '$QA/scripts/stack.sh' down + fi + done" +} + +seed() { + if has sandbox; then + (cd "$REPO" && eval "$(sandbox_env) $(make_cmd sandbox-seed | grep "$BIN/sandbox")") + else + (cd "$REPO" && eval "$(make_cmd seed)") + fi +} + +# ── lifecycle ──────────────────────────────────────────────────────────── +up() { + local c + for c in docker go pnpm setsid curl; do command -v "$c" >/dev/null || die "$c is required"; done + load_ports + load_mode "${1:-}" + if alive backend && alive web; then + if [ "$(cat "$RUN/mode" 2>/dev/null)" = "$MODE" ]; then status; return 0; fi + say "switching to $MODE" + down >/dev/null + fi + heal_ports + echo "$MODE" >"$RUN/mode" + + docker ps --format '{{.Names}}' | grep -q '^warmbly-postgres-1$' || (cd "$REPO" && make infra) + until docker exec warmbly-postgres-1 pg_isready -U warmbly >/dev/null 2>&1; do sleep 1; done + db_exists || { say "creating database $DB"; psql_ -d postgres -c "CREATE DATABASE $DB" >/dev/null; } + [ -d "$REPO/web/node_modules" ] || { say "installing web deps"; (cd "$REPO/web" && pnpm install --frozen-lockfile); } + mkdir -p "$RUN/blobs" + build_bins + + # Containers reach each other by name on the stack's network; the host reaches them on loopback ports. + docker network inspect "$NET" >/dev/null 2>&1 || docker network create --label "$LABEL" "$NET" >/dev/null + say "starting redis, nats and mailpit (private to this stack)" + container redis -p "127.0.0.1:$REDIS:6379" redis:7-alpine redis-server --save "" --appendonly no + container nats -p "127.0.0.1:$NATS:4222" nats:2.10-alpine -js + container mailpit -p "127.0.0.1:$MAILPIT:8025" -p "127.0.0.1:$SMTP:1025" \ + -e MP_SMTP_AUTH_ACCEPT_ANY=1 -e MP_SMTP_AUTH_ALLOW_INSECURE=1 axllent/mailpit:latest + has sandbox && container dovecot -p "127.0.0.1:$IMAPS:31993" -e 'USER_PASSWORD={PLAIN}sandbox' dovecot/dovecot:latest + wait_for redis "docker exec $(cname redis) redis-cli ping" 30 + wait_for mailpit "curl -fs localhost:$MAILPIT/api/v1/info" 30 + + say "starting backend on :$API (applies migrations on boot)" + make_cmd backend >"$RUN/backend.sh" + start backend "exec bash '$RUN/backend.sh'" + wait_for backend "curl -fs localhost:$API/health" 300 + + # Seed on an empty users table, not on a missing database: a failed first run leaves it empty. + if ! db_seeded; then + if has sandbox; then say "seeding the Sunrise Labs sandbox into $DB"; else say "seeding $DB (rich + full fixtures)"; fi + seed >"$RUN/seed.log" 2>&1 || { echo "seed failed:"; tail -30 "$RUN/seed.log"; exit 1; } + fi + + if has full; then + say "starting consumer, worker and realtime" + make_cmd consumer >"$RUN/consumer.sh" + start consumer "exec bash '$RUN/consumer.sh'" + make_cmd worker >"$RUN/worker.sh" + start worker "exec bash '$RUN/worker.sh'" + check_image "$REALTIME_IMAGE" realtime/ + container realtime -p "127.0.0.1:$RT:4000" -e PORT=4000 -e PHX_HOST=localhost -e APP_ENV=dev \ + -e "DATABASE_URL=postgres://warmbly:warmbly@host.docker.internal:15432/$DB?sslmode=disable" -e DATABASE_SSL=false \ + -e "REDIS_URL=redis://$(cname redis):6379" -e "JWT_SECRET=$AUTH_SECRET" -e PUBSUB_ENABLED=false \ + -e SECRET_KEY_BASE=local-development-secret-key-base-minimum-64-characters-for-phoenix -e CHECK_ORIGIN=false \ + "$REALTIME_IMAGE" + fi + if has sandbox; then + check_image "$TRACKING_IMAGE" tracking/ + container tracking -p "127.0.0.1:$TRACK:3000" -e APP_ENV=dev -e AWS_CONFIG_ENABLED=false -e TRACKING_HOST=0.0.0.0 \ + -e TRACKING_PORT=3000 -e EVENTBUS_PROVIDER=nats -e "NATS_URL=nats://$(cname nats):4222" -e CODEC_PROVIDER=json \ + -e "BACKEND_INTERNAL_URL=http://host.docker.internal:$API" -e "INTERNAL_API_TOKEN=$INTERNAL_TOKEN" "$TRACKING_IMAGE" + say "starting the simulator (delivers, opens, clicks and replies like the internet)" + make_cmd sandbox-simulate | sed "s#$BIN/sandbox#$(sandbox_env) $BIN/sandbox#" >"$RUN/simulator.sh" + start simulator "exec bash '$RUN/simulator.sh'" + fi + + say "starting dashboard on :$WEB" + start web "cd web && export VITE_APP_URL=http://localhost:$WEB VITE_API_URL=http://localhost:$API VITE_TURNSTILE_KEY=1x00000000000000000000AA VITE_TURNSTILE_BYPASS_TOKEN=warmbly-local-turnstile-bypass; exec pnpm dev --port $WEB --strictPort" + wait_for web "curl -fs localhost:$WEB" 120 + if has full; then wait_for realtime "curl -fs localhost:$RT/health" 90; fi + if has sandbox; then wait_for tracking "curl -fs localhost:$TRACK/health" 60; fi + + write_env + start_watchdog + status +} + +down() { + local n + for n in watchdog simulator web worker consumer backend; do stop_proc "$n"; done + docker ps -aq --filter "label=$LABEL" | xargs -r docker rm -f >/dev/null + docker network rm "$NET" >/dev/null 2>&1 || true + rm -f "$RUN/env.json" + say "stopped (data kept in $DB)" +} + +rss_mb() { ps -o rss= -g "$(cat "$RUN/$1.pid")" 2>/dev/null | awk '{s+=$1} END {printf "%d", s/1024}'; } + +status() { + local n mb total=0 name mem ids idle=off + load_ports + MODE=$(cat "$RUN/mode" 2>/dev/null || echo lite) + printf ' mode %s\n\n' "$MODE" + for n in backend web consumer worker simulator; do + case $n in consumer | worker) has full || continue ;; simulator) has sandbox || continue ;; esac + if alive "$n"; then + mb=$(rss_mb "$n"); total=$((total + mb)) + printf ' %-10s up %5s MB\n' "$n" "$mb" + else + printf ' %-10s down\n' "$n" + fi + done + ids=$(docker ps -q --filter "label=$LABEL") + if [ -n "$ids" ]; then + # shellcheck disable=SC2086 + while read -r name mem; do + mb=$(awk -v m="$mem" 'BEGIN { v = m + 0; if (m ~ /GiB/) v *= 1024; else if (m ~ /KiB/) v /= 1024; printf "%d", v }') + total=$((total + mb)) + printf ' %-10s up %5s MB container\n' "${name#"wqa-$slug-"}" "$mb" + done < <(docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' $ids | awk '{print $1, $2}') + fi + [ "$IDLE_MIN" != 0 ] && idle="after $IDLE_MIN min unused" + cat </dev/null || echo lite) + down + up "$mode" + ;; + status) status ;; + logs) if [ -n "${2:-}" ]; then tail -f "$RUN/$2.log"; else tail -f "$RUN"/*.log; fi ;; + reset-data) + target=${2:-$(cat "$RUN/mode" 2>/dev/null || echo lite)} + down + psql_ -d postgres -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" >/dev/null + rm -rf "$QA/.artifacts/auth" "$RUN/blobs" "$RUN/mode" + say "dropped $DB" + up "$target" + ;; + ls) ls_stacks ;; + *) sed -n '2,20p' "$0"; exit 1 ;; +esac diff --git a/qa/tsconfig.json b/qa/tsconfig.json new file mode 100644 index 000000000..8a99542ec --- /dev/null +++ b/qa/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["lib", "flows", "scripts", "playwright.config.ts"] +}