mirror of
https://github.com/warmbly/warmbly.git
synced 2026-10-05 00:02:12 +00:00
feat: add the qa/ proof harness that records scripted Playwright flows as 1080p H.264 walkthroughs and stills against an isolated per-worktree stack (lite, full, sandbox) and publishes them to the PR with gh --attach, with before/after follow-up comments, a machine-wide recording lock, idle auto-stop, a qa-ci typecheck job and AGENTS.md rules for when to record
This commit is contained in:
Matthew Meszaros
@@ -34,6 +34,7 @@ jobs:
|
||||
admin: ${{ steps.filter.outputs.admin }}
|
||||
site: ${{ steps.filter.outputs.site }}
|
||||
forms: ${{ steps.filter.outputs.forms }}
|
||||
qa: ${{ steps.filter.outputs.qa }}
|
||||
make: ${{ steps.filter.outputs.make }}
|
||||
ios: ${{ steps.filter.outputs.ios }}
|
||||
installer: ${{ steps.filter.outputs.installer }}
|
||||
@@ -80,6 +81,8 @@ jobs:
|
||||
- 'web/src/components/app/forms/designCore.ts'
|
||||
- 'web/src/components/app/forms/form-theme.css'
|
||||
- 'scripts/check-forms-mirror.sh'
|
||||
qa:
|
||||
- 'qa/**'
|
||||
make:
|
||||
- 'integrations/make/**'
|
||||
ios:
|
||||
@@ -186,6 +189,29 @@ jobs:
|
||||
- name: Build
|
||||
run: pnpm build
|
||||
|
||||
# The proof harness records against a live local stack, so CI only checks that it compiles and its flows load.
|
||||
qa-ci:
|
||||
name: QA harness CI
|
||||
needs: changes
|
||||
if: needs.changes.outputs.qa == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: qa
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
working-directory: qa
|
||||
node-version: "24"
|
||||
|
||||
- name: Typecheck
|
||||
run: pnpm typecheck
|
||||
|
||||
- name: Flows load
|
||||
run: pnpm exec playwright test --list
|
||||
|
||||
admin-ci:
|
||||
name: Admin CI
|
||||
needs: changes
|
||||
@@ -490,7 +516,7 @@ jobs:
|
||||
ci-status:
|
||||
name: CI Status
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images]
|
||||
needs: [changes, migrations-ci, go-ci, web-ci, qa-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Check CI status
|
||||
|
||||
@@ -32,7 +32,7 @@ CI is strict. `go build ./...` succeeding is not enough — `golangci-lint` runs
|
||||
|
||||
Other CI-touching rules:
|
||||
|
||||
- the frontend trees (`admin/`, `web/`, `site/`) each have their own CI jobs; run `pnpm typecheck` in any tree you touched and `pnpm lint` when the rules are non-trivial
|
||||
- the frontend trees (`admin/`, `web/`, `site/`, `qa/`) each have their own CI jobs; run `pnpm typecheck` in any tree you touched and `pnpm lint` when the rules are non-trivial
|
||||
- never push without first re-running the relevant `*build*` / `*typecheck*` / `*lint*` step on the affected tree
|
||||
- a `make lint` (or `gofmt -l`) failure is always a real CI failure; do not push hoping it will pass
|
||||
|
||||
@@ -173,10 +173,24 @@ is `make cli-check` (and `make cli-sha` after any edit).
|
||||
Do not:
|
||||
|
||||
- do not run `go build ./...`, `pnpm build`, or docker image builds as a "did it work" check. They are slow and are not what CI gates on. `go run` (via the make dev targets) already compiles; `make fmt` + `make lint` + `pnpm typecheck` are the real signals.
|
||||
- do not write or run Python/Playwright (or any browser-automation) scripts to test the app. Manual, in-browser verification is the user's job against the native dev stack (`make infra` + `make backend` + `make web`). Do not add screenshot/e2e test harnesses to this repo.
|
||||
- do not write Python or ad-hoc browser scripts, and do not drive the app step by step from screenshots or accessibility snapshots. Browser automation lives in `qa/` and only records proof for a PR (below); it is not a test gate, and manual verification against the native dev stack stays the user's job.
|
||||
- do not run the Go test suite as a default gate unless the task is specifically about those tests.
|
||||
- do not push hoping CI passes; a `gofmt -l` / `make lint` / `pnpm typecheck` failure is always a real CI failure.
|
||||
|
||||
### Visual proof on pull requests
|
||||
|
||||
Record proof only for a UI change a reviewer should actually see; most changes need none. `qa/` is the harness and `qa/README.md` is its playbook; read it before the first recording.
|
||||
|
||||
- record for: a new page, dialog, drawer or multi-step flow; a redesigned or re-laid-out screen; a changed interaction (new controls, new states, a different path through a task); a visible bug fix where the before was visibly broken; or when the user asks for it
|
||||
- skip for: anything without a visible change (backend, API, migrations, workers, tests, CI, docs, refactors, dependency bumps) and small visual tweaks (copy, a label, an icon, spacing or colour nudges), which the PR text describes instead. When unsure, skip
|
||||
- a follow-up commit is re-recorded only when it changes what the proof shows
|
||||
|
||||
- write a scripted flow in `qa/flows/` from the code you changed, run `pnpm proof <filter>` in `qa/`, and read only pass or the failing step. Never screenshot your way around the page; `pnpm aria <path>` prints the accessibility tree when you need a selector
|
||||
- record against this worktree's own stack (`pnpm stack up`, in the smallest of `lite`, `full`, `sandbox` the flow needs), never another session's and never anything but localhost: the repository is public and so is everything attached to its PRs
|
||||
- publish with `pnpm share` once the PR exists. The first publish goes into the PR description; every later one is a comment showing before/after of only the stills that changed. Record follow-ups with `pnpm proof:fresh` so the comparison starts from fresh seed data. Media reaches GitHub only through `gh --attach`, never an image host, a commit or another repository
|
||||
- keep shot names stable across commits, because follow-up comments diff stills by name
|
||||
- the machine is shared: record from the main agent, never from fanned-out subagents (a machine-wide lock runs one recording at a time, so they would only queue while each holds a context), and `pnpm stack down` once the proof is published
|
||||
|
||||
## Security And Compliance Invariants
|
||||
|
||||
Warmbly's Google OAuth client is assessed against **ADA CASA v2.1.1 at Assurance Level 1**, which maps to OWASP ASVS 4.0.3. The evidence pack is a claim about the code on `main`: a change that breaks one of the invariants below does not just introduce a bug, it makes a submitted statement untrue and puts the OAuth client's verification at risk. Treat them as constraints on every change, not as a checklist run before an audit.
|
||||
@@ -361,6 +375,7 @@ API keys with the `REALTIME_SUBSCRIBE` permission (bit 11) can connect to the sa
|
||||
- `realtime/`: websocket fanout service
|
||||
- `web/`: in-product frontend (dashboard). Customer-facing only: it holds no platform-admin screens, and operator tooling must not be added back here
|
||||
- `admin/`: platform admin panel (:5174), the single operator surface. Workers, users, orgs, warmup, campaigns, analytics, audit. Every route sits behind `RequireAdmin` and the backend's `RequireAdminPermission` gates
|
||||
- `qa/`: the proof harness (scripted Playwright flows recording 1080p walkthroughs and stills for pull requests, published with `gh --attach`). Its own package; see `qa/README.md`
|
||||
- `site/`: public marketing site (Astro 5 + Tailwind v4). `site/public/install.sh` is the self-host installer served at warmbly.com/install.sh and `site/public/cli.sh` is the CLI installer served at warmbly.com/cli.sh (with `cli.ps1` for Windows), each with its checksum next to it; see the rules above before touching either
|
||||
- `deploy/`: production deploy manifests, infrastructure, and runtime config. `deploy/split-cloud/` is the three-provider shape (control plane on a container host, bus + cache + fleet on machines you own, database + root key + object store in a cloud region), documented at `docs/content/docs/development/split-deployment.mdx`
|
||||
- `docs/`: documentation site (docs.warmbly.com); product guides, API reference, and self-hosting/engineering docs under `content/docs/development/`
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules/
|
||||
# Recordings, stills, saved sessions and the per-worktree stack state. Never committed.
|
||||
.artifacts/
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
# Proof harness
|
||||
|
||||
Recorded proof for pull requests. A flow is a short scripted Playwright walk
|
||||
through the dashboard. Running it produces a 1920x1080 H.264 walkthrough and
|
||||
named full-resolution stills, and `pnpm share` puts them on the PR:
|
||||
|
||||
- **first publish**: a Proof section appended to the PR description, stills
|
||||
inline, walkthrough videos last
|
||||
- **every publish after that**: a comment with before/after pairs for only the
|
||||
stills that changed since the previous publish, the commits in between, and
|
||||
the videos of the flows that changed. Nothing changed means nothing is posted
|
||||
|
||||
It is not a test gate and CI does not run it (CI only typechecks it and checks
|
||||
the flows load). It exists so a reviewer can see a change without checking it
|
||||
out.
|
||||
|
||||
## When to record
|
||||
|
||||
Only for a UI change worth watching. Most PRs need no proof.
|
||||
|
||||
- **record**: a new page, dialog, drawer or multi-step flow; a redesigned or
|
||||
re-laid-out screen; a changed interaction (new controls, new states, a
|
||||
different path through a task); a visible bug fix where the before was
|
||||
visibly broken; or when the user asks for it
|
||||
- **skip**: anything without a visible change (backend, API, migrations,
|
||||
workers, tests, CI, docs, refactors, dependency bumps) and small visual
|
||||
tweaks (copy, a label, an icon, spacing or colour nudges). Describe those in
|
||||
the PR text. When unsure, skip
|
||||
- **follow-ups**: re-record only when the commit changes what the proof shows
|
||||
|
||||
## Why scripted
|
||||
|
||||
The model never looks at the page while the browser runs. You write the flow
|
||||
from the code you just changed (you know the routes, labels and fields), run
|
||||
it, and read one line: pass, or the failing step. A 15-step flow costs a few
|
||||
thousand tokens to write and nothing to re-run. Driving a browser step by step
|
||||
from screenshots or snapshots costs that much on every step of every run, and
|
||||
leaves dead air in the video while the model thinks.
|
||||
|
||||
## Be frugal with the machine
|
||||
|
||||
Several agents share this machine, and every stack and browser holds real
|
||||
memory. The harness enforces the expensive parts; the rest is on you.
|
||||
|
||||
- **one recording at a time, machine-wide.** `pnpm proof` (and `pnpm aria`)
|
||||
take a lock in the system temp directory; a second one waits for the first
|
||||
instead of starting its own Chromium. Never fan recording out to subagents:
|
||||
they would only queue behind each other while each holds a context
|
||||
- **one stack per worktree**, in the smallest mode the flow needs. `lite` is
|
||||
enough for anything that only reads or edits records
|
||||
- **stop it when you are done**: `pnpm stack down` after `pnpm share`. A stack
|
||||
nobody records against stops itself after 60 minutes (`QA_STACK_IDLE_MIN`),
|
||||
but do not lean on that
|
||||
- the browser and the encoder never run at once: frames are spooled to disk
|
||||
during the run and encoded after the browser exits, one flow at a time, with
|
||||
bounded encoder threads
|
||||
|
||||
Measured on this repo's dashboard:
|
||||
|
||||
| | memory |
|
||||
| --- | --- |
|
||||
| `lite` stack (backend, dashboard, Redis, NATS, Mailpit) | ~300 MB idle, ~500 MB once used |
|
||||
| `full` stack (+ consumer, worker, realtime) | ~560 MB idle |
|
||||
| `sandbox` stack (+ tracking, Dovecot, simulator) | ~760 MB idle |
|
||||
| Chromium during a recording (proportional set size) | ~800 MB |
|
||||
| ffmpeg encoding, after Chromium has exited | ~580 MB |
|
||||
|
||||
## One-time setup
|
||||
|
||||
```bash
|
||||
make infra # repo root: the shared postgres (each stack gets its own database in it)
|
||||
cd qa && pnpm bootstrap # deps, Playwright's Chromium, the small stack images, then `pnpm check`
|
||||
```
|
||||
|
||||
`pnpm check` verifies node 22.18+, ffmpeg with libx264, Chromium (and that it
|
||||
launches), gh 2.99+ and signed in, docker, the shared postgres, the images each
|
||||
mode needs, and this worktree's stack. Each failure prints its fix.
|
||||
|
||||
## The loop
|
||||
|
||||
```bash
|
||||
pnpm stack up # this worktree's own stack (lite unless told otherwise)
|
||||
pnpm proof contacts # run the flows whose file or title matches
|
||||
git push && gh pr create ... # the PR has to exist first
|
||||
pnpm share # publish the last run to this branch's PR
|
||||
pnpm stack down # free the memory
|
||||
```
|
||||
|
||||
After a follow-up commit: push, record the flows your change touched again,
|
||||
and `pnpm share`. It posts a comment, never a second description section.
|
||||
Record follow-ups with `pnpm proof:fresh <filter>`, which re-seeds the stack
|
||||
(about 15 seconds) before recording, so the comparison shows what the commit
|
||||
changed and not what background jobs did to the data in the meantime.
|
||||
|
||||
`pnpm share --dry-run` prints the body and the gh command without uploading
|
||||
anything, and works before the PR exists. `--no-video` posts stills only,
|
||||
`--force` comments even when no still changed, `--replace-body` rewrites the
|
||||
description's Proof section instead of commenting (for a re-record nobody has
|
||||
reviewed yet), `--allow-failed` publishes a run with failing flows.
|
||||
|
||||
`share` refuses a run with failing flows and a run recorded at a different
|
||||
commit than HEAD, and warns when the tree was dirty or HEAD is not pushed.
|
||||
|
||||
## The stack
|
||||
|
||||
`pnpm stack up [lite|full|sandbox]` runs this worktree's Warmbly natively
|
||||
against the shared postgres, with everything else private to it: its own
|
||||
database (`warmbly_qa_<worktree>`), Redis, NATS and Mailpit, on ports picked
|
||||
once per worktree and remembered in `.artifacts/stack/ports`. Nothing crosses
|
||||
between stacks: no cached profile, no worker event, no login code. The Go
|
||||
services run from binaries built once per `up` (`go build`, cached), not from
|
||||
one `go run` toolchain per service. Linux and WSL.
|
||||
|
||||
| Mode | Runs | Seed and account |
|
||||
| --- | --- | --- |
|
||||
| `lite` | backend, dashboard | rich seed (`SEED_RICH` + `SEED_FULL`), `dev@warmbly.com` |
|
||||
| `full` | lite + consumer, worker, realtime | same seed; sends, syncs and live updates work |
|
||||
| `sandbox` | full + tracking, Dovecot, simulator | Sunrise Labs (`sandbox@warmbly.test`): live mailboxes, campaign mail, opens, clicks, replies |
|
||||
|
||||
`lite` and `full` share a seed and switch freely. The sandbox is a different
|
||||
organization, so moving to or from it needs `pnpm stack reset-data <mode>`;
|
||||
`up` refuses rather than mix them. In the sandbox the simulator keeps changing
|
||||
data while you record, so its stills differ from run to run by design.
|
||||
|
||||
Other commands: `status` (what runs, ports, memory), `logs [name]`, `restart`
|
||||
(rebuilds the binaries; the dashboard hot-reloads on its own), `reset-data
|
||||
[mode]` (drop the database, fresh Redis, reseed), `ls` (every QA stack on the
|
||||
machine), `down`.
|
||||
|
||||
Realtime and tracking run from the local `ghcr.io/warmbly/warmbly/*:prod`
|
||||
images. `up` warns when one was built before the last commit to its source,
|
||||
with the command that rebuilds it.
|
||||
|
||||
Sign-in happens once per stack through the API and the login code in the
|
||||
stack's Mailpit; the first-run wizard is completed through its own endpoint,
|
||||
and the session is saved under `.artifacts/auth`. A flow that lands on the
|
||||
sign-in page, onboarding or the workspace picker fails and says why.
|
||||
|
||||
Recording against a stack the harness did not start is possible but explicit:
|
||||
`QA_WEB_URL` and `QA_API_URL` (and `QA_MAILPIT_URL`, `QA_EMAIL`,
|
||||
`QA_PASSWORD`). Without either, `pnpm proof` refuses rather than record
|
||||
whatever happens to answer on the default ports, which is usually another
|
||||
session's stack.
|
||||
|
||||
## Writing a flow
|
||||
|
||||
Flows live in `flows/<area>.flow.ts`, one `test` per walkthrough. Extend the
|
||||
area's file when your change lands in it; add one when it does not exist.
|
||||
|
||||
```ts
|
||||
import { expect, test } from "../lib/proof.ts";
|
||||
|
||||
test("search contacts and open a contact's details", async ({ page, proof }) => {
|
||||
await page.goto("/app/contacts");
|
||||
await expect(page.getByRole("table")).toBeVisible();
|
||||
await proof.chapter("Contacts", "Search the list, then open one contact");
|
||||
|
||||
await page.getByRole("textbox", { name: /Search by name/ }).pressSequentially("beth", { delay: 60 });
|
||||
await expect(page.getByRole("row", { name: /Beth Chen/ })).toBeVisible();
|
||||
await proof.shot("search-results", { caption: "Search narrowed to one contact" });
|
||||
});
|
||||
```
|
||||
|
||||
- `proof.chapter(title, description?)` shows a title card in the video
|
||||
- `proof.shot(name, { caption?, target?, ignore? })` saves a still. The name is
|
||||
kebab-case and is the key follow-up comments diff by, so keep it stable
|
||||
across commits. `target` shoots one element. `ignore` takes locators whose
|
||||
content changes on its own (relative times, live counters): the still is
|
||||
published untouched, and those regions are skipped when it is compared with
|
||||
the previous publish
|
||||
- `proof.dwell(ms?)` holds a result on screen long enough to read
|
||||
- `test.use({ seed: "sandbox" })` for a flow written against the Sunrise Labs
|
||||
data. A flow is skipped, with the command that fixes it, when the stack holds
|
||||
the other seed
|
||||
- `test.use({ signedIn: false })` for a flow that starts on the sign-in pages
|
||||
|
||||
Conventions:
|
||||
|
||||
- `expect(...)` the content before every `chapter` and `shot`, so neither lands
|
||||
on a loading state
|
||||
- locate by role and accessible name (`getByRole`, `getByLabel`, `getByText`);
|
||||
no CSS classes
|
||||
- `pressSequentially(text, { delay: 60 })` where typing should be visible,
|
||||
`fill` where it should not
|
||||
- no `waitForTimeout` except through `dwell`
|
||||
- keep a walkthrough under about 30 seconds; split longer stories into tests
|
||||
- the title reads as a sentence about what the reviewer sees; it heads the PR
|
||||
section
|
||||
|
||||
When you need a selector you cannot read off the code, print the page's
|
||||
accessibility tree with the saved session instead of opening a browser:
|
||||
|
||||
```bash
|
||||
pnpm aria /app/contacts # whole page
|
||||
pnpm aria /app/contacts main # one region, fewer tokens
|
||||
```
|
||||
|
||||
Do not read the video or screenshot your way around the page. Read a still
|
||||
only when the change is a visual judgment (spacing, colour, layout) that the
|
||||
assertions cannot make.
|
||||
|
||||
## When a flow fails
|
||||
|
||||
The list reporter prints the failing step and its locator. Failures keep a
|
||||
Playwright trace (DOM snapshots, network, console) and the reporter prints its
|
||||
path; open it with `pnpm exec playwright show-trace <trace.zip>` when the error
|
||||
line is not enough. A failed flow's frames are dropped instead of encoded
|
||||
(`QA_VIDEO_ON_FAIL=1` keeps them).
|
||||
|
||||
## Recording settings
|
||||
|
||||
1920x1080 at 30 fps, H.264 CRF 18 (`slow` preset), yuv420p so every browser's
|
||||
player shows it. A clip over 10 MB is re-encoded smaller. Frames come from
|
||||
Playwright's screencast at full size and are encoded here, because Playwright's
|
||||
built-in `video` option is VP8 at 1 Mbit/s and smears text. The video starts at
|
||||
the first painted frame, not on the blank page before the app renders. The
|
||||
animated cursor, click ripples and action labels are Playwright's
|
||||
`screencast.showActions`. Trace screenshots stay off: they would start a
|
||||
smaller screencast first, and the recorder refuses frames below 1080p.
|
||||
|
||||
| Variable | Default | |
|
||||
| --- | --- | --- |
|
||||
| `QA_FPS` | `30` | frame rate |
|
||||
| `QA_CRF` | `18` | quality, lower is better and bigger |
|
||||
| `QA_PRESET` | `slow` | x264 preset |
|
||||
| `QA_ENCODE_THREADS` | `4` | encoder threads, which also bound its memory |
|
||||
| `QA_SLOWMO` | `250` | ms between actions, so a viewer can follow |
|
||||
| `QA_MAX_VIDEO_MB` | `10` | re-encode above this |
|
||||
| `QA_VIDEO_ON_FAIL` | | `1` encodes failed flows too |
|
||||
| `QA_STACK_IDLE_MIN` | `60` | stop an unused stack after this long; `0` never |
|
||||
| `QA_LOCK_WAIT_MIN` | `30` | how long a recording waits for another to finish |
|
||||
| `QA_WEB_URL`, `QA_API_URL`, `QA_MAILPIT_URL` | this worktree's stack | record a stack the harness did not start |
|
||||
| `QA_EMAIL`, `QA_PASSWORD` | the mode's seeded account | account to sign in as |
|
||||
| `QA_ALLOW_HOST` | | extra hostnames allowed besides localhost |
|
||||
|
||||
## Safety
|
||||
|
||||
The repository is public and so is everything attached to its pull requests.
|
||||
The harness refuses to record anything but localhost (or a host named in
|
||||
`QA_ALLOW_HOST`), so recordings only ever show seed data. Never point it at
|
||||
production or a customer workspace, and never attach media to a PR any other
|
||||
way: no image hosts, no commits of screenshots, no other repositories.
|
||||
Everything the harness writes stays under the ignored `.artifacts/`.
|
||||
@@ -0,0 +1,19 @@
|
||||
import { expect, test } from "../lib/proof.ts";
|
||||
|
||||
test("search contacts and open a contact's details", async ({ page, proof }) => {
|
||||
await page.goto("/app/contacts");
|
||||
await expect(page.getByRole("table")).toBeVisible();
|
||||
await proof.chapter("Contacts", "Search the list, then open one contact");
|
||||
|
||||
await page.getByRole("textbox", { name: /Search by name/ }).pressSequentially("beth", { delay: 60 });
|
||||
const row = page.getByRole("row", { name: /Beth Chen/ });
|
||||
await expect(row).toBeVisible();
|
||||
await expect(page.getByRole("row", { name: /Carlos Diaz/ })).toBeHidden();
|
||||
await proof.shot("search-results", { caption: "Search narrowed to one contact" });
|
||||
|
||||
await row.click();
|
||||
await expect(page.getByText("beth.chen@initech.test").last()).toBeVisible();
|
||||
await proof.dwell();
|
||||
// "Checked ... 4m ago" drifts between runs; ignored so it never reads as a change.
|
||||
await proof.shot("contact-details", { caption: "Contact details drawer", ignore: [page.getByText(/\d+[smhd] ago/)] });
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
import { expect, test } from "../lib/proof.ts";
|
||||
|
||||
test("dashboard opens on the mailboxes page", async ({ page, proof }) => {
|
||||
await page.goto("/app/emails");
|
||||
await expect(page.getByRole("row", { name: /dev\.outbound@warmbly\.test/ })).toBeVisible();
|
||||
await proof.chapter("Mailboxes", "The signed-in dashboard landing page");
|
||||
await proof.shot("mailboxes", { caption: "Mailboxes list" });
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { expect, test } from "../lib/proof.ts";
|
||||
|
||||
test.use({ seed: "sandbox" });
|
||||
|
||||
test("open a reply in the unified inbox", async ({ page, proof }) => {
|
||||
await page.goto("/app/unibox");
|
||||
const thread = page.getByRole("button", { name: /^Hana Jules .*Wayne Enterprises/ });
|
||||
await expect(thread).toBeVisible();
|
||||
// Relative times ("9h", "1d") move on their own; ignored so a re-run tomorrow is not a change.
|
||||
const times = page.getByText(/^\d+[mhdw]$/);
|
||||
await proof.chapter("Unified inbox", "Replies from every mailbox in one list");
|
||||
await proof.shot("inbox-list", { caption: "Inbox with replies across mailboxes", ignore: [times] });
|
||||
|
||||
await thread.click();
|
||||
await expect(page.getByText("Thursday at 2pm works for a demo").last()).toBeVisible();
|
||||
await proof.dwell();
|
||||
await proof.shot("conversation", { caption: "The reply opened in the conversation pane", ignore: [times] });
|
||||
});
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { authFile, env } from "./env.ts";
|
||||
|
||||
type Tokens = {
|
||||
access_token: string;
|
||||
access_token_expires_at: string;
|
||||
refresh_token: string;
|
||||
refresh_token_expires_at: string;
|
||||
};
|
||||
|
||||
type StorageState = {
|
||||
cookies: [];
|
||||
origins: { origin: string; localStorage: { name: string; value: string }[] }[];
|
||||
};
|
||||
|
||||
const TOKEN_KEYS = ["access_token", "access_token_expires_at", "refresh_token", "refresh_token_expires_at"] as const;
|
||||
|
||||
// Reuses the saved session while it still works: every fresh login sends a code email, and those are budgeted.
|
||||
export async function ensureSession(): Promise<string> {
|
||||
const file = authFile();
|
||||
const saved = readTokens(file);
|
||||
if (saved && (await sessionWorks(saved))) {
|
||||
await finishOnboarding(saved.access_token);
|
||||
return file;
|
||||
}
|
||||
|
||||
const tokens = await login();
|
||||
await finishOnboarding(tokens.access_token);
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
const state: StorageState = {
|
||||
cookies: [],
|
||||
origins: [
|
||||
{
|
||||
origin: env.webURL,
|
||||
// The dashboard reads `auth_token` (one JSON object) and keeps the four flat keys alongside it.
|
||||
localStorage: [
|
||||
{ name: "auth_token", value: JSON.stringify(Object.fromEntries(TOKEN_KEYS.map((k) => [k, tokens[k]]))) },
|
||||
...TOKEN_KEYS.map((k) => ({ name: k, value: tokens[k] })),
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
writeFileSync(file, JSON.stringify(state, null, 2));
|
||||
return file;
|
||||
}
|
||||
|
||||
function readTokens(file: string): Tokens | undefined {
|
||||
if (!existsSync(file)) return undefined;
|
||||
const state = JSON.parse(readFileSync(file, "utf8")) as StorageState;
|
||||
const items = state.origins.find((o) => o.origin === env.webURL)?.localStorage ?? [];
|
||||
const get = (k: string) => items.find((i) => i.name === k)?.value ?? "";
|
||||
const tokens = Object.fromEntries(TOKEN_KEYS.map((k) => [k, get(k)])) as Tokens;
|
||||
if (!tokens.refresh_token || Date.parse(tokens.refresh_token_expires_at) < Date.now() + 3600_000) return undefined;
|
||||
return tokens;
|
||||
}
|
||||
|
||||
async function sessionWorks(tokens: Tokens): Promise<boolean> {
|
||||
if (Date.parse(tokens.access_token_expires_at) > Date.now() + 60_000) {
|
||||
const res = await fetch(`${env.apiURL}/v1/auth/me`, { headers: { Authorization: `Bearer ${tokens.access_token}` } });
|
||||
return res.ok;
|
||||
}
|
||||
// An expired access token is fine: the dashboard refreshes it on its first request.
|
||||
return true;
|
||||
}
|
||||
|
||||
async function call<T>(method: string, path: string, body?: object, token?: string): Promise<T> {
|
||||
const res = await fetch(`${env.apiURL}/v1${path}`, {
|
||||
method,
|
||||
headers: { "Content-Type": "application/json", ...(token ? { Authorization: `Bearer ${token}` } : {}) },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
const json = (await res.json().catch(() => ({}))) as T & { error?: string; code?: string };
|
||||
if (!res.ok) throw new Error(`${method} ${path} answered ${res.status}: ${json.code ?? ""} ${json.error ?? ""}`.trim());
|
||||
return json;
|
||||
}
|
||||
|
||||
function post<T>(path: string, body: object): Promise<T> {
|
||||
return call<T>("POST", path, body);
|
||||
}
|
||||
|
||||
// Seeded accounts have not been through the first-run wizard, and the dashboard sends them there
|
||||
// before anything else. Answer it once through the same endpoint the wizard uses.
|
||||
async function finishOnboarding(token: string): Promise<void> {
|
||||
if (!token) return;
|
||||
const me = await call<{ first_name?: string; last_name?: string; onboarding_completed_at?: string | null }>("GET", "/auth/me", undefined, token).catch(() => undefined);
|
||||
if (!me || me.onboarding_completed_at) return;
|
||||
await call("PATCH", "/auth/me/onboarding", {
|
||||
first_name: me.first_name || "Dev",
|
||||
last_name: me.last_name || "User",
|
||||
referral_source: "other",
|
||||
}, token);
|
||||
}
|
||||
|
||||
async function login(): Promise<Tokens> {
|
||||
const startedAt = Date.now();
|
||||
const start = await post<{ session?: string; code_required: boolean; token?: Tokens; two_fa_required?: boolean }>(
|
||||
"/auth/login",
|
||||
{ email: env.email, password: env.password, turnstile: env.turnstile },
|
||||
);
|
||||
if (start.token) return start.token;
|
||||
if (start.two_fa_required) throw new Error(`${env.email} has 2FA enabled; use an account without it for QA`);
|
||||
if (!start.session) throw new Error("login answered with neither a token nor a code session");
|
||||
|
||||
const code = await loginCode(startedAt);
|
||||
const confirm = await post<Partial<Tokens> & { two_fa_required?: boolean }>("/auth/login/confirm", {
|
||||
session: start.session,
|
||||
code,
|
||||
turnstile: env.turnstile,
|
||||
});
|
||||
if (confirm.two_fa_required) throw new Error(`${env.email} has 2FA enabled; use an account without it for QA`);
|
||||
if (!confirm.access_token) throw new Error("login confirm returned no token");
|
||||
return confirm as Tokens;
|
||||
}
|
||||
|
||||
type MailpitSummary = { ID: string; Created: string };
|
||||
|
||||
async function loginCode(since: number): Promise<string> {
|
||||
const query = encodeURIComponent(`to:"${env.email}" subject:"Your Login Code"`);
|
||||
for (let i = 0; i < 40; i++) {
|
||||
const res = await fetch(`${env.mailpitURL}/api/v1/search?query=${query}&limit=5`);
|
||||
if (res.ok) {
|
||||
const { messages } = (await res.json()) as { messages: MailpitSummary[] };
|
||||
const fresh = messages.find((m) => Date.parse(m.Created) >= since - 2000);
|
||||
if (fresh) {
|
||||
const msg = (await (await fetch(`${env.mailpitURL}/api/v1/message/${fresh.ID}`)).json()) as { Text: string };
|
||||
const code = msg.Text.match(/\b(\d{6})\b/)?.[1];
|
||||
if (code) return code;
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
throw new Error(`no login code for ${env.email} reached Mailpit at ${env.mailpitURL}`);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const QA_DIR = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
export const ARTIFACTS = join(QA_DIR, ".artifacts");
|
||||
export const RUN_DIR = join(ARTIFACTS, "run");
|
||||
export const AUTH_DIR = join(ARTIFACTS, "auth");
|
||||
export const STACK_DIR = join(ARTIFACTS, "stack");
|
||||
|
||||
export const VIDEO = {
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
fps: Number(process.env.QA_FPS ?? 30),
|
||||
crf: Number(process.env.QA_CRF ?? 18),
|
||||
preset: process.env.QA_PRESET ?? "slow",
|
||||
};
|
||||
|
||||
type StackEnv = { webURL?: string; apiURL?: string; mailpitURL?: string; email?: string; password?: string; mode?: string };
|
||||
|
||||
function stackEnv(): StackEnv {
|
||||
const file = join(STACK_DIR, "env.json");
|
||||
if (!existsSync(file)) return {};
|
||||
return JSON.parse(readFileSync(file, "utf8")) as StackEnv;
|
||||
}
|
||||
|
||||
const stack = stackEnv();
|
||||
|
||||
export const env = {
|
||||
webURL: trimSlash(process.env.QA_WEB_URL ?? stack.webURL ?? "http://localhost:5173"),
|
||||
apiURL: trimSlash(process.env.QA_API_URL ?? stack.apiURL ?? "http://localhost:8080"),
|
||||
mailpitURL: trimSlash(process.env.QA_MAILPIT_URL ?? stack.mailpitURL ?? "http://localhost:18025"),
|
||||
email: process.env.QA_EMAIL ?? stack.email ?? "dev@warmbly.com",
|
||||
password: process.env.QA_PASSWORD ?? stack.password ?? "password123",
|
||||
// "external": QA_WEB_URL names a stack the harness did not start. "none": nothing to record.
|
||||
mode: process.env.QA_WEB_URL ? "external" : (stack.mode ?? "none"),
|
||||
turnstile: "warmbly-local-turnstile-bypass",
|
||||
};
|
||||
|
||||
function trimSlash(url: string): string {
|
||||
return url.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
// The repo is public and recordings are uploaded to it, so only a local stack with seed data may be recorded.
|
||||
export function assertLocal(url: string): void {
|
||||
const host = new URL(url).hostname;
|
||||
const allowed = (process.env.QA_ALLOW_HOST ?? "").split(",").filter(Boolean);
|
||||
const local = ["localhost", "127.0.0.1", "[::1]", "::1"].includes(host) || host.endsWith(".localhost");
|
||||
if (!local && !allowed.includes(host)) {
|
||||
throw new Error(
|
||||
`refusing to record ${url}: recordings are published to a public repo, so only a local dev stack is allowed ` +
|
||||
`(set QA_ALLOW_HOST=${host} if this host is a dev machine running seed data)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The stack's idle watchdog stops it after a stretch with no recording; every use resets the clock.
|
||||
export function touchStack(): void {
|
||||
if (existsSync(STACK_DIR)) writeFileSync(join(STACK_DIR, "last-used"), new Date().toISOString());
|
||||
}
|
||||
|
||||
export function authFile(): string {
|
||||
return join(AUTH_DIR, `${new URL(env.webURL).host.replace(/[^a-z0-9]+/gi, "_")}.json`);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ensureSession } from "./auth.ts";
|
||||
import { ARTIFACTS, assertLocal, env, QA_DIR, RUN_DIR, touchStack } from "./env.ts";
|
||||
import { acquireRecordingLock, releaseRecordingLock } from "./lock.ts";
|
||||
|
||||
export type RunInfo = { sha: string; branch: string; dirty: boolean; webURL: string; startedAt: string };
|
||||
|
||||
async function reachable(url: string): Promise<boolean> {
|
||||
try {
|
||||
return (await fetch(url, { signal: AbortSignal.timeout(5000) })).ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function git(...args: string[]): string {
|
||||
return execFileSync("git", args, { cwd: QA_DIR, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
// Keeps only this run's traces: earlier runs' output directories belong to processes that have exited.
|
||||
function pruneOldResults(): void {
|
||||
const dir = join(ARTIFACTS, "test-results");
|
||||
if (!existsSync(dir)) return;
|
||||
for (const name of readdirSync(dir)) {
|
||||
const pid = Number(name.replace(/^run-/, ""));
|
||||
if (pid === process.pid) continue;
|
||||
let alive = false;
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
alive = true;
|
||||
} catch {
|
||||
// Exited.
|
||||
}
|
||||
if (!alive) rmSync(join(dir, name), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export default async function globalSetup(): Promise<void> {
|
||||
if (env.mode === "none") {
|
||||
throw new Error("this worktree has no stack running. Start one with `pnpm stack up` (in qa/), or set QA_WEB_URL and QA_API_URL to a local stack you started yourself.");
|
||||
}
|
||||
assertLocal(env.webURL);
|
||||
assertLocal(env.apiURL);
|
||||
|
||||
const down = [];
|
||||
if (!(await reachable(env.webURL))) down.push(`dashboard ${env.webURL}`);
|
||||
if (!(await reachable(`${env.apiURL}/health`))) down.push(`backend ${env.apiURL}`);
|
||||
if (!(await reachable(`${env.mailpitURL}/api/v1/info`))) down.push(`mailpit ${env.mailpitURL}`);
|
||||
if (down.length) {
|
||||
throw new Error(`not reachable: ${down.join(", ")}. Start this worktree's stack with \`pnpm stack up\` (in qa/), or run \`pnpm doctor\`.`);
|
||||
}
|
||||
|
||||
touchStack();
|
||||
await acquireRecordingLock(QA_DIR);
|
||||
try {
|
||||
rmSync(RUN_DIR, { recursive: true, force: true });
|
||||
mkdirSync(RUN_DIR, { recursive: true });
|
||||
pruneOldResults();
|
||||
await ensureSession();
|
||||
} catch (err) {
|
||||
releaseRecordingLock();
|
||||
throw err;
|
||||
}
|
||||
|
||||
const run: RunInfo = {
|
||||
sha: git("rev-parse", "HEAD"),
|
||||
branch: git("rev-parse", "--abbrev-ref", "HEAD"),
|
||||
dirty: git("status", "--porcelain").length > 0,
|
||||
webURL: env.webURL,
|
||||
startedAt: new Date().toISOString(),
|
||||
};
|
||||
writeFileSync(join(RUN_DIR, "run.json"), JSON.stringify(run, null, 2));
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { RUN_DIR } from "./env.ts";
|
||||
import { releaseRecordingLock } from "./lock.ts";
|
||||
import type { FlowMeta } from "./proof.ts";
|
||||
import { encodeFrames } from "./recorder.ts";
|
||||
|
||||
// Runs after the workers, and their browser, have exited: encodes each flow's frames one at a time.
|
||||
export default async function globalTeardown(): Promise<void> {
|
||||
try {
|
||||
if (!existsSync(RUN_DIR)) return;
|
||||
for (const entry of readdirSync(RUN_DIR, { withFileTypes: true })) {
|
||||
const metaFile = join(RUN_DIR, entry.name, "meta.json");
|
||||
if (!entry.isDirectory() || !existsSync(metaFile)) continue;
|
||||
const meta = JSON.parse(readFileSync(metaFile, "utf8")) as FlowMeta;
|
||||
if (!meta.frames) continue;
|
||||
const out = join(RUN_DIR, entry.name, "video.mp4");
|
||||
const started = Date.now();
|
||||
await encodeFrames(meta.frames, out);
|
||||
meta.video = out;
|
||||
meta.frames = undefined;
|
||||
writeFileSync(metaFile, JSON.stringify(meta, null, 2));
|
||||
console.log(`encoded ${entry.name}/video.mp4 in ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
||||
}
|
||||
} finally {
|
||||
releaseRecordingLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
// One recording at a time on this machine, whichever worktree or agent starts it:
|
||||
// each holds a Chromium and an encoder, and parallel ones only starve each other.
|
||||
const LOCK = join(tmpdir(), "warmbly-qa-recording.lock");
|
||||
const WAIT_MIN = Number(process.env.QA_LOCK_WAIT_MIN ?? 30);
|
||||
|
||||
type Holder = { pid: number; label: string; since: string };
|
||||
|
||||
function alive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return (err as NodeJS.ErrnoException).code === "EPERM";
|
||||
}
|
||||
}
|
||||
|
||||
function holder(): Holder | undefined {
|
||||
try {
|
||||
return JSON.parse(readFileSync(LOCK, "utf8")) as Holder;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function acquireRecordingLock(label: string): Promise<void> {
|
||||
const deadline = Date.now() + WAIT_MIN * 60_000;
|
||||
let announced = false;
|
||||
for (;;) {
|
||||
try {
|
||||
const fd = openSync(LOCK, "wx");
|
||||
writeSync(fd, JSON.stringify({ pid: process.pid, label, since: new Date().toISOString() } satisfies Holder));
|
||||
closeSync(fd);
|
||||
return;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
||||
}
|
||||
const h = holder();
|
||||
if (!h || !alive(h.pid)) {
|
||||
try {
|
||||
unlinkSync(LOCK);
|
||||
} catch {
|
||||
// Another waiter cleared it first.
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (h.pid === process.pid) return;
|
||||
if (Date.now() > deadline) throw new Error(`gave up after ${WAIT_MIN} min waiting for the recording in ${h.label} (pid ${h.pid})`);
|
||||
if (!announced) {
|
||||
console.log(`waiting: another recording is running (${h.label}, pid ${h.pid}, since ${h.since})`);
|
||||
announced = true;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseRecordingLock(): void {
|
||||
if (holder()?.pid !== process.pid) return;
|
||||
try {
|
||||
unlinkSync(LOCK);
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test as base, expect, type Locator, type Page, type TestInfo } from "@playwright/test";
|
||||
import { env, RUN_DIR } from "./env.ts";
|
||||
import { Recorder } from "./recorder.ts";
|
||||
|
||||
export type Box = { x: number; y: number; width: number; height: number };
|
||||
// `ignore` holds regions, in the still's own pixels, that comparisons skip.
|
||||
export type Shot = { name: string; file: string; caption: string; ignore?: Box[] };
|
||||
|
||||
export type FlowMeta = {
|
||||
slug: string;
|
||||
title: string;
|
||||
file: string;
|
||||
status: TestInfo["status"];
|
||||
// Spooled frames waiting for the end-of-run encode; `video` replaces it once encoded.
|
||||
frames?: string;
|
||||
video?: string;
|
||||
shots: Shot[];
|
||||
};
|
||||
|
||||
// Pages a signed-in flow must never land on: each means the session or the account setup is wrong.
|
||||
const GATES: [RegExp, string][] = [
|
||||
[/^\/auth\//, "the sign-in page: the saved session was refused. Delete qa/.artifacts/auth and run again"],
|
||||
[/^\/onboarding/, "onboarding: the account has not finished setup. Delete qa/.artifacts/auth so global setup completes it"],
|
||||
[/^\/select-org/, "the workspace picker: the account belongs to several workspaces; open the one the flow needs first"],
|
||||
];
|
||||
|
||||
// Dev-only chrome that would otherwise sit in every frame (the React Query devtools button).
|
||||
const HIDE_DEV_CHROME = `.tsqd-parent-container, .tsqd-open-btn-container { display: none !important; }`;
|
||||
|
||||
export class Proof {
|
||||
readonly shots: Shot[] = [];
|
||||
readonly dir: string;
|
||||
private readonly page: Page;
|
||||
|
||||
constructor(page: Page, dir: string) {
|
||||
this.page = page;
|
||||
this.dir = dir;
|
||||
}
|
||||
|
||||
// A title card in the video, so a reviewer knows what the next few seconds show.
|
||||
async chapter(title: string, description?: string): Promise<void> {
|
||||
await this.page.screencast.showChapter(title, { description, duration: 1600 });
|
||||
}
|
||||
|
||||
// A named full-resolution still. Keep names stable: follow-up comments diff stills by name.
|
||||
// `ignore` names content that changes on its own (relative times, live counters): the still is
|
||||
// published as is, and those regions are skipped when it is compared with the previous publish.
|
||||
async shot(name: string, opts: { caption?: string; target?: Locator; ignore?: Locator[] } = {}): Promise<void> {
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(name)) throw new Error(`shot name "${name}" must be kebab-case`);
|
||||
if (this.shots.some((s) => s.name === name)) throw new Error(`shot "${name}" taken twice in one flow`);
|
||||
const file = join(this.dir, "shots", `${name}.png`);
|
||||
const origin = opts.target ? await opts.target.boundingBox() : { x: 0, y: 0 };
|
||||
const ignore: Box[] = [];
|
||||
for (const locator of opts.ignore ?? []) {
|
||||
for (const el of await locator.all()) {
|
||||
const b = await el.boundingBox();
|
||||
if (b && origin) ignore.push({ x: Math.floor(b.x - origin.x), y: Math.floor(b.y - origin.y), width: Math.ceil(b.width) + 1, height: Math.ceil(b.height) + 1 });
|
||||
}
|
||||
}
|
||||
const options = { path: file, animations: "disabled" as const, caret: "hide" as const };
|
||||
await this.page.screencast.hideOverlays();
|
||||
try {
|
||||
if (opts.target) await opts.target.screenshot(options);
|
||||
else await this.page.screenshot(options);
|
||||
} finally {
|
||||
await this.page.screencast.showOverlays();
|
||||
}
|
||||
this.shots.push({ name, file, caption: opts.caption ?? name, ignore: ignore.length ? ignore : undefined });
|
||||
}
|
||||
|
||||
// Lets a result sit on screen long enough to be read in the video.
|
||||
async dwell(ms = 1200): Promise<void> {
|
||||
await this.page.waitForTimeout(ms);
|
||||
}
|
||||
}
|
||||
|
||||
export function slugOf(info: TestInfo): string {
|
||||
return info.titlePath
|
||||
.slice(1)
|
||||
.join(" ")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
export type Seed = "rich" | "sandbox";
|
||||
|
||||
// The seed each stack mode loads; a stack the harness did not start ("external") is trusted as is.
|
||||
const SEED_OF_MODE: Record<string, Seed | undefined> = { lite: "rich", full: "rich", sandbox: "sandbox" };
|
||||
|
||||
export const test = base.extend<{ proof: Proof; signedIn: boolean; seed: Seed }>({
|
||||
// `test.use({ signedIn: false })` for a flow that starts on the sign-in pages.
|
||||
signedIn: [true, { option: true }],
|
||||
// `test.use({ seed: "sandbox" })` for a flow written against the Sunrise Labs sandbox data.
|
||||
seed: ["rich", { option: true }],
|
||||
storageState: async ({ signedIn, storageState }, use) => {
|
||||
await use(signedIn ? storageState : { cookies: [], origins: [] });
|
||||
},
|
||||
proof: [
|
||||
async ({ page, signedIn, seed }, use, info) => {
|
||||
const loaded = SEED_OF_MODE[env.mode];
|
||||
const fix = seed === "sandbox" ? "pnpm stack reset-data sandbox" : "pnpm stack reset-data lite";
|
||||
if (loaded !== undefined && loaded !== seed) {
|
||||
const why = `written for the ${seed} seed, but the stack has the ${loaded} seed (${fix})`;
|
||||
console.log(`skipped "${info.title}": ${why}`);
|
||||
info.skip(true, why);
|
||||
}
|
||||
const slug = slugOf(info);
|
||||
const dir = join(RUN_DIR, slug);
|
||||
mkdirSync(join(dir, "shots"), { recursive: true });
|
||||
await page.addInitScript((css) => {
|
||||
const add = () => document.head.appendChild(Object.assign(document.createElement("style"), { textContent: css }));
|
||||
if (document.head) add();
|
||||
else document.addEventListener("DOMContentLoaded", add);
|
||||
}, HIDE_DEV_CHROME);
|
||||
|
||||
const frames = join(dir, "frames");
|
||||
const recorder = new Recorder(page, frames);
|
||||
await recorder.start();
|
||||
await page.screencast.showActions({ cursor: "pointer", duration: 700, fontSize: 20, position: "top-right" });
|
||||
const proof = new Proof(page, dir);
|
||||
|
||||
await use(proof);
|
||||
|
||||
const captured = await recorder.stop();
|
||||
// A failed flow's video is never published, so its frames are not worth encoding unless asked for.
|
||||
const keep = captured && (info.status === "passed" || process.env.QA_VIDEO_ON_FAIL === "1");
|
||||
if (!keep) rmSync(frames, { recursive: true, force: true });
|
||||
const meta: FlowMeta = {
|
||||
slug,
|
||||
title: info.title,
|
||||
file: info.file,
|
||||
status: info.status,
|
||||
frames: keep ? frames : undefined,
|
||||
shots: proof.shots,
|
||||
};
|
||||
writeFileSync(join(dir, "meta.json"), JSON.stringify(meta, null, 2));
|
||||
const path = new URL(page.url()).pathname;
|
||||
const gate = signedIn ? GATES.find(([re]) => re.test(path)) : undefined;
|
||||
if (gate) throw new Error(`the dashboard sent this flow to ${gate[1]}.`);
|
||||
},
|
||||
{ auto: true },
|
||||
],
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -0,0 +1,125 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { rename, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Page } from "@playwright/test";
|
||||
import { VIDEO } from "./env.ts";
|
||||
|
||||
// GitHub refuses larger attachments on some plans; above this the clip is re-encoded smaller.
|
||||
const MAX_BYTES = Number(process.env.QA_MAX_VIDEO_MB ?? 10) * 1024 * 1024;
|
||||
// x264 holds frames per thread, so this bounds the encoder's memory as much as its CPU.
|
||||
const THREADS = String(process.env.QA_ENCODE_THREADS ?? 4);
|
||||
|
||||
type Frame = { file: string; ts: number };
|
||||
|
||||
// Spools Playwright's screencast to disk as full-size JPEGs while the flow runs.
|
||||
// Encoding waits until the browser is closed (`encodeFrames`), so the two never hold memory at once.
|
||||
export class Recorder {
|
||||
private readonly page: Page;
|
||||
private readonly dir: string;
|
||||
private readonly frames: Frame[] = [];
|
||||
private lowRes = "";
|
||||
|
||||
constructor(page: Page, dir: string) {
|
||||
this.page = page;
|
||||
this.dir = dir;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
mkdirSync(this.dir, { recursive: true });
|
||||
await this.page.screencast.start({
|
||||
size: { width: VIDEO.width, height: VIDEO.height },
|
||||
quality: 95,
|
||||
onFrame: ({ data, timestamp }) => this.onFrame(data, timestamp),
|
||||
});
|
||||
}
|
||||
|
||||
// Frames arrive only when the page repaints; their timestamps carry the pauses in between.
|
||||
private async onFrame(data: Buffer, timestamp: number): Promise<void> {
|
||||
if (!this.frames.length && !(await this.painted())) return;
|
||||
if (!this.frames.length) {
|
||||
const size = jpegSize(data);
|
||||
if (size && size.width < VIDEO.width) this.lowRes = `${size.width}x${size.height}`;
|
||||
}
|
||||
const file = `f${String(this.frames.length).padStart(6, "0")}.jpg`;
|
||||
this.frames.push({ file, ts: timestamp });
|
||||
await writeFile(join(this.dir, file), data);
|
||||
}
|
||||
|
||||
// The video starts at the first frame with visible text, not on the blank page before the app renders.
|
||||
private async painted(): Promise<boolean> {
|
||||
if (this.page.url() === "about:blank") return false;
|
||||
return this.page
|
||||
.evaluate(
|
||||
() =>
|
||||
performance.getEntriesByType("paint").some((e) => e.name === "first-contentful-paint") &&
|
||||
(document.body?.innerText ?? "").trim().length > 0,
|
||||
)
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
// Writes the frame list ffmpeg reads later. Returns false when nothing was captured.
|
||||
async stop(tailMs = 800): Promise<boolean> {
|
||||
await this.page.screencast.stop().catch(() => {});
|
||||
if (this.lowRes) {
|
||||
throw new Error(`screencast frames arrived at ${this.lowRes}: another screencast (trace screenshots?) started first and set the size`);
|
||||
}
|
||||
if (!this.frames.length) return false;
|
||||
const end = Date.now() + tailMs;
|
||||
const lines = ["ffconcat version 1.0"];
|
||||
this.frames.forEach((f, i) => {
|
||||
const next = this.frames[i + 1]?.ts ?? end;
|
||||
lines.push(`file '${f.file}'`, `duration ${Math.max(0.001, (next - f.ts) / 1000).toFixed(4)}`);
|
||||
});
|
||||
// The concat demuxer ignores the last entry's duration unless the file is listed once more.
|
||||
lines.push(`file '${this.frames[this.frames.length - 1].file}'`);
|
||||
writeFileSync(join(this.dir, "frames.ffconcat"), `${lines.join("\n")}\n`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Encodes a spooled flow to 1080p H.264, then removes the frames.
|
||||
export async function encodeFrames(framesDir: string, out: string): Promise<void> {
|
||||
const list = join(framesDir, "frames.ffconcat");
|
||||
if (!existsSync(list)) throw new Error(`no frame list in ${framesDir}`);
|
||||
const { width, height, fps, crf, preset } = VIDEO;
|
||||
const vf = [
|
||||
`fps=${fps}`,
|
||||
`scale=${width}:${height}:force_original_aspect_ratio=decrease:flags=lanczos:in_range=pc:out_range=tv`,
|
||||
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:white`,
|
||||
"format=yuv420p",
|
||||
].join(",");
|
||||
await ffmpeg(["-f", "concat", "-safe", "0", "-i", list, "-vf", vf, ...x264(crf, preset), out]);
|
||||
for (let next = crf + 5; statSync(out).size > MAX_BYTES && next <= 38; next += 5) {
|
||||
const tmp = out.replace(/\.mp4$/, `.crf${next}.mp4`);
|
||||
await ffmpeg(["-i", out, ...x264(next, preset), tmp]);
|
||||
await rename(tmp, out);
|
||||
}
|
||||
rmSync(framesDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function x264(crf: number, preset: string): string[] {
|
||||
return [
|
||||
...["-c:v", "libx264", "-preset", preset, "-crf", String(crf), "-threads", THREADS],
|
||||
...["-x264-params", "rc-lookahead=20", "-pix_fmt", "yuv420p", "-color_range", "tv", "-movflags", "+faststart"],
|
||||
];
|
||||
}
|
||||
|
||||
async function ffmpeg(args: string[]): Promise<void> {
|
||||
let stderr = "";
|
||||
const child = spawn("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { stdio: ["ignore", "ignore", "pipe"] });
|
||||
child.stderr.on("data", (d: Buffer) => (stderr += d.toString()));
|
||||
const code = await new Promise<number | null>((resolve) => child.on("close", resolve));
|
||||
if (code !== 0) throw new Error(`ffmpeg exited ${code}: ${stderr.trim()}`);
|
||||
}
|
||||
|
||||
// Reads the frame size from the JPEG's start-of-frame marker.
|
||||
function jpegSize(buf: Buffer): { width: number; height: number } | undefined {
|
||||
for (let i = 2; i + 9 < buf.length; ) {
|
||||
if (buf[i] !== 0xff) return undefined;
|
||||
const marker = buf[i + 1];
|
||||
if (marker >= 0xc0 && marker <= 0xc2) return { height: buf.readUInt16BE(i + 5), width: buf.readUInt16BE(i + 7) };
|
||||
i += 2 + buf.readUInt16BE(i + 2);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "warmbly-qa",
|
||||
"private": true,
|
||||
"description": "Recorded proof for pull requests: scripted Playwright flows, 1080p H.264 walkthroughs and stills, published with gh",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.18"
|
||||
},
|
||||
"scripts": {
|
||||
"bootstrap": "sh scripts/bootstrap.sh",
|
||||
"check": "node scripts/doctor.ts",
|
||||
"stack": "bash scripts/stack.sh",
|
||||
"proof": "playwright test",
|
||||
"proof:fresh": "bash scripts/stack.sh reset-data && playwright test",
|
||||
"aria": "node scripts/aria.ts",
|
||||
"share": "node scripts/publish.ts",
|
||||
"typecheck": "tsc --noEmit -p ."
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "1.63.0",
|
||||
"@types/node": "^26.6.4",
|
||||
"@types/pngjs": "^6.0.5",
|
||||
"pixelmatch": "^7.2.0",
|
||||
"pngjs": "^7.0.0",
|
||||
"typescript": "~5.9.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { defineConfig } from "@playwright/test";
|
||||
import { authFile, env, VIDEO } from "./lib/env.ts";
|
||||
|
||||
// Playwright empties its output directory when a run starts, so a run waiting on the recording lock
|
||||
// would delete the traces of the one recording. Each run gets its own; workers inherit the id.
|
||||
process.env.QA_RUN_ID ??= String(process.pid);
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./flows",
|
||||
testMatch: "**/*.flow.ts",
|
||||
outputDir: `./.artifacts/test-results/run-${process.env.QA_RUN_ID}`,
|
||||
globalSetup: "./lib/global-setup.ts",
|
||||
// Encodes the run's videos after the browser has exited, then releases the machine-wide recording lock.
|
||||
globalTeardown: "./lib/global-teardown.ts",
|
||||
// One flow at a time: a 1080p encode per parallel worker starves the browser and the video stutters.
|
||||
workers: 1,
|
||||
fullyParallel: false,
|
||||
retries: 0,
|
||||
timeout: 180_000,
|
||||
expect: { timeout: 15_000 },
|
||||
reporter: [["list"]],
|
||||
use: {
|
||||
browserName: "chromium",
|
||||
// Full Chromium in new headless mode renders exactly like the desktop browser.
|
||||
channel: "chromium",
|
||||
baseURL: env.webURL,
|
||||
storageState: authFile(),
|
||||
viewport: { width: VIDEO.width, height: VIDEO.height },
|
||||
deviceScaleFactor: 1,
|
||||
colorScheme: "light",
|
||||
locale: "en-US",
|
||||
actionTimeout: 15_000,
|
||||
navigationTimeout: 30_000,
|
||||
video: "off",
|
||||
screenshot: "off",
|
||||
// Trace screenshots would start a small screencast first, and the recorder would share its size.
|
||||
trace: { mode: "retain-on-failure", screenshots: false, snapshots: true },
|
||||
// Human pacing, so a reviewer can follow each click in the video.
|
||||
launchOptions: { slowMo: Number(process.env.QA_SLOWMO ?? 250) },
|
||||
},
|
||||
});
|
||||
Generated
+97
@@ -0,0 +1,97 @@
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
devDependencies:
|
||||
'@playwright/test':
|
||||
specifier: 1.63.0
|
||||
version: 1.63.0
|
||||
'@types/node':
|
||||
specifier: ^26.6.4
|
||||
version: 26.6.4
|
||||
'@types/pngjs':
|
||||
specifier: ^6.0.5
|
||||
version: 6.0.5
|
||||
pixelmatch:
|
||||
specifier: ^7.2.0
|
||||
version: 7.2.0
|
||||
pngjs:
|
||||
specifier: ^7.0.0
|
||||
version: 7.0.0
|
||||
typescript:
|
||||
specifier: ~5.9.3
|
||||
version: 5.9.3
|
||||
|
||||
packages:
|
||||
|
||||
'@playwright/test@1.63.0':
|
||||
resolution: {integrity: sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==}
|
||||
engines: {node: '>=20'}
|
||||
hasBin: true
|
||||
|
||||
'@types/node@26.6.4':
|
||||
resolution: {integrity: sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg==}
|
||||
|
||||
'@types/pngjs@6.0.5':
|
||||
resolution: {integrity: sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==}
|
||||
|
||||
pixelmatch@7.2.0:
|
||||
resolution: {integrity: sha512-xhcb4yHu9sM/G7foGzoLtXYcC0zHEaOXXjRKhGup0fw78Nf2Tkiapv4EQyMzrbcmQPsllAI7DbFY2UT7PlI9Pg==}
|
||||
hasBin: true
|
||||
|
||||
playwright-core@1.63.0:
|
||||
resolution: {integrity: sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==}
|
||||
engines: {node: '>=20'}
|
||||
hasBin: true
|
||||
|
||||
playwright@1.63.0:
|
||||
resolution: {integrity: sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==}
|
||||
engines: {node: '>=20'}
|
||||
hasBin: true
|
||||
|
||||
pngjs@7.0.0:
|
||||
resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==}
|
||||
engines: {node: '>=14.19.0'}
|
||||
|
||||
typescript@5.9.3:
|
||||
resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
|
||||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
undici-types@8.9.0:
|
||||
resolution: {integrity: sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==}
|
||||
|
||||
snapshots:
|
||||
|
||||
'@playwright/test@1.63.0':
|
||||
dependencies:
|
||||
playwright: 1.63.0
|
||||
|
||||
'@types/node@26.6.4':
|
||||
dependencies:
|
||||
undici-types: 8.9.0
|
||||
|
||||
'@types/pngjs@6.0.5':
|
||||
dependencies:
|
||||
'@types/node': 26.6.4
|
||||
|
||||
pixelmatch@7.2.0:
|
||||
dependencies:
|
||||
pngjs: 7.0.0
|
||||
|
||||
playwright-core@1.63.0: {}
|
||||
|
||||
playwright@1.63.0:
|
||||
dependencies:
|
||||
playwright-core: 1.63.0
|
||||
|
||||
pngjs@7.0.0: {}
|
||||
|
||||
typescript@5.9.3: {}
|
||||
|
||||
undici-types@8.9.0: {}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Prints a page's accessibility tree with the saved session: the cheap way to find selectors for a flow.
|
||||
// node scripts/aria.ts /app/contacts [css-selector]
|
||||
import { chromium } from "@playwright/test";
|
||||
import { existsSync } from "node:fs";
|
||||
import { assertLocal, authFile, env, QA_DIR, touchStack } from "../lib/env.ts";
|
||||
import { acquireRecordingLock, releaseRecordingLock } from "../lib/lock.ts";
|
||||
|
||||
const [path = "/app/emails", selector = "body"] = process.argv.slice(2);
|
||||
if (env.mode === "none") {
|
||||
console.error("this worktree has no stack running; start one with `pnpm stack up`");
|
||||
process.exit(1);
|
||||
}
|
||||
assertLocal(env.webURL);
|
||||
if (!existsSync(authFile())) {
|
||||
console.error("no saved session yet; run any flow once (`pnpm proof`) so global setup signs in");
|
||||
process.exit(1);
|
||||
}
|
||||
touchStack();
|
||||
await acquireRecordingLock(`${QA_DIR} (aria)`);
|
||||
const browser = await chromium.launch({ channel: "chromium" });
|
||||
const context = await browser.newContext({ storageState: authFile(), viewport: { width: 1920, height: 1080 } });
|
||||
const page = await context.newPage();
|
||||
await page.goto(new URL(path, env.webURL).toString());
|
||||
await page.waitForLoadState("networkidle");
|
||||
console.log(await page.locator(selector).first().ariaSnapshot());
|
||||
await browser.close();
|
||||
releaseRecordingLock();
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Installs the harness: dependencies, Playwright's Chromium and the small images every stack runs. Then checks it all.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm exec playwright install chromium
|
||||
for image in redis:7-alpine nats:2.10-alpine axllent/mailpit:latest; do
|
||||
docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image"
|
||||
done
|
||||
exec node scripts/doctor.ts
|
||||
@@ -0,0 +1,97 @@
|
||||
// Checks everything a recording needs and prints the fix for whatever is missing.
|
||||
// node scripts/doctor.ts
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { chromium } from "@playwright/test";
|
||||
import { authFile, env } from "../lib/env.ts";
|
||||
|
||||
let failed = false;
|
||||
|
||||
function report(ok: boolean, what: string, fix?: string): void {
|
||||
console.log(`${ok ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m"} ${what}`);
|
||||
if (!ok) {
|
||||
failed = true;
|
||||
if (fix) console.log(` fix: ${fix}`);
|
||||
}
|
||||
}
|
||||
|
||||
function run(cmd: string, args: string[]): { ok: boolean; out: string } {
|
||||
const res = spawnSync(cmd, args, { encoding: "utf8" });
|
||||
return { ok: res.status === 0, out: `${res.stdout ?? ""}${res.stderr ?? ""}` };
|
||||
}
|
||||
|
||||
function atLeast(version: string, min: number[]): boolean {
|
||||
const parts = version.split(".").map(Number);
|
||||
for (let i = 0; i < min.length; i++) {
|
||||
if ((parts[i] ?? 0) !== min[i]) return (parts[i] ?? 0) > min[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function reachable(url: string): Promise<boolean> {
|
||||
try {
|
||||
return (await fetch(url, { signal: AbortSignal.timeout(4000) })).ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const node = process.versions.node;
|
||||
report(atLeast(node, [22, 18]), `node ${node} (22.18+ runs the TypeScript scripts directly)`, "install Node 22.18 or newer");
|
||||
|
||||
const ffmpeg = run("ffmpeg", ["-hide_banner", "-encoders"]);
|
||||
report(ffmpeg.ok, "ffmpeg on PATH", "install ffmpeg (pacman -S ffmpeg / apt install ffmpeg / brew install ffmpeg)");
|
||||
if (ffmpeg.ok) report(/libx264/.test(ffmpeg.out), "ffmpeg has libx264", "install an ffmpeg build with libx264 (the distro package has it)");
|
||||
|
||||
const exe = chromium.executablePath();
|
||||
report(existsSync(exe), "Playwright Chromium installed", "pnpm exec playwright install chromium");
|
||||
if (existsSync(exe)) {
|
||||
try {
|
||||
const browser = await chromium.launch({ channel: "chromium" });
|
||||
await browser.close();
|
||||
report(true, "Chromium launches");
|
||||
} catch (err) {
|
||||
report(false, `Chromium launches (${String(err).split("\n")[0]})`, "sudo pnpm exec playwright install-deps chromium");
|
||||
}
|
||||
}
|
||||
|
||||
const gh = run("gh", ["--version"]);
|
||||
const ghVersion = gh.out.match(/gh version (\d+\.\d+\.\d+)/)?.[1] ?? "";
|
||||
report(gh.ok && atLeast(ghVersion, [2, 99]), `gh ${ghVersion || "missing"} (2.99+ uploads media with --attach)`, "install or upgrade the GitHub CLI: https://cli.github.com");
|
||||
if (gh.ok) report(run("gh", ["auth", "status"]).ok, "gh is signed in", "gh auth login");
|
||||
|
||||
console.log("");
|
||||
const docker = run("docker", ["info", "--format", "{{.ServerVersion}}"]);
|
||||
report(docker.ok, "docker is running", "start the docker daemon");
|
||||
if (docker.ok) {
|
||||
const names = run("docker", ["ps", "--format", "{{.Names}}"]).out;
|
||||
report(/^warmbly-postgres-1$/m.test(names), "shared postgres (warmbly-postgres-1)", "make infra (from the repo root)");
|
||||
const images: [string, string][] = [
|
||||
["redis:7-alpine", "every mode"],
|
||||
["nats:2.10-alpine", "every mode"],
|
||||
["axllent/mailpit:latest", "every mode"],
|
||||
["ghcr.io/warmbly/warmbly/realtime:prod", "full and sandbox"],
|
||||
["dovecot/dovecot:latest", "sandbox"],
|
||||
["ghcr.io/warmbly/warmbly/tracking:prod", "sandbox"],
|
||||
];
|
||||
for (const [image, modes] of images) {
|
||||
const present = run("docker", ["image", "inspect", image]).ok;
|
||||
const required = modes === "every mode" || (modes.includes(env.mode) && env.mode !== "external");
|
||||
if (present) report(true, `image ${image}`);
|
||||
else if (required) report(false, `image ${image} (${modes})`, `docker pull ${image}`);
|
||||
else console.log(` - image ${image} not pulled (needed for ${modes}): docker pull ${image}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
if (env.mode === "none") {
|
||||
report(false, "this worktree's stack is running", "pnpm stack up (lite; `full` or `sandbox` for more)");
|
||||
} else {
|
||||
console.log(` stack: ${env.mode === "external" ? "QA_WEB_URL" : `mode ${env.mode}`}, signs in as ${env.email}`);
|
||||
report(await reachable(env.webURL), `dashboard ${env.webURL}`, "pnpm stack up");
|
||||
report(await reachable(`${env.apiURL}/health`), `backend ${env.apiURL}`, "pnpm stack up");
|
||||
report(await reachable(`${env.mailpitURL}/api/v1/info`), `mailpit ${env.mailpitURL} (login codes)`, "pnpm stack up");
|
||||
console.log(` session: ${existsSync(authFile()) ? "saved" : "none yet; the first `pnpm proof` signs in"}`);
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
@@ -0,0 +1,262 @@
|
||||
// Publishes the last recorded run to the current branch's pull request.
|
||||
// First publish: a Proof section appended to the PR description (stills inline, walkthrough videos last).
|
||||
// Every later publish: a comment with before/after pairs for the stills that changed since the previous one.
|
||||
//
|
||||
// node scripts/publish.ts publish
|
||||
// node scripts/publish.ts --dry-run write the body and print the gh command, upload nothing
|
||||
// node scripts/publish.ts --no-video stills only
|
||||
// node scripts/publish.ts --force comment even when no still changed
|
||||
// node scripts/publish.ts --replace-body rewrite the PR description's Proof section instead of commenting
|
||||
// node scripts/publish.ts --allow-failed publish a run with failing flows
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
import pixelmatch from "pixelmatch";
|
||||
import pngjs from "pngjs";
|
||||
import { ARTIFACTS, assertLocal, QA_DIR, RUN_DIR } from "../lib/env.ts";
|
||||
import type { RunInfo } from "../lib/global-setup.ts";
|
||||
import type { Box, FlowMeta, Shot } from "../lib/proof.ts";
|
||||
|
||||
const { PNG } = pngjs;
|
||||
const MARK = "<!-- warmbly-proof:start -->";
|
||||
const MARK_END = "<!-- warmbly-proof:end -->";
|
||||
const MAX_ATTACHMENTS = 50;
|
||||
|
||||
const flags = new Set(process.argv.slice(2));
|
||||
for (const f of flags) {
|
||||
if (!["--dry-run", "--no-video", "--force", "--replace-body", "--allow-failed"].includes(f)) die(`unknown flag ${f}`);
|
||||
}
|
||||
|
||||
function die(msg: string): never {
|
||||
console.error(`publish: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function git(...args: string[]): string {
|
||||
return execFileSync("git", args, { cwd: QA_DIR, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
function short(sha: string): string {
|
||||
return sha.slice(0, 9);
|
||||
}
|
||||
|
||||
type Baseline = { sha: string; shots: Record<string, string>; ignore?: Record<string, Box[]> };
|
||||
type Pr = { number: number; body: string; url: string };
|
||||
type Change = { flow: FlowMeta; shot: Shot; key: string; kind: "new" | "changed"; before?: string };
|
||||
|
||||
function loadRun(): { run: RunInfo; flows: FlowMeta[] } {
|
||||
const runFile = join(RUN_DIR, "run.json");
|
||||
if (!existsSync(runFile)) die("no recorded run; record one with `pnpm proof` first");
|
||||
const run = JSON.parse(readFileSync(runFile, "utf8")) as RunInfo;
|
||||
const flows = readdirSync(RUN_DIR, { withFileTypes: true })
|
||||
.filter((d) => d.isDirectory() && existsSync(join(RUN_DIR, d.name, "meta.json")))
|
||||
.map((d) => JSON.parse(readFileSync(join(RUN_DIR, d.name, "meta.json"), "utf8")) as FlowMeta)
|
||||
.sort((a, b) => a.slug.localeCompare(b.slug));
|
||||
if (!flows.length) die("the last run recorded no flows");
|
||||
return { run, flows };
|
||||
}
|
||||
|
||||
function currentPr(): Pr {
|
||||
const res = spawnSync("gh", ["pr", "view", "--json", "number,body,url"], { cwd: QA_DIR, encoding: "utf8" });
|
||||
// A dry run previews the first publish before the PR exists.
|
||||
if (res.status !== 0 && flags.has("--dry-run")) return { number: 0, body: "", url: "(no pull request yet)" };
|
||||
if (res.status !== 0) die(`no pull request for this branch yet. Open it with \`gh pr create\`, then publish.\n${res.stderr.trim()}`);
|
||||
return JSON.parse(res.stdout) as Pr;
|
||||
}
|
||||
|
||||
// Blanks the regions a flow marked as changing on its own, in both images, before comparing.
|
||||
function blank(img: InstanceType<typeof PNG>, boxes: Box[]): void {
|
||||
for (const b of boxes) {
|
||||
for (let y = Math.max(0, b.y); y < Math.min(img.height, b.y + b.height); y++) {
|
||||
img.data.fill(0, (y * img.width + Math.max(0, b.x)) * 4, (y * img.width + Math.min(img.width, b.x + b.width)) * 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function differs(a: string, b: string, ignore: Box[]): boolean {
|
||||
const x = PNG.sync.read(readFileSync(a));
|
||||
const y = PNG.sync.read(readFileSync(b));
|
||||
if (x.width !== y.width || x.height !== y.height) return true;
|
||||
blank(x, ignore);
|
||||
blank(y, ignore);
|
||||
const changed = pixelmatch(x.data, y.data, undefined, x.width, x.height, { threshold: 0.1 });
|
||||
return changed > Math.max(40, x.width * x.height * 0.0002);
|
||||
}
|
||||
|
||||
function keyOf(flow: FlowMeta, shot: Shot): string {
|
||||
return `${flow.slug}__${shot.name}`;
|
||||
}
|
||||
|
||||
// Attachments are referenced in the body by the exact path passed to --attach, which gh rewrites to the upload.
|
||||
class Attachments {
|
||||
readonly dir = join(ARTIFACTS, "publish");
|
||||
readonly paths: string[] = [];
|
||||
|
||||
constructor() {
|
||||
rmSync(this.dir, { recursive: true, force: true });
|
||||
mkdirSync(this.dir, { recursive: true });
|
||||
}
|
||||
|
||||
add(src: string, name: string): string {
|
||||
const dest = join(this.dir, name);
|
||||
copyFileSync(src, dest);
|
||||
const ref = `./${relative(QA_DIR, dest)}`;
|
||||
this.paths.push(ref);
|
||||
return ref;
|
||||
}
|
||||
}
|
||||
|
||||
function sentence(title: string): string {
|
||||
return title.charAt(0).toUpperCase() + title.slice(1);
|
||||
}
|
||||
|
||||
function image(alt: string, ref: string): string {
|
||||
return `![${alt.replace(/[[\]]/g, "")}](${ref})`;
|
||||
}
|
||||
|
||||
function initialSection(run: RunInfo, flows: FlowMeta[], files: Attachments, videos: FlowMeta[]): string {
|
||||
const lines = [
|
||||
MARK,
|
||||
"## Proof",
|
||||
"",
|
||||
`Recorded at \`${short(run.sha)}\` against a local stack with seed data, 1920x1080.`,
|
||||
"",
|
||||
];
|
||||
for (const flow of flows) {
|
||||
lines.push(`### ${sentence(flow.title)}`, "");
|
||||
for (const shot of flow.shots) lines.push(`**${shot.caption}**`, "", image(shot.caption, files.add(shot.file, `${flow.slug}-${shot.name}.png`)), "");
|
||||
}
|
||||
if (videos.length) {
|
||||
lines.push("### Walkthrough", "", "Videos below, in this order:", "");
|
||||
videos.forEach((f, i) => lines.push(`${i + 1}. ${sentence(f.title)}`));
|
||||
lines.push("");
|
||||
}
|
||||
lines.push(MARK_END);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function updateComment(run: RunInfo, base: Baseline | undefined, changes: Change[], unchanged: number, videos: FlowMeta[], files: Attachments): string {
|
||||
const lines: string[] = ["<!-- warmbly-proof:update -->"];
|
||||
if (base) {
|
||||
lines.push(`### Proof update: \`${short(base.sha)}..${short(run.sha)}\``, "");
|
||||
try {
|
||||
const subjects = git("log", "--format=%s", "--max-count=20", `${base.sha}..${run.sha}`).split("\n").filter(Boolean);
|
||||
for (const s of subjects) lines.push(`- ${s}`);
|
||||
if (subjects.length) lines.push("");
|
||||
} catch {
|
||||
// A rebase can drop the old commit; the range header still says what moved.
|
||||
}
|
||||
} else {
|
||||
lines.push(`### Proof update at \`${short(run.sha)}\``, "", "No earlier publish is recorded on this machine, so every screen is shown.", "");
|
||||
}
|
||||
|
||||
let flowTitle = "";
|
||||
for (const c of changes) {
|
||||
if (c.flow.title !== flowTitle) {
|
||||
flowTitle = c.flow.title;
|
||||
lines.push(`#### ${sentence(flowTitle)}`, "");
|
||||
}
|
||||
const after = files.add(c.shot.file, `${c.flow.slug}-${c.shot.name}.png`);
|
||||
if (c.kind === "changed" && c.before) {
|
||||
const before = files.add(c.before, `before-${c.flow.slug}-${c.shot.name}.png`);
|
||||
lines.push(`**${c.shot.caption}** (changed)`, "", "| Before | After |", "| --- | --- |");
|
||||
lines.push(`| ${image(`${c.shot.caption}, before`, before)} | ${image(`${c.shot.caption}, after`, after)} |`, "");
|
||||
} else {
|
||||
lines.push(`**${c.shot.caption}** (new)`, "", image(c.shot.caption, after), "");
|
||||
}
|
||||
}
|
||||
if (unchanged) lines.push(`Unchanged: ${unchanged} screen${unchanged === 1 ? "" : "s"}.`, "");
|
||||
if (videos.length) {
|
||||
lines.push("Walkthrough videos below, in this order:", "");
|
||||
videos.forEach((f, i) => lines.push(`${i + 1}. ${sentence(f.title)}`));
|
||||
}
|
||||
return lines.join("\n").trimEnd();
|
||||
}
|
||||
|
||||
function saveBaseline(file: string, run: RunInfo, flows: FlowMeta[]): void {
|
||||
const dir = join(file, "..", "shots");
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const shots: Record<string, string> = {};
|
||||
const ignore: Record<string, Box[]> = {};
|
||||
for (const flow of flows) {
|
||||
for (const shot of flow.shots) {
|
||||
const key = keyOf(flow, shot);
|
||||
shots[key] = join(dir, `${key}.png`);
|
||||
copyFileSync(shot.file, shots[key]);
|
||||
if (shot.ignore) ignore[key] = shot.ignore;
|
||||
}
|
||||
}
|
||||
writeFileSync(file, JSON.stringify({ sha: run.sha, shots, ignore } satisfies Baseline, null, 2));
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const { run, flows } = loadRun();
|
||||
assertLocal(run.webURL);
|
||||
|
||||
const failed = flows.filter((f) => f.status !== "passed");
|
||||
if (failed.length && !flags.has("--allow-failed")) {
|
||||
die(`flows failed, fix them before publishing: ${failed.map((f) => f.title).join(", ")}`);
|
||||
}
|
||||
const unencoded = flows.filter((f) => f.frames);
|
||||
if (unencoded.length) die(`the run ended before its videos were encoded (${unencoded.map((f) => f.title).join(", ")}); record again`);
|
||||
const head = git("rev-parse", "HEAD");
|
||||
if (run.sha !== head) die(`the recording is from ${short(run.sha)} but HEAD is ${short(head)}; record again`);
|
||||
if (run.dirty) console.warn("publish: warning: the recording was made with uncommitted changes in the tree");
|
||||
if (!git("branch", "-r", "--contains", head)) console.warn("publish: warning: HEAD is not pushed yet, so reviewers cannot see this commit");
|
||||
|
||||
const pr = currentPr();
|
||||
const baselineFile = join(ARTIFACTS, "published", `pr-${pr.number}`, "state.json");
|
||||
const base = existsSync(baselineFile) ? (JSON.parse(readFileSync(baselineFile, "utf8")) as Baseline) : undefined;
|
||||
const files = new Attachments();
|
||||
const withVideo = (list: FlowMeta[]) => (flags.has("--no-video") ? [] : list.filter((f) => f.video && existsSync(f.video)));
|
||||
const bodyFile = join(files.dir, "body.md");
|
||||
let args: string[];
|
||||
|
||||
const firstPublish = !pr.body?.includes(MARK);
|
||||
if (firstPublish || flags.has("--replace-body")) {
|
||||
const videos = withVideo(flows);
|
||||
const kept = (pr.body ?? "").split(MARK)[0].trimEnd();
|
||||
writeFileSync(bodyFile, `${kept ? `${kept}\n\n` : ""}${initialSection(run, flows, files, videos)}\n`);
|
||||
for (const f of videos) files.add(f.video!, `${f.slug}.mp4`);
|
||||
args = ["pr", "edit", String(pr.number), "--body-file", bodyFile];
|
||||
} else {
|
||||
const changes: Change[] = [];
|
||||
let unchanged = 0;
|
||||
for (const flow of flows) {
|
||||
for (const shot of flow.shots) {
|
||||
const key = keyOf(flow, shot);
|
||||
const before = base?.shots[key];
|
||||
if (!before || !existsSync(before)) changes.push({ flow, shot, key, kind: "new" });
|
||||
else if (differs(before, shot.file, [...(base?.ignore?.[key] ?? []), ...(shot.ignore ?? [])])) changes.push({ flow, shot, key, kind: "changed", before });
|
||||
else unchanged++;
|
||||
}
|
||||
}
|
||||
if (!changes.length && !flags.has("--force")) {
|
||||
console.log(`publish: no screen changed since ${base ? short(base.sha) : "the last publish"}; nothing posted (--force posts anyway)`);
|
||||
return;
|
||||
}
|
||||
const changedFlows = new Set(changes.map((c) => c.flow.slug));
|
||||
const videos = withVideo(flows.filter((f) => changedFlows.has(f.slug) || flags.has("--force")));
|
||||
writeFileSync(bodyFile, `${updateComment(run, base, changes, unchanged, videos, files)}\n`);
|
||||
for (const f of videos) files.add(f.video!, `${f.slug}.mp4`);
|
||||
args = ["pr", "comment", String(pr.number), "--body-file", bodyFile];
|
||||
}
|
||||
|
||||
if (files.paths.length > MAX_ATTACHMENTS) {
|
||||
die(`${files.paths.length} attachments is over gh's limit of ${MAX_ATTACHMENTS}; record fewer flows or pass --no-video`);
|
||||
}
|
||||
for (const p of files.paths) args.push("--attach", p);
|
||||
|
||||
if (flags.has("--dry-run")) {
|
||||
console.log(readFileSync(bodyFile, "utf8"));
|
||||
console.log(`\n(dry run) gh ${args.join(" ")}`);
|
||||
return;
|
||||
}
|
||||
const res = spawnSync("gh", args, { cwd: QA_DIR, stdio: "inherit" });
|
||||
if (res.status !== 0) die("gh failed; nothing was recorded as published");
|
||||
saveBaseline(baselineFile, run, flows);
|
||||
console.log(`publish: ${firstPublish || flags.has("--replace-body") ? "description updated" : "comment posted"} on ${pr.url}`);
|
||||
}
|
||||
|
||||
main();
|
||||
Executable
+395
@@ -0,0 +1,395 @@
|
||||
#!/usr/bin/env bash
|
||||
# This worktree's own Warmbly stack, isolated from every other stack on the machine:
|
||||
# its own database, Redis, NATS and Mailpit, so no cache, event or login code crosses over.
|
||||
#
|
||||
# stack.sh up [lite|full|sandbox] start (the mode is remembered; lite on first run)
|
||||
# stack.sh down stop everything; the data stays
|
||||
# stack.sh restart rebuild the Go binaries and start again
|
||||
# stack.sh status what runs, where, and how much memory it holds
|
||||
# stack.sh logs [name] follow the logs (backend, web, consumer, worker, simulator)
|
||||
# stack.sh reset-data [mode] drop the database and start again with fresh seed data
|
||||
# stack.sh ls every QA stack on this machine
|
||||
#
|
||||
# Modes:
|
||||
# lite backend + dashboard on the rich seed (SEED_RICH + SEED_FULL), dev@warmbly.com
|
||||
# full lite + consumer, worker and realtime: sends, syncs and live updates work
|
||||
# sandbox full + tracking, Dovecot and the simulator on the Sunrise Labs sandbox seed
|
||||
# (sandbox@warmbly.test): live mailboxes, opens, clicks and replies
|
||||
#
|
||||
# The stack stops itself after QA_STACK_IDLE_MIN minutes (default 60) with no recording; 0 keeps it up.
|
||||
set -euo pipefail
|
||||
|
||||
QA=$(cd "$(dirname "$0")/.." && pwd)
|
||||
REPO=$(cd "$QA/.." && pwd)
|
||||
RUN=$QA/.artifacts/stack
|
||||
BIN=$RUN/bin
|
||||
mkdir -p "$RUN"
|
||||
|
||||
slug=$(basename "$REPO" | tr -c '[:alnum:]\n' '_' | tr '[:upper:]' '[:lower:]')
|
||||
DB=${QA_DB:-warmbly_qa_$slug}
|
||||
LABEL=warmbly-qa=$slug
|
||||
IDLE_MIN=${QA_STACK_IDLE_MIN:-60}
|
||||
REALTIME_IMAGE=${QA_REALTIME_IMAGE:-ghcr.io/warmbly/warmbly/realtime:prod}
|
||||
TRACKING_IMAGE=${QA_TRACKING_IMAGE:-ghcr.io/warmbly/warmbly/tracking:prod}
|
||||
AUTH_SECRET=local-dev-auth-secret-minimum-32-characters-long
|
||||
INTERNAL_TOKEN=local-dev-internal-token
|
||||
MODE=lite
|
||||
|
||||
say() { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33mwarning:\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
alive() { [ -s "$RUN/$1.pid" ] && kill -0 "$(cat "$RUN/$1.pid")" 2>/dev/null; }
|
||||
psql_() { docker exec warmbly-postgres-1 psql -U warmbly "$@"; }
|
||||
cname() { echo "wqa-$slug-$1"; }
|
||||
NET=wqa-$slug
|
||||
running() { [ "$(docker inspect -f '{{.State.Running}}' "$(cname "$1")" 2>/dev/null)" = true ]; }
|
||||
|
||||
listening() {
|
||||
if command -v ss >/dev/null; then ss -ltn | awk '{print $4}' | grep -qE ":$1\$"
|
||||
else lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1; fi
|
||||
}
|
||||
|
||||
has() { # does the current mode include this tier
|
||||
case "$1" in
|
||||
full) [ "$MODE" = full ] || [ "$MODE" = sandbox ] ;;
|
||||
sandbox) [ "$MODE" = sandbox ] ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── ports ────────────────────────────────────────────────────────────────
|
||||
PORT_KEYS="API WEB REDIS NATS SMTP MAILPIT RT TRACK IMAPS VENDOR"
|
||||
base_of() {
|
||||
case "$1" in
|
||||
API) echo 18180 ;; WEB) echo 15280 ;; REDIS) echo 16480 ;; NATS) echo 14322 ;;
|
||||
SMTP) echo 11125 ;; MAILPIT) echo 18125 ;; RT) echo 14100 ;; TRACK) echo 13100 ;;
|
||||
IMAPS) echo 10994 ;; VENDOR) echo 18199 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
assigned() { # port already given to another key of this stack
|
||||
local k
|
||||
for k in $PORT_KEYS; do [ "${!k:-}" = "$1" ] && return 0; done
|
||||
return 1
|
||||
}
|
||||
|
||||
pick() { # first port at or above $1 that is free and not assigned here
|
||||
local p=$1
|
||||
while listening "$p" || assigned "$p"; do p=$((p + 1)); done
|
||||
echo "$p"
|
||||
}
|
||||
|
||||
save_ports() {
|
||||
local k
|
||||
: >"$RUN/ports"
|
||||
for k in $PORT_KEYS; do echo "$k=${!k}" >>"$RUN/ports"; done
|
||||
}
|
||||
|
||||
load_ports() {
|
||||
local k changed=""
|
||||
# shellcheck disable=SC1091
|
||||
[ -s "$RUN/ports" ] && . "$RUN/ports"
|
||||
for k in $PORT_KEYS; do
|
||||
if [ -z "${!k:-}" ]; then printf -v "$k" '%s' "$(pick "$(base_of "$k")")"; changed=1; fi
|
||||
done
|
||||
[ -n "$changed" ] && save_ports
|
||||
return 0
|
||||
}
|
||||
|
||||
# A port another process took while the stack was down is reassigned rather than fought over.
|
||||
heal_ports() {
|
||||
local k new
|
||||
for k in $PORT_KEYS; do
|
||||
if listening "${!k}"; then
|
||||
printf -v "$k" '%s' ""
|
||||
new=$(pick "$(base_of "$k")")
|
||||
warn "a port for $k was taken by another process; using $new"
|
||||
printf -v "$k" '%s' "$new"
|
||||
fi
|
||||
done
|
||||
save_ports
|
||||
}
|
||||
|
||||
db_exists() { psql_ -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$DB'" 2>/dev/null | grep -q 1; }
|
||||
db_seeded() { db_exists && [ "$(psql_ -d "$DB" -tAc "SELECT count(*) FROM users" 2>/dev/null || echo 0)" != 0 ]; }
|
||||
|
||||
load_mode() {
|
||||
local saved
|
||||
saved=$(cat "$RUN/mode" 2>/dev/null || true)
|
||||
MODE=${1:-${saved:-lite}}
|
||||
case "$MODE" in lite | full | sandbox) ;; *) die "unknown mode $MODE (lite, full, sandbox)" ;; esac
|
||||
# lite and full share a seed; the sandbox seed is a different organization and account.
|
||||
if [ -n "$saved" ] && [ "$saved" != "$MODE" ] && { [ "$saved" = sandbox ] || [ "$MODE" = sandbox ]; } && db_seeded; then
|
||||
die "$DB holds the $saved seed; $MODE needs a fresh one: stack.sh reset-data $MODE"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── commands derived from the Makefile, so the env stays in step with `make dev` ─
|
||||
rewrite() {
|
||||
local mock=""
|
||||
has sandbox && mock="http://127.0.0.1:$VENDOR"
|
||||
sed \
|
||||
-e "s#/warmbly_dev?#/$DB?#g" \
|
||||
-e "s#API_HOST=0.0.0.0:8080#API_HOST=0.0.0.0:$API#" \
|
||||
-e "s#localhost:8080#localhost:$API#g" \
|
||||
-e "s#APP_URL=http://localhost:5173#APP_URL=http://localhost:$WEB#" \
|
||||
-e "s#CORS_ALLOW_ORIGINS= #CORS_ALLOW_ORIGINS=http://localhost:$WEB #" \
|
||||
-e "s#redis://localhost:16379#redis://localhost:$REDIS#g" \
|
||||
-e "s#nats://localhost:4222#nats://localhost:$NATS#g" \
|
||||
-e "s#SMTP_PORT=11025#SMTP_PORT=$SMTP#g" \
|
||||
-e "s#ws://localhost:4000#ws://localhost:$RT#g" \
|
||||
-e "s#TRACKING_DOMAIN=localhost:3000#TRACKING_DOMAIN=localhost:$TRACK#g" \
|
||||
-e "s#BLOB_FS_ROOT=/tmp/warmbly-blobs#BLOB_FS_ROOT=$RUN/blobs#g" \
|
||||
-e "s#MAILVENDOR_SANDBOX_URL=[^ ]*#MAILVENDOR_SANDBOX_URL=$mock#" \
|
||||
-e "s#go run ./cmd/\\([a-z]*\\)#$BIN/\\1#g"
|
||||
}
|
||||
|
||||
make_cmd() { (cd "$REPO" && make -s -n "$1") | rewrite; }
|
||||
|
||||
# The sandbox reads its own endpoints from the environment (internal/sandbox/config.go).
|
||||
sandbox_env() {
|
||||
echo "MAILPIT_URL=http://localhost:$MAILPIT TRACKING_URL=http://localhost:$TRACK DOVECOT_IMAP_ADDR=localhost:$IMAPS" \
|
||||
"SANDBOX_SMTP_PORT=$SMTP SANDBOX_IMAP_PORT=$IMAPS SANDBOX_VENDOR_ADDR=127.0.0.1:$VENDOR"
|
||||
}
|
||||
|
||||
start() { # name, command
|
||||
rm -f "$RUN/$1.pid"
|
||||
(cd "$REPO" && setsid -f bash -c "echo \$\$ >'$RUN/$1.pid'; $2" >"$RUN/$1.log" 2>&1 </dev/null)
|
||||
for _ in $(seq 1 50); do [ -s "$RUN/$1.pid" ] && return 0; sleep 0.1; done
|
||||
die "$1 did not start; see $RUN/$1.log"
|
||||
}
|
||||
|
||||
wait_for() { # name, check, seconds
|
||||
for _ in $(seq 1 "$3"); do
|
||||
if [ -s "$RUN/$1.pid" ] && ! alive "$1"; then echo "$1 exited:"; tail -30 "$RUN/$1.log"; exit 1; fi
|
||||
eval "$2" >/dev/null 2>&1 && return 0
|
||||
sleep 1
|
||||
done
|
||||
die "$1 was not ready within $3s (logs: stack.sh logs $1, or docker logs $(cname "$1"))"
|
||||
}
|
||||
|
||||
stop_proc() {
|
||||
local pid
|
||||
if alive "$1"; then
|
||||
pid=$(cat "$RUN/$1.pid")
|
||||
kill -TERM -- "-$pid" 2>/dev/null || true
|
||||
for _ in $(seq 1 20); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done
|
||||
kill -KILL -- "-$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$RUN/$1.pid"
|
||||
}
|
||||
|
||||
container() { # name, docker run args...
|
||||
local name=$1
|
||||
shift
|
||||
running "$name" && return 0
|
||||
docker rm -f "$(cname "$name")" >/dev/null 2>&1 || true
|
||||
docker run -d --name "$(cname "$name")" --network "$NET" --label "$LABEL" \
|
||||
--add-host host.docker.internal:host-gateway "$@" >/dev/null
|
||||
}
|
||||
|
||||
# Warns when the image was built before the last commit to its source, so a recording never shows stale code silently.
|
||||
check_image() { # image, source dir
|
||||
local built changed
|
||||
built=$(docker inspect -f '{{.Created}}' "$1" 2>/dev/null) || die "image $1 is missing: cd $REPO && docker compose -p warmbly build ${2%/}"
|
||||
changed=$(cd "$REPO" && git log -1 --format=%cI -- "$2")
|
||||
if [ -n "$changed" ] && [ "$(date -d "$changed" +%s)" -gt "$(date -d "$built" +%s)" ]; then
|
||||
warn "$1 was built before the last change to $2; rebuild: cd $REPO && docker compose -p warmbly build ${2%/}"
|
||||
fi
|
||||
}
|
||||
|
||||
build_bins() {
|
||||
local cmds=(./cmd/backend ./cmd/seed)
|
||||
has full && cmds+=(./cmd/consumer ./cmd/worker)
|
||||
has sandbox && cmds+=(./cmd/sandbox)
|
||||
say "building ${cmds[*]##*/} (one go build, cached between runs)"
|
||||
mkdir -p "$BIN"
|
||||
(cd "$REPO" && go build -o "$BIN/" "${cmds[@]}") || die "go build failed"
|
||||
}
|
||||
|
||||
account() { if has sandbox; then echo sandbox@warmbly.test; else echo dev@warmbly.com; fi; }
|
||||
|
||||
write_env() {
|
||||
cat >"$RUN/env.json" <<JSON
|
||||
{
|
||||
"mode": "$MODE",
|
||||
"webURL": "http://localhost:$WEB",
|
||||
"apiURL": "http://localhost:$API",
|
||||
"mailpitURL": "http://localhost:$MAILPIT",
|
||||
"email": "$(account)",
|
||||
"password": "password123",
|
||||
"database": "$DB"
|
||||
}
|
||||
JSON
|
||||
}
|
||||
|
||||
# Stops the stack once nothing has used it for IDLE_MIN minutes, so a forgotten stack frees its memory.
|
||||
start_watchdog() {
|
||||
[ "$IDLE_MIN" = 0 ] && return 0
|
||||
date -Is >"$RUN/last-used"
|
||||
start watchdog "while sleep 60; do
|
||||
last=\$(stat -c %Y '$RUN/last-used' 2>/dev/null || echo 0)
|
||||
if [ \$(( \$(date +%s) - last )) -ge $((IDLE_MIN * 60)) ]; then
|
||||
echo \"unused for $IDLE_MIN min, stopping\"; rm -f '$RUN/watchdog.pid'; exec bash '$QA/scripts/stack.sh' down
|
||||
fi
|
||||
done"
|
||||
}
|
||||
|
||||
seed() {
|
||||
if has sandbox; then
|
||||
(cd "$REPO" && eval "$(sandbox_env) $(make_cmd sandbox-seed | grep "$BIN/sandbox")")
|
||||
else
|
||||
(cd "$REPO" && eval "$(make_cmd seed)")
|
||||
fi
|
||||
}
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────────────────────
|
||||
up() {
|
||||
local c
|
||||
for c in docker go pnpm setsid curl; do command -v "$c" >/dev/null || die "$c is required"; done
|
||||
load_ports
|
||||
load_mode "${1:-}"
|
||||
if alive backend && alive web; then
|
||||
if [ "$(cat "$RUN/mode" 2>/dev/null)" = "$MODE" ]; then status; return 0; fi
|
||||
say "switching to $MODE"
|
||||
down >/dev/null
|
||||
fi
|
||||
heal_ports
|
||||
echo "$MODE" >"$RUN/mode"
|
||||
|
||||
docker ps --format '{{.Names}}' | grep -q '^warmbly-postgres-1$' || (cd "$REPO" && make infra)
|
||||
until docker exec warmbly-postgres-1 pg_isready -U warmbly >/dev/null 2>&1; do sleep 1; done
|
||||
db_exists || { say "creating database $DB"; psql_ -d postgres -c "CREATE DATABASE $DB" >/dev/null; }
|
||||
[ -d "$REPO/web/node_modules" ] || { say "installing web deps"; (cd "$REPO/web" && pnpm install --frozen-lockfile); }
|
||||
mkdir -p "$RUN/blobs"
|
||||
build_bins
|
||||
|
||||
# Containers reach each other by name on the stack's network; the host reaches them on loopback ports.
|
||||
docker network inspect "$NET" >/dev/null 2>&1 || docker network create --label "$LABEL" "$NET" >/dev/null
|
||||
say "starting redis, nats and mailpit (private to this stack)"
|
||||
container redis -p "127.0.0.1:$REDIS:6379" redis:7-alpine redis-server --save "" --appendonly no
|
||||
container nats -p "127.0.0.1:$NATS:4222" nats:2.10-alpine -js
|
||||
container mailpit -p "127.0.0.1:$MAILPIT:8025" -p "127.0.0.1:$SMTP:1025" \
|
||||
-e MP_SMTP_AUTH_ACCEPT_ANY=1 -e MP_SMTP_AUTH_ALLOW_INSECURE=1 axllent/mailpit:latest
|
||||
has sandbox && container dovecot -p "127.0.0.1:$IMAPS:31993" -e 'USER_PASSWORD={PLAIN}sandbox' dovecot/dovecot:latest
|
||||
wait_for redis "docker exec $(cname redis) redis-cli ping" 30
|
||||
wait_for mailpit "curl -fs localhost:$MAILPIT/api/v1/info" 30
|
||||
|
||||
say "starting backend on :$API (applies migrations on boot)"
|
||||
make_cmd backend >"$RUN/backend.sh"
|
||||
start backend "exec bash '$RUN/backend.sh'"
|
||||
wait_for backend "curl -fs localhost:$API/health" 300
|
||||
|
||||
# Seed on an empty users table, not on a missing database: a failed first run leaves it empty.
|
||||
if ! db_seeded; then
|
||||
if has sandbox; then say "seeding the Sunrise Labs sandbox into $DB"; else say "seeding $DB (rich + full fixtures)"; fi
|
||||
seed >"$RUN/seed.log" 2>&1 || { echo "seed failed:"; tail -30 "$RUN/seed.log"; exit 1; }
|
||||
fi
|
||||
|
||||
if has full; then
|
||||
say "starting consumer, worker and realtime"
|
||||
make_cmd consumer >"$RUN/consumer.sh"
|
||||
start consumer "exec bash '$RUN/consumer.sh'"
|
||||
make_cmd worker >"$RUN/worker.sh"
|
||||
start worker "exec bash '$RUN/worker.sh'"
|
||||
check_image "$REALTIME_IMAGE" realtime/
|
||||
container realtime -p "127.0.0.1:$RT:4000" -e PORT=4000 -e PHX_HOST=localhost -e APP_ENV=dev \
|
||||
-e "DATABASE_URL=postgres://warmbly:warmbly@host.docker.internal:15432/$DB?sslmode=disable" -e DATABASE_SSL=false \
|
||||
-e "REDIS_URL=redis://$(cname redis):6379" -e "JWT_SECRET=$AUTH_SECRET" -e PUBSUB_ENABLED=false \
|
||||
-e SECRET_KEY_BASE=local-development-secret-key-base-minimum-64-characters-for-phoenix -e CHECK_ORIGIN=false \
|
||||
"$REALTIME_IMAGE"
|
||||
fi
|
||||
if has sandbox; then
|
||||
check_image "$TRACKING_IMAGE" tracking/
|
||||
container tracking -p "127.0.0.1:$TRACK:3000" -e APP_ENV=dev -e AWS_CONFIG_ENABLED=false -e TRACKING_HOST=0.0.0.0 \
|
||||
-e TRACKING_PORT=3000 -e EVENTBUS_PROVIDER=nats -e "NATS_URL=nats://$(cname nats):4222" -e CODEC_PROVIDER=json \
|
||||
-e "BACKEND_INTERNAL_URL=http://host.docker.internal:$API" -e "INTERNAL_API_TOKEN=$INTERNAL_TOKEN" "$TRACKING_IMAGE"
|
||||
say "starting the simulator (delivers, opens, clicks and replies like the internet)"
|
||||
make_cmd sandbox-simulate | sed "s#$BIN/sandbox#$(sandbox_env) $BIN/sandbox#" >"$RUN/simulator.sh"
|
||||
start simulator "exec bash '$RUN/simulator.sh'"
|
||||
fi
|
||||
|
||||
say "starting dashboard on :$WEB"
|
||||
start web "cd web && export VITE_APP_URL=http://localhost:$WEB VITE_API_URL=http://localhost:$API VITE_TURNSTILE_KEY=1x00000000000000000000AA VITE_TURNSTILE_BYPASS_TOKEN=warmbly-local-turnstile-bypass; exec pnpm dev --port $WEB --strictPort"
|
||||
wait_for web "curl -fs localhost:$WEB" 120
|
||||
if has full; then wait_for realtime "curl -fs localhost:$RT/health" 90; fi
|
||||
if has sandbox; then wait_for tracking "curl -fs localhost:$TRACK/health" 60; fi
|
||||
|
||||
write_env
|
||||
start_watchdog
|
||||
status
|
||||
}
|
||||
|
||||
down() {
|
||||
local n
|
||||
for n in watchdog simulator web worker consumer backend; do stop_proc "$n"; done
|
||||
docker ps -aq --filter "label=$LABEL" | xargs -r docker rm -f >/dev/null
|
||||
docker network rm "$NET" >/dev/null 2>&1 || true
|
||||
rm -f "$RUN/env.json"
|
||||
say "stopped (data kept in $DB)"
|
||||
}
|
||||
|
||||
rss_mb() { ps -o rss= -g "$(cat "$RUN/$1.pid")" 2>/dev/null | awk '{s+=$1} END {printf "%d", s/1024}'; }
|
||||
|
||||
status() {
|
||||
local n mb total=0 name mem ids idle=off
|
||||
load_ports
|
||||
MODE=$(cat "$RUN/mode" 2>/dev/null || echo lite)
|
||||
printf ' mode %s\n\n' "$MODE"
|
||||
for n in backend web consumer worker simulator; do
|
||||
case $n in consumer | worker) has full || continue ;; simulator) has sandbox || continue ;; esac
|
||||
if alive "$n"; then
|
||||
mb=$(rss_mb "$n"); total=$((total + mb))
|
||||
printf ' %-10s up %5s MB\n' "$n" "$mb"
|
||||
else
|
||||
printf ' %-10s down\n' "$n"
|
||||
fi
|
||||
done
|
||||
ids=$(docker ps -q --filter "label=$LABEL")
|
||||
if [ -n "$ids" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
while read -r name mem; do
|
||||
mb=$(awk -v m="$mem" 'BEGIN { v = m + 0; if (m ~ /GiB/) v *= 1024; else if (m ~ /KiB/) v /= 1024; printf "%d", v }')
|
||||
total=$((total + mb))
|
||||
printf ' %-10s up %5s MB container\n' "${name#"wqa-$slug-"}" "$mb"
|
||||
done < <(docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' $ids | awk '{print $1, $2}')
|
||||
fi
|
||||
[ "$IDLE_MIN" != 0 ] && idle="after $IDLE_MIN min unused"
|
||||
cat <<INFO
|
||||
|
||||
total ~$total MB (auto-stop: $idle)
|
||||
|
||||
Dashboard http://localhost:$WEB/app/emails
|
||||
API http://localhost:$API
|
||||
Mailpit http://localhost:$MAILPIT (this stack's mail and login codes)
|
||||
Login $(account) / password123
|
||||
Database $DB
|
||||
INFO
|
||||
}
|
||||
|
||||
ls_stacks() {
|
||||
local out
|
||||
out=$(docker ps --filter label=warmbly-qa --format '{{.Label "warmbly-qa"}}' | sort | uniq -c)
|
||||
if [ -z "$out" ]; then echo " no QA stacks running"; else echo "$out" | awk '{printf " %s (%s containers)\n", $2, $1}'; fi
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
up) up "${2:-}" ;;
|
||||
down) down ;;
|
||||
restart)
|
||||
mode=$(cat "$RUN/mode" 2>/dev/null || echo lite)
|
||||
down
|
||||
up "$mode"
|
||||
;;
|
||||
status) status ;;
|
||||
logs) if [ -n "${2:-}" ]; then tail -f "$RUN/$2.log"; else tail -f "$RUN"/*.log; fi ;;
|
||||
reset-data)
|
||||
target=${2:-$(cat "$RUN/mode" 2>/dev/null || echo lite)}
|
||||
down
|
||||
psql_ -d postgres -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" >/dev/null
|
||||
rm -rf "$QA/.artifacts/auth" "$RUN/blobs" "$RUN/mode"
|
||||
say "dropped $DB"
|
||||
up "$target"
|
||||
;;
|
||||
ls) ls_stacks ;;
|
||||
*) sed -n '2,20p' "$0"; exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"erasableSyntaxOnly": true,
|
||||
"skipLibCheck": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["lib", "flows", "scripts", "playwright.config.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user