diff --git a/.dockerignore b/.dockerignore index 2989c715..9d14a99d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,14 +11,17 @@ .vscode .agentd -# Frontend bits — the web service bind-mounts ./web directly, so we -# never want the heavyweight frontend tree to land in the Go build -# context. -web/node_modules -web/.vite +# Frontend trees: the Go images never need them, and a host node_modules +# copied over the forms image's own install makes pnpm abort the build (#292). +**/node_modules +**/.vite web/dist web/build web/coverage +forms/dist +admin/dist +site/dist +site/.astro # Compiled host binaries that would shadow / inflate the context. bin diff --git a/.env.example b/.env.example index 31477c50..318098fd 100644 --- a/.env.example +++ b/.env.example @@ -460,6 +460,35 @@ BILLING_PROVIDER=none # WORKER_INSTALLER_PATH= +# === Updates ================================================================== +# +# The admin panel's top bar shows the running version and turns into an +# "Update available" pill when a newer release exists. The backend checks +# GitHub Releases on the interval below; the check is one unauthenticated API +# read and can be turned off. Full page: https://docs.warmbly.com/development/updates/ +# UPDATE_CHECK_ENABLED=true +# UPDATE_CHECK_INTERVAL=30m # minimum 5m +# UPDATE_CHANNEL=stable # stable | dev (dev also offers prereleases) +# RELEASES_GITHUB_REPO=warmbly/warmbly # point a fork's instance at the fork +# RELEASES_GITHUB_TOKEN= # optional; only raises the API rate limit +# +# "Update and restart" in that pill needs the updater: a sidecar that pulls this +# checkout, rebuilds the images and recreates the containers, then waits for the +# backend to answer again. It holds the docker socket (root on this host), so it +# runs only under the "updater" compose profile. `make up` enables the profile; +# for a plain `docker compose up`, uncomment the next line. Comment it out to +# update by hand with `git pull && make up` instead. +# COMPOSE_PROFILES=updater +# +# The backend reaches it at UPDATER_URL with UPDATER_TOKEN (defaults to +# INTERNAL_API_TOKEN). UPDATER_URL=none makes the panel report-only. +# UPDATER_URL=http://updater:8095 +# UPDATER_TOKEN= +# UPDATER_FETCH_INTERVAL=30m # how often the updater git-fetches for the "commits behind" count +# UPDATER_PRUNE=true # docker image prune after a successful update +# UPDATER_ALLOW_DIRTY=false # let an update run over local modifications in the checkout +# WARMBLY_REPO_DIR= # the checkout's absolute path; defaults to $PWD at compose time + # === Tracking service (Rust, open/click) ====================================== # # The shipped docker-compose.yml pins the listen address, so these two only take diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2419e25f..b9760d31 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: service: - description: "Service to build (all, backend, consumer, worker, forms, tracking, realtime)" + description: "Service to build (all, backend, consumer, worker, forms, updater, tracking, realtime)" required: false default: "all" @@ -61,6 +61,15 @@ jobs: - 'cmd/forms/**' - 'forms/**' - 'deploy/docker/forms.Dockerfile' + updater: + - 'go.mod' + - 'go.sum' + # internal/** and not internal/updater/**: the binary links + # internal/version and whatever else it grows, and an image that + # silently skips a rebuild is worse than one rebuilt too often. + - 'internal/**' + - 'cmd/updater/**' + - 'deploy/docker/updater.Dockerfile' tracking: - 'tracking/**' realtime: @@ -75,6 +84,7 @@ jobs: CONSUMER: ${{ steps.filter.outputs.consumer }} WORKER: ${{ steps.filter.outputs.worker }} FORMS: ${{ steps.filter.outputs.forms }} + UPDATER: ${{ steps.filter.outputs.updater }} TRACKING: ${{ steps.filter.outputs.tracking }} REALTIME: ${{ steps.filter.outputs.realtime }} run: | @@ -82,7 +92,7 @@ jobs: native="" if [ "$EVENT" = "workflow_dispatch" ]; then sel="${SELECTED:-all}" - for s in backend consumer worker forms; do + for s in backend consumer worker forms updater; do if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then go="$go\"$s\","; fi done for s in tracking realtime; do @@ -93,6 +103,7 @@ jobs: if [ "$CONSUMER" = "true" ]; then go="$go\"consumer\","; fi if [ "$WORKER" = "true" ]; then go="$go\"worker\","; fi if [ "$FORMS" = "true" ]; then go="$go\"forms\","; fi + if [ "$UPDATER" = "true" ]; then go="$go\"updater\","; fi if [ "$TRACKING" = "true" ]; then native="$native\"tracking\","; fi if [ "$REALTIME" = "true" ]; then native="$native\"realtime\","; fi fi @@ -132,6 +143,10 @@ jobs: context: . file: deploy/docker/${{ matrix.service }}.Dockerfile push: true + build-args: | + VERSION=dev-${{ github.sha }} + COMMIT=${{ github.sha }} + BUILT_AT=${{ github.event.head_commit.timestamp }} tags: | ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d119212..0d3b8cf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: forms: ${{ steps.filter.outputs.forms }} make: ${{ steps.filter.outputs.make }} ios: ${{ steps.filter.outputs.ios }} + installer: ${{ steps.filter.outputs.installer }} + cli-installer: ${{ steps.filter.outputs.cli-installer }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 @@ -70,6 +72,16 @@ jobs: - 'integrations/make/**' ios: - 'ios/**' + installer: + - 'site/public/install.sh' + - 'site/public/install.sh.sha256' + - 'scripts/check-installer.sh' + cli-installer: + - 'site/public/cli.sh' + - 'site/public/cli.sh.sha256' + - 'site/public/cli.ps1' + - 'scripts/check-cli-installer.sh' + - 'scripts/build-cli.sh' migrations-ci: name: Migrations @@ -229,6 +241,37 @@ jobs: - name: Build run: pnpm build + # The installer is served verbatim from site/public, so what CI checks is + # exactly what a `curl | sh` executes. The checksum is published next to it + # for anyone who would rather download, verify and read before running, and + # it is only useful if it cannot drift. + installer-ci: + name: Installer CI + needs: changes + if: needs.changes.outputs.installer == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check the installer + run: ./scripts/check-installer.sh + + cli-installer-ci: + name: CLI Installer CI + needs: changes + if: needs.changes.outputs.cli-installer == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # dash is what /bin/sh is on Debian and Ubuntu, which is what most people + # will pipe this into. Ubuntu runners already have shellcheck and pwsh. + - name: Install dash + run: sudo apt-get update && sudo apt-get install -y dash + + - name: Check the CLI installer + run: ./scripts/check-cli-installer.sh + make-ci: name: Make App CI needs: changes @@ -398,7 +441,7 @@ jobs: ci-status: name: CI Status runs-on: ubuntu-latest - needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] + needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] if: always() steps: - name: Check CI status diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cbace67e..38692006 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - service: [backend, consumer, worker] + service: [backend, consumer, worker, forms, updater, cli] runs-on: ubuntu-latest permissions: contents: read @@ -60,6 +60,10 @@ jobs: context: . file: deploy/docker/${{ matrix.service }}.Dockerfile push: true + build-args: | + VERSION=${{ github.ref_name }} + COMMIT=${{ github.sha }} + BUILT_AT=${{ github.event.head_commit.timestamp }} tags: | ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.ref_name }} ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:v${{ needs.validate-tag.outputs.minor }} @@ -210,17 +214,95 @@ jobs: -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:prod \ $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *) + # The `warmbly` CLI is a plain static binary, so it cross-compiles for every + # platform on one runner. + # + # Assets are named WITHOUT the version, so + # releases/latest/download/warmbly_linux_amd64.tar.gz always resolves. That is + # what lets the install script find the newest build with no GitHub API call, + # which matters because the unauthenticated API is rate limited and a curl + # installer that fails on a busy CI runner is not an installer. + build-cli: + name: Build CLI + needs: validate-tag + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Cross-compile and package + env: + VERSION: ${{ github.ref_name }} + COMMIT: ${{ github.sha }} + BUILT_AT: ${{ github.event.head_commit.timestamp }} + run: | + set -euo pipefail + ./scripts/build-cli.sh dist + + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: warmbly-cli + path: dist/ + retention-days: 1 + create-release: name: Create GitHub Release - needs: [validate-tag, build-go, build-frontend, merge-native] + needs: [validate-tag, build-go, build-frontend, merge-native, build-cli] runs-on: ubuntu-latest permissions: contents: write + # The manifest step logs in to GHCR and inspects each published image; + # without read access that fails on a private package. + packages: read steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Download the CLI binaries + uses: actions/download-artifact@v4 + with: + name: warmbly-cli + path: /tmp/cli + + # The installer verifies what it pulled against this file, so it is what + # makes "curl | sh" checkable after the fact rather than only before it. + # One line per service, because the thing that reads it is a POSIX shell. + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish the image manifest + env: + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + { + printf '{\n' + printf ' "tag": "%s",\n' "$TAG" + printf ' "registry": "%s",\n' "$IMAGE_PREFIX" + printf ' "images": {\n' + first=1 + for s in backend consumer worker forms updater web admin tracking realtime; do + digest=$(docker buildx imagetools inspect "$IMAGE_PREFIX/$s:$TAG" \ + --format '{{json .Manifest.Digest}}' | tr -d '"') + [ "$first" = 1 ] || printf ',\n' + first=0 + printf ' "%s": "%s"' "$s" "$digest" + done + printf '\n }\n}\n' + } > /tmp/images.json + cat /tmp/images.json + - name: Generate release notes env: GITHUB_SERVER_URL: ${{ github.server_url }} @@ -236,6 +318,35 @@ jobs: cat /tmp/changelog.md cat <<'EOF' + ## Install + + ``` + curl -fsSL https://warmbly.com/install.sh | sh -s -- --version ${{ github.ref_name }} + ``` + + Add `--wizard` to be asked where each store lives, what is kept and + for how long, and how it is backed up. + + ## The warmbly CLI + + ``` + curl -fsSL https://warmbly.com/cli.sh | sh # macOS, Linux + irm https://warmbly.com/cli.ps1 | iex # Windows + brew install warmbly/tap/warmbly # Homebrew + scoop install warmbly # Scoop + ``` + + Or take an archive below and unpack it yourself; `checksums.txt` + verifies every one of them. Already installed? `warmbly upgrade`. + + `images.json` below lists the manifest digest of every image in this + release. The installer checks what it pulled against it, and you can + too: + + ``` + docker image inspect ${{ env.IMAGE_PREFIX }}/backend:${{ github.ref_name }} --format '{{index .RepoDigests 0}}' + ``` + ## Docker Images All images are available at `ghcr.io/${{ github.repository_owner }}/warmbly`: @@ -249,6 +360,9 @@ jobs: | Realtime | `${{ env.IMAGE_PREFIX }}/realtime:${{ github.ref_name }}` | | Dashboard (web) | `${{ env.IMAGE_PREFIX }}/web:${{ github.ref_name }}` | | Admin | `${{ env.IMAGE_PREFIX }}/admin:${{ github.ref_name }}` | + | Forms | `${{ env.IMAGE_PREFIX }}/forms:${{ github.ref_name }}` | + | Updater | `${{ env.IMAGE_PREFIX }}/updater:${{ github.ref_name }}` | + | CLI | `${{ env.IMAGE_PREFIX }}/cli:${{ github.ref_name }}` | ## Deployment @@ -256,11 +370,50 @@ jobs: EOF } > /tmp/release-body.md + # The formula and manifest are generated with the archives, so their + # checksums can never drift from what they describe. Pushing them is + # skipped, loudly, when the tap token is not configured: a release must + # not fail because a downstream package repo is not set up yet. + - name: Publish the Homebrew formula and Scoop manifest + env: + TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + if [ -z "${TAP_TOKEN:-}" ]; then + echo "HOMEBREW_TAP_TOKEN is not set; skipping the tap push." + echo "The formula and manifest are still attached to the release." + exit 0 + fi + # A prerelease must never become the default `brew install`. + case "$TAG" in + *-*) echo "$TAG is a prerelease; not updating the taps."; exit 0 ;; + esac + git config --global user.name "warmbly-release" + git config --global user.email "release@warmbly.com" + git clone --depth 1 \ + "https://x-access-token:${TAP_TOKEN}@github.com/warmbly/homebrew-tap.git" /tmp/tap + mkdir -p /tmp/tap/Formula /tmp/tap/bucket + cp /tmp/cli/warmbly.rb /tmp/tap/Formula/warmbly.rb + cp /tmp/cli/warmbly.json /tmp/tap/bucket/warmbly.json + cd /tmp/tap + git add Formula/warmbly.rb bucket/warmbly.json + if git diff --cached --quiet; then + echo "the tap already describes $TAG" + else + git commit -m "warmbly $TAG" + git push + echo "pushed warmbly $TAG to the tap" + fi + - name: Create Release uses: softprops/action-gh-release@v2 with: tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} body_path: /tmp/release-body.md + files: | + /tmp/images.json + /tmp/cli/* draft: false prerelease: ${{ contains(github.ref_name, '-') }} diff --git a/.gitignore b/.gitignore index e3561e75..60c36b64 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,12 @@ /worker /backend /seed +/forms +/migrate +/updater +/warmblyctl +/warmbly +/cli # Test binary, built with `go test -c` *.test @@ -24,6 +30,10 @@ profile.cov # Dependency directories (remove the comment below to include it) vendor/ +# Local CLI builds (make warmbly / make warmbly-dist) +/bin/ +/dist/ + # Go workspace file go.work go.work.sum @@ -100,3 +110,4 @@ build-errors.log # Xcode per-user state (ios/) xcuserdata/ .DS_Store +/.worktrees/ diff --git a/AGENTS.md b/AGENTS.md index e5de9870..7dcf1212 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,15 @@ At a product level, the app does four main things: The backend API is the control plane. Workers are the execution plane. +It ships as a hosted service and as a self-host, and the two are the same code. +The front door for the self-host is one command, +`curl -fsSL https://warmbly.com/install.sh | sh`, which pulls the published +release images and needs no clone and no compiler. `--wizard` turns it into an +interactive install that asks the data-control questions up front: where each +store lives, what is kept and for how long, how it is backed up. The script is +`site/public/install.sh` and it has its own rules below; the docs are +`docs/content/docs/development/install.mdx` and `data-control.mdx`. + ## Working In This Repo CI is strict. `go build ./...` succeeding is not enough — `golangci-lint` runs `gofmt` as part of its checks, and a single unformatted import block or mis-indented doc comment will fail the PR even when the code compiles cleanly. Before declaring any Go change done: @@ -39,6 +48,7 @@ Docs stay in sync: - the customer docs site lives in `docs/` (Fumadocs, served at docs.warmbly.com); content is MDX under `docs/content/docs/` in three sections: `guides/` (product behavior), `learn/` (fundamentals), `api/` (API reference) - any change that alters user-visible behavior must update the matching docs page in the same change: a new or changed endpoint updates `api/endpoints.mdx` (scope map) and, where relevant, `api/authentication.mdx`; a new or changed API permission updates `api/permissions.mdx` including the permission table, presets, and all three language tabs in the constants section; a new or changed error code updates `api/error-codes.mdx`; a new or changed product feature, default, limit, or setting updates the relevant `guides/` page (or adds one, registered in `guides/meta.json` under the right section group) - removing or renaming a feature, endpoint, or permission means removing or updating its docs too; do not leave stale docs behind +- self-hosting behavior has its own pages under `docs/content/docs/development/`: a change to the installer or to what it asks updates `install.mdx`; a change to where a store lives, how long something is kept, or how an instance is backed up or moved updates `data-control.mdx`; a new environment variable updates `configuration.mdx`, and a new database-backed setting updates its table there as well as the admin panel - follow the docs conventions: frontmatter `title` is the H1 (no `#` heading in the body), no decorative sidebar icons (pages and `meta.json` sections carry no `icon`; the source loader has the lucide icon plugin disabled, and code-sample tabs use the real language logo instead), sentence-case headings, no em dashes in prose, internal links use trailing slashes (`/guides/mailboxes/`) - verify with `pnpm types:check` and `pnpm lint` in `docs/` (the site is a fully static export; `pnpm build` writes `out/`) @@ -81,6 +91,69 @@ Rows move as `jsonb` in both directions, so adding a *column* to an existing tab The same applies to the customer-facing side of a feature: if it stores org data, its docs page and `docs/content/docs/guides/workspace-export-import.mdx` should agree about whether that data moves. +### The installer is a published artifact + +`site/public/install.sh` is the one-command self-host installer, served +verbatim from the static site at `https://warmbly.com/install.sh`. What is in +the repo is byte for byte what a stranger pipes into their shell, which makes +it the highest-consequence file here that is not Go. + +It is a wizard: an animated stepper, arrow-key and vim menus, live pull and +health screens, a review pass, and a `--demo` mode that plays the whole thing +while installing nothing. `docs/content/docs/development/install.mdx` documents +it and `data-control.mdx` documents what its questions decide; the +`warmbly-install` skill is the agent-facing version. + +Rules, all of them learned from breaking them: + +- **POSIX sh, not bash.** It runs under whatever `/bin/sh` the host has, which + on Debian and Ubuntu is dash. A `sh -n` that passes under your own shell + proves nothing about that; `make installer-check` runs `dash -n` and + `shellcheck -s sh` +- **`set -eu`, everything in a function, `main "$@"` on the last line**, so a + truncated download executes nothing. Watch for `[ x ] && y` as a function's + LAST command: it returns non-zero when the test fails, and under `set -e` + that ends the run. Use an `if`, or end with `return 0` +- **Nothing drawn inside a redraw loop may be wider than the terminal.** A + wrapped line is two physical rows while every `ESC[nA` counts logical ones, + so one long option hint makes the menu draw over itself and over whatever was + on screen before it. Everything in a loop goes through `fit` +- **The screen is not ours.** It appends by default, `--clear` is opt-in, and + `ESC[3J` (erase scrollback) is never sent +- **Regenerate the checksum.** `site/public/install.sh.sha256` is what makes + "download, verify, read, run" a real alternative to piping into a shell. + `make installer-sha`, and CI fails when the two disagree +- **Every answer is a flag and a `WARMBLY_*` variable.** An install that can + only be driven by keyboard cannot be driven by Ansible, cloud-init or an + agent, and the wizard exists to be optional +- **Idempotent.** A second run adopts the existing `.env`, never regenerates a + secret (a new `CREDENTIALS_ENCRYPTION_KEY` is permanent data loss) and never + moves an existing data root + +Run `make installer-check` before pushing a change to it (POSIX parse, +shellcheck, `--help`, `--demo`, `--print-env`, a compose file per answer shape, +a pty width regression test, and the checksum). `make installer-demo` is how +you see a UI change without installing anything. + +`site/public/cli.sh` is the second published script, served at +`https://warmbly.com/cli.sh`, and it installs the `warmbly` CLI rather than an +instance. Every rule above applies to it, plus two of its own: + +- **It verifies what it downloads.** The release publishes `checksums.txt` + next to the archives, and a mismatch installs nothing rather than warning. + Never weaken that to a warning +- **Release assets are named without the version**, so + `releases/latest/download/warmbly__.tar.gz` resolves with no + GitHub API call. The unauthenticated API is rate limited per IP, which is + what breaks a curl installer on a shared runner. `scripts/build-cli.sh` and + the platform list in `cli.sh` have to agree; `make cli-check` fails when they + do not + +`make cli-check` runs the whole thing (POSIX parse, shellcheck, `--help`, +`--dry-run`, a real install from a local mirror, checksum tampering, uninstall, +the PowerShell parse and the checksum), and `make cli-sha` regenerates the +checksum after any edit. `site/public/cli.ps1` is the Windows half. + ### Verification: what to run, what to skip Keep the loop fast. The signals that matter are formatting, lint, and typecheck — not local builds or browser automation. @@ -92,6 +165,11 @@ Always, before calling a Go change done: For frontend changes, run `pnpm typecheck` and `pnpm lint` in any tree you touched. +For a change to `site/public/install.sh`, run `make installer-check`; it is the +same script CI runs and it regenerates nothing, so a stale checksum fails there +exactly as it will in CI. For `site/public/cli.sh` or `cli.ps1`, the equivalent +is `make cli-check` (and `make cli-sha` after any edit). + Do not: - do not run `go build ./...`, `pnpm build`, or docker image builds as a "did it work" check. They are slow and are not what CI gates on. `go run` (via the make dev targets) already compiles; `make fmt` + `make lint` + `pnpm typecheck` are the real signals. @@ -116,6 +194,8 @@ Infra runs in docker; the Go services and frontends run natively on the host for - `make web` / `make admin` / `make site` — frontend dev servers (5173 / 5174 / 4321), pointed at the native backend. - `make seed` — load fixtures (after the backend has applied migrations). - `make fmt` / `make lint` — format and lint Go. +- `make installer-demo` — walk the self-host installer's wizard with nothing installed: the real questions and review, a played pull and start. No Docker, no network, no file written. `WARMBLY_DEMO_FAST=1` collapses the animations while iterating on them. +- `make installer-check` / `make installer-sha` — everything CI runs against `site/public/install.sh`, and the checksum regeneration that has to follow any edit to it. Prefer native `make backend` over rebuilding the docker backend image: docker rebuilds are slow because the image bakes in the migrations and the compiled binary, so a one-line change means a full image build + container recreate. The native targets skip all of that. The dockerized hot-reload flow (`make app`) and prod-image smoke test (`make up`) remain available when you specifically need containers. @@ -197,16 +277,18 @@ API keys with the `REALTIME_SUBSCRIBE` permission (bit 11) can connect to the sa - `cmd/backend`: API and business orchestration - `cmd/consumer`: consumes Kafka events and updates platform state - `cmd/worker`: execution worker for send/sync operations +- `cmd/cli`: the `warmbly` CLI, the customer-facing one. A signed-in, multi-host client of the public REST API (`internal/cli/*` holds its config, HTTP client and renderers). It never serves HTTP and never touches Postgres; `cmd/warmblyctl` is the operator's CLI and keeps the database half. The directory is `cli` and the binary is `warmbly`, so every build target names its output explicitly (`-o warmbly`), and `go install` needs the rename documented in `docs/content/docs/api/cli.mdx` - `cmd/forms`: the public face of hosted lead-capture forms (`internal/formserver`): serves the built `forms/` app, per-form page shells (with the per-form embed CSP), the embed loader and public submissions on their own origin (`FORMS_DOMAIN`). No database; the backend's internal API is its only dependency, like the tracking service - `forms/`: the public form page app (React + TanStack Router/Query/Form, Vite CSR build). Renders a published form from the same-origin `/api/forms/:publicID`, submits to `/api/forms/:publicID/submit`; the Go forms service hosts the build - `tracking/`: open and click tracking service - `realtime/`: websocket fanout service - `web/`: in-product frontend (dashboard). Customer-facing only: it holds no platform-admin screens, and operator tooling must not be added back here - `admin/`: platform admin panel (:5174), the single operator surface. Workers, users, orgs, warmup, campaigns, analytics, audit. Every route sits behind `RequireAdmin` and the backend's `RequireAdminPermission` gates -- `site/`: public marketing site (Astro 5 + Tailwind v4) +- `site/`: public marketing site (Astro 5 + Tailwind v4). `site/public/install.sh` is the self-host installer served at warmbly.com/install.sh and `site/public/cli.sh` is the CLI installer served at warmbly.com/cli.sh (with `cli.ps1` for Windows), each with its checksum next to it; see the rules above before touching either - `deploy/`: production deploy manifests, infrastructure, and runtime config - `docs/`: documentation site (docs.warmbly.com); product guides, API reference, and self-hosting/engineering docs under `content/docs/development/` -- `scripts/`: one-off tooling (codegen, migrations, local dev utilities) +- `scripts/`: one-off tooling (codegen, migrations, installer checks, local dev utilities) +- `skills/`: agent playbooks shipped with the repo (`warmbly-cli` for the `warmbly` CLI, `warmbly-api` for the same product surface through `warmblyctl`, `warmbly-ops` for instance administration, `warmbly-install` for standing an instance up and moving it). A command an operator can run is not usable by an agent until it is in one of these ## Worker Topology @@ -787,6 +869,8 @@ These files are the fastest way to rebuild context: - `README.md` - `docs/content/docs/development/architecture.mdx` +- `docs/content/docs/development/install.mdx` and `data-control.mdx` (what a self-hoster is asked, and what each answer decides) +- `site/public/install.sh` (the installer itself) - `cmd/worker/main.go` - `internal/app/worker/assignment.go` - `internal/tasks/email_task.go` diff --git a/Makefile b/Makefile index a3a2ec9d..4481f169 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,21 @@ export PATH := $(GO_BIN):$(PATH) # fresh clones without any environment setup. COMPOSE := docker compose -p warmbly +# Build identity stamped into the Go images (shown in the admin panel's top bar +# and read by the update check). Empty outside a git checkout, which the +# binaries report as "dev". +export WARMBLY_BUILD_VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null) +export WARMBLY_BUILD_COMMIT ?= $(shell git rev-parse HEAD 2>/dev/null) +export WARMBLY_BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +# The updater sidecar (one-click "Update and restart" from the admin panel) +# holds the docker socket, so it lives behind a compose profile. `make up` +# turns it on; UPDATER=false leaves it off. +UPDATER ?= true +ifeq ($(UPDATER),true) +UP_PROFILES := --profile updater +endif + GOLANGCI_LINT_VERSION ?= v1.64.8 PROTOC_GEN_GO_VERSION ?= v1.36.11 PROTOC_GEN_GO_GRPC_VERSION ?= v1.6.1 @@ -23,10 +38,11 @@ PROTO_DIR := internal/tasks/proto PROTO_GEN_FILES := $(PROTO_DIR)/tasks.pb.go .PHONY: poollink-dev poollink-dev-down poollink-dev-reset setup-tools fmt lint check-migrations proto check-proto \ - up claim doctor cli seed-demo seed seed-plan sandbox sandbox-seed sandbox-simulate reset logs status stop down test-seed \ + up upgrade claim doctor cli seed-demo seed seed-plan sandbox sandbox-seed sandbox-simulate reset logs status stop down test-seed \ restart restart-go restart-all infra infra-down app app-down app-logs \ backend forms forms-web consumer worker run dev tracking realtime web \ - admin site docs grant-admin revoke-admin gen-key db-reset db-wipe migrate + admin site docs grant-admin revoke-admin gen-key installer-sha installer-check installer-demo \ + db-reset db-wipe migrate warmbly warmbly-dist cli-sha cli-check setup-tools: @echo "Installing required Go tools into $(GO_BIN)" @@ -34,6 +50,33 @@ setup-tools: GOBIN=$(GO_BIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@$(PROTOC_GEN_GO_VERSION) GOBIN=$(GO_BIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$(PROTOC_GEN_GO_GRPC_VERSION) +# Build the `warmbly` CLI into ./bin, stamped with this checkout's version so +# `warmbly version` reports something meaningful. This is the customer CLI; the +# operator one (warmblyctl) ships in the backend image and runs there. +warmbly: + @mkdir -p bin + go build -ldflags="-s -w \ + -X github.com/warmbly/warmbly/internal/version.Version=$(WARMBLY_BUILD_VERSION) \ + -X github.com/warmbly/warmbly/internal/version.Commit=$(WARMBLY_BUILD_COMMIT) \ + -X github.com/warmbly/warmbly/internal/version.BuiltAt=$(WARMBLY_BUILD_TIME)" \ + -o bin/warmbly ./cmd/cli + @echo "built bin/warmbly ($(WARMBLY_BUILD_VERSION))" + @echo "put it on your PATH: sudo install -m 0755 bin/warmbly /usr/local/bin/warmbly" + +# Everything a release publishes for the CLI: an archive per platform, the +# checksums, and the Homebrew and Scoop manifests. Same script the release +# workflow runs, so an artifact can be reproduced locally. +warmbly-dist: + ./scripts/build-cli.sh dist + +# The published installer at https://warmbly.com/cli.sh. Regenerate the +# checksum after any edit to it; CI fails when the two disagree. +cli-sha: + @cd site/public && sha256sum cli.sh > cli.sh.sha256 && cat cli.sh.sha256 + +cli-check: + @./scripts/check-cli-installer.sh + # Format all Go code. CI's golangci-lint enforces gofmt, so this is the # formatting signal to run before committing, not `go build`. fmt: @@ -79,7 +122,7 @@ ADMIN_URL = http://$(WEB_HOST):5174 # everything detached. Dashboard :5173, admin :5174, API :8080. up: @command -v docker >/dev/null || { echo "docker is required: https://docs.docker.com/get-docker/"; exit 1; } - $(COMPOSE) up -d --build + $(COMPOSE) $(UP_PROFILES) up -d --build @echo "" @echo "Warmbly is starting. The first run builds the images once." @echo "" @@ -88,6 +131,12 @@ up: @echo " Health: make doctor Logs: make logs" @echo " Demo data: make seed-demo Guide: https://docs.warmbly.com/development/first-run/" +# The by-hand equivalent of "Update and restart" in the admin panel: move the +# checkout forward, rebuild, recreate what changed. Migrations apply on boot. +upgrade: + git pull --ff-only + @$(MAKE) --no-print-directory up + # Report how to get into this instance, and end on a command that works. # # State is queried FIRST (warmblyctl, falling back to a direct count), so an @@ -605,6 +654,7 @@ forms-web: consumer: $(GO_DEV_ENV) \ $(AI_DEV_ENV) \ + GEODB_PATH=data/GeoLite2-City.mmdb \ go run ./cmd/consumer # Send/sync worker. No Postgres by design. WORKER_ID is an explicit UUID @@ -642,6 +692,24 @@ run: gen-key: @openssl rand -base64 32 +# The one-command installer, served verbatim at https://warmbly.com/install.sh. +# The published checksum is what makes "download, verify, read, run" a real +# alternative to piping into a shell, so it is regenerated with the script and +# CI fails when the two disagree. +installer-sha: + @cd site/public && sha256sum install.sh > install.sh.sha256 && cat install.sh.sha256 + +# Everything CI runs against the installer: POSIX parse, shellcheck, --help, +# --print-env, a compose file per answer shape, and the checksum. +installer-check: + @./scripts/check-installer.sh + +# Walk the installer's wizard without installing anything: the real questions, +# the real review, and a played pull and start. Writes no file, pulls no image, +# needs no Docker. This is the "what does it look like" target. +installer-demo: + @sh site/public/install.sh --demo + # ─── one-command dev stack ─────────────────────────────────────────────── # # `make dev` is the "just make it work" target for a fresh clone or a fresh diff --git a/README.md b/README.md index a0d73998..5533235d 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,29 @@ the native services, and how seeding works are in the Warmbly runs with **no cloud account of any kind**: no AWS, no GCP, no Stripe, no Kafka. One command brings up the whole platform on local, open-source pieces: +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +That pulls the published release images, writes a real `.env`, starts the stack +and prints the one-time link that claims the instance and makes you its admin. +No clone, no compiler, under two minutes. Add `--wizard` and it asks where each +store lives, what is kept and for how long, and how it is backed up. The script +is checksummed at `install.sh.sha256` if you would rather download, verify and +read it first, `--dry-run` prints every file it would write without touching +anything, and `--demo` walks the whole wizard while installing nothing. Full +flag list: [install](https://docs.warmbly.com/development/install/). + +To build it from source instead: + ```bash git clone https://github.com/warmbly/warmbly && cd warmbly make up ``` -That is the whole install: `make up` waits for the backend and prints a one-time -link that claims the instance and makes you its admin. You need Docker with -Compose v2 and about 10 GB of free disk. If anything looks wrong, `make doctor` -prints the instance state and every failing check. +Either way you need Docker with Compose v2; the build path also wants about +10 GB of free disk. If anything looks wrong, `make doctor` prints the instance +state and every failing check. From here the [self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) @@ -134,7 +148,9 @@ The full docs live at **[docs.warmbly.com](https://docs.warmbly.com)**. | Read this | To learn | |-----------|----------| -| [Self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) | Step-by-step install, then production, backups, and scaling the worker fleet | +| [Install](https://docs.warmbly.com/development/install/) | The one-command install and its wizard: where the data lives, what is kept, how it is backed up | +| [Self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) | Building from source, then production, backups, and scaling the worker fleet | +| [Data control](https://docs.warmbly.com/development/data-control/) | Every store's location, every retention window, instance backups, and moving to another host | | [First run](https://docs.warmbly.com/development/first-run/) | Claiming the instance, reissuing the setup link, and what to do when accounts already exist | | [Accounts and access](https://docs.warmbly.com/development/accounts-and-access/) | Registration modes, inviting people with or without a mail relay, SSO, and recovering access | | [`warmblyctl`](https://docs.warmbly.com/development/warmblyctl/) | The operator CLI: creating accounts, setting passwords, granting admin, and instance status | diff --git a/admin/Dockerfile b/admin/Dockerfile index 7a3f3642..318d61ea 100644 --- a/admin/Dockerfile +++ b/admin/Dockerfile @@ -5,6 +5,8 @@ # the heavy pnpm build runs only once. FROM --platform=$BUILDPLATFORM node:22-alpine AS build WORKDIR /app +# No TTY in a build: CI=true makes pnpm reinstall instead of prompting. +ENV CI=true RUN corepack enable && corepack prepare pnpm@11.9.0 --activate COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ RUN pnpm install --frozen-lockfile diff --git a/admin/src/app/dashboard/AuditPage.tsx b/admin/src/app/dashboard/AuditPage.tsx index 3bc1ac2e..43739d1f 100644 --- a/admin/src/app/dashboard/AuditPage.tsx +++ b/admin/src/app/dashboard/AuditPage.tsx @@ -27,7 +27,7 @@ const KNOWN_ACTIONS = [ ]; const KNOWN_TARGETS = [ - "worker", "aws_credentials", "worker_profile", "release", + "worker", "aws_credentials", "worker_profile", "release", "instance", "user", "email_account", "campaign", "plan", ]; diff --git a/admin/src/app/dashboard/HealthPage.tsx b/admin/src/app/dashboard/HealthPage.tsx index 4c06654c..0a4aef45 100644 --- a/admin/src/app/dashboard/HealthPage.tsx +++ b/admin/src/app/dashboard/HealthPage.tsx @@ -2,14 +2,25 @@ // as decided by the running backend. The endpoint returns only checks that // are not ok, so an empty response is a real all-clear and not a stub. -import { Link } from "react-router-dom"; -import { AlertTriangle, CheckCircle2, Info, RefreshCw, XCircle } from "lucide-react"; +import { useState } from "react"; +import { Link, useSearchParams } from "react-router-dom"; +import { + AlertTriangle, + ArrowUpCircle, + CheckCircle2, + Info, + Loader2, + RefreshCw, + XCircle, +} from "lucide-react"; import { PageHeader } from "@/components/layout/PageHeader"; import { ErrorState } from "@/components/ErrorState"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import { InstanceFindings } from "./InstanceHealthPanel"; +import { UpdateDialog } from "@/components/layout/UpdateDialog"; import { useInstanceHealth } from "@/hooks/useInstanceHealth"; +import { buildLabel, isUpdating, useUpdateState } from "@/hooks/useUpdateState"; import type { InstanceHealthSummary } from "@/lib/api/client/admin/instance"; export default function HealthPage() { @@ -43,6 +54,8 @@ export default function HealthPage() { + + {healthQ.isLoading && (
@@ -89,6 +102,61 @@ export default function HealthPage() { ); } +// Version and update status, above the findings: the same facts as the pill +// in the top bar, on the page an operator opens to ask "is this instance ok". +function UpdateCard() { + const updateQ = useUpdateState(); + // ?update=1 is how the dashboard's version pill deep-links an admin + // straight into the dialog. + const [params] = useSearchParams(); + const [open, setOpen] = useState(params.get("update") === "1"); + const state = updateQ.data; + if (!state) return null; + + const updating = isUpdating(state); + const available = state.update_available; + const tone = updating + ? "border-sky-200 bg-sky-50/60" + : available + ? "border-amber-200 bg-amber-50/60" + : "border-border bg-white"; + + return ( +
+ {updating ? ( + + ) : available ? ( + + ) : ( + + )} +
+ + {updating + ? "Updating this instance" + : available + ? `${state.latest?.tag && state.reason === "release" ? state.latest.tag : "A newer version"} is available` + : "Up to date"} + + + {" "} + running {buildLabel(state)} + {state.updater.checkout && !state.updater.checkout.detached + ? ` on ${state.updater.checkout.branch}` + : ""} + {state.checked_at + ? `, checked ${new Date(state.checked_at).toLocaleTimeString()}` + : ""} + +
+ + +
+ ); +} + function SummaryStrip({ summary, total, diff --git a/admin/src/app/dashboard/InstanceSettingsPage.tsx b/admin/src/app/dashboard/InstanceSettingsPage.tsx index edaf7a2f..71004f2c 100644 --- a/admin/src/app/dashboard/InstanceSettingsPage.tsx +++ b/admin/src/app/dashboard/InstanceSettingsPage.tsx @@ -76,11 +76,57 @@ const SYNC_FIELDS = [ type SyncFieldKey = (typeof SYNC_FIELDS)[number]["key"]; +// Retention windows, mirroring internal/config/constants.go. Each one is also +// how long the personal data in that log is held, so the help text says what +// the data is rather than only what the number does. +const RETENTION_MIN_DAYS = 1; +const RETENTION_MAX_DAYS = 3650; + +const RETENTION_FIELDS = [ + { + key: "engagementDays", + setting: "engagement_event_days", + label: "Opens and clicks (days)", + help: "Per-event open and click logs, with the client, device and approximate location of each. Campaign counts and routing read a separate summary that is never pruned, so shortening this changes what a contact's timeline can show, not what a campaign does.", + }, + { + key: "formDays", + setting: "form_event_days", + label: "Form funnel events (days)", + help: "Views, starts, field-level drop-off and submissions for hosted forms. Funnel reports range up to 90 days, so anything below that shortens the report too. Submitted contacts are unaffected.", + }, + { + key: "auditDays", + setting: "audit_log_days", + label: "Audit log (days)", + help: "Who did what, from which IP address and user agent, with the change payload. This window is how long that record is held, and it is the one most likely to be set by a retention policy.", + }, +] as const; + +type RetentionFieldKey = (typeof RETENTION_FIELDS)[number]["key"]; + +// The two presets are the ends of the band people actually choose between. +const RETENTION_PRESETS = [ + { + id: "default", + label: "Defaults", + description: "365 / 180 / 90 days", + values: { engagementDays: "365", formDays: "180", auditDays: "90" }, + }, + { + id: "minimal", + label: "Minimal retention", + description: "30 / 30 / 30 days", + values: { engagementDays: "30", formDays: "30", auditDays: "30" }, + }, +] as const; + interface FormState { linksEnabled: boolean; ttlHours: string; allowInvitedSignup: boolean; sync: Record; + retention: Record; enforceDomainAuth: boolean; authGraceHours: string; } @@ -96,6 +142,11 @@ function toForm(s: InstanceSettings): FormState { dailyPerMailbox: String(s.sync.daily_messages_per_mailbox), dailyPerOrg: String(s.sync.daily_messages_per_org), }, + retention: { + engagementDays: String(s.retention.engagement_event_days), + formDays: String(s.retention.form_event_days), + auditDays: String(s.retention.audit_log_days), + }, enforceDomainAuth: s.deliverability.enforce_domain_auth, authGraceHours: String(s.deliverability.auth_grace_hours), }; @@ -137,6 +188,12 @@ export default function InstanceSettingsPage() { !!server && !!form && SYNC_FIELDS.some((f) => form.sync[f.key] !== String(server.sync[f.setting])); + const retentionDirty = + !!server && + !!form && + RETENTION_FIELDS.some( + (f) => form.retention[f.key] !== String(server.retention[f.setting]), + ); const dirty = !!server && !!form && @@ -145,9 +202,15 @@ export default function InstanceSettingsPage() { form.allowInvitedSignup !== server.access.allow_invited_signup || form.enforceDomainAuth !== server.deliverability.enforce_domain_auth || form.authGraceHours !== String(server.deliverability.auth_grace_hours) || + retentionDirty || syncDirty); const syncValid = form !== null && SYNC_FIELDS.every((f) => syncFieldValid(form.sync[f.key], f.min, f.max)); + const retentionValid = + form !== null && + RETENTION_FIELDS.every((f) => + syncFieldValid(form.retention[f.key], RETENTION_MIN_DAYS, RETENTION_MAX_DAYS), + ); const authGrace = form ? Number(form.authGraceHours) : NaN; const authGraceValid = @@ -177,6 +240,12 @@ export default function InstanceSettingsPage() { toast.error("Every sync budget must be a whole number inside its range"); return; } + if (!retentionValid) { + toast.error( + `Every retention window must be a whole number of days between ${RETENTION_MIN_DAYS} and ${RETENTION_MAX_DAYS.toLocaleString()}`, + ); + return; + } if (!authGraceValid) { toast.error( `The authentication grace period must be a whole number of hours between ${AUTH_GRACE_MIN_HOURS} and ${AUTH_GRACE_MAX_HOURS}`, @@ -192,6 +261,11 @@ export default function InstanceSettingsPage() { daily_messages_per_mailbox: Number(form.sync.dailyPerMailbox), daily_messages_per_org: Number(form.sync.dailyPerOrg), }, + retention: { + engagement_event_days: Number(form.retention.engagementDays), + form_event_days: Number(form.retention.formDays), + audit_log_days: Number(form.retention.auditDays), + }, deliverability: { enforce_domain_auth: form.enforceDomainAuth, auth_grace_hours: authGrace, @@ -370,6 +444,93 @@ export default function InstanceSettingsPage() { + + + Data retention + + How long event-level history is kept on this instance. Every window + below is also how long the personal data in that log is held, so + these are the settings a retention or privacy policy applies to. A + sweep runs a few times a day and reads these values each pass, so a + change takes effect without a restart. Deletion is permanent: + shortening a window removes what already sits outside it on the + next sweep. + + + +
+ Presets + {RETENTION_PRESETS.map((preset) => { + const active = RETENTION_FIELDS.every( + (f) => form.retention[f.key] === preset.values[f.key], + ); + return ( + + ); + })} +
+
+ {RETENTION_FIELDS.map((f) => { + const valid = syncFieldValid( + form.retention[f.key], + RETENTION_MIN_DAYS, + RETENTION_MAX_DAYS, + ); + return ( +
+ + + setForm({ + ...form, + retention: { + ...form.retention, + [f.key]: e.target.value, + }, + }) + } + aria-invalid={!valid} + className="mt-1" + /> +

+ {f.help} Between {RETENTION_MIN_DAYS} and{" "} + {RETENTION_MAX_DAYS.toLocaleString()} days. +

+ {!valid && ( +

+ Enter a whole number of days between{" "} + {RETENTION_MIN_DAYS} and{" "} + {RETENTION_MAX_DAYS.toLocaleString()}. +

+ )} +
+ ); + })} +
+ + + Sending-domain authentication diff --git a/admin/src/app/dashboard/NotificationsPage.tsx b/admin/src/app/dashboard/NotificationsPage.tsx new file mode 100644 index 00000000..89b9bee9 --- /dev/null +++ b/admin/src/app/dashboard/NotificationsPage.tsx @@ -0,0 +1,433 @@ +// Operator notification channels: where this deployment tells its operator +// that something happened. Discord, Slack, a generic signed webhook, or email. +// +// The channels live in the same instance settings document as the rest of the +// writable configuration, so this page reads and writes /admin/instance/settings +// and only reaches for its own endpoints for the event catalog and the test +// delivery probe. +// +// Targets and secrets come back redacted. An unchanged field is sent back as +// the preview the server returned (or empty) and resolves to the stored value, +// so saving an unrelated toggle can never wipe a credential. + +import { useEffect, useMemo, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { + Bell, + Check, + Hash, + Mail, + Plus, + Save, + Send, + Trash2, + Webhook, +} from "lucide-react"; +import { PageHeader } from "@/components/layout/PageHeader"; +import { ErrorState } from "@/components/ErrorState"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Switch } from "@/components/ui/switch"; +import { + getInstanceSettings, + getNotificationEvents, + putInstanceSettings, + testNotificationChannel, + type InstanceSettings, + type NotifyChannel, + type NotifyChannelType, + type NotifyEventDef, +} from "@/lib/api/client/admin/instance"; + +const SETTINGS_KEY = ["admin", "instance", "settings"]; +const EVENTS_KEY = ["admin", "instance", "notification-events"]; + +const TYPES: { + value: NotifyChannelType; + label: string; + icon: typeof Hash; + placeholder: string; + help: string; +}[] = [ + { + value: "discord", + label: "Discord", + icon: Hash, + placeholder: "https://discord.com/api/webhooks/…", + help: "Server settings, then Integrations, then New Webhook. Copy the webhook URL.", + }, + { + value: "slack", + label: "Slack", + icon: Hash, + placeholder: "https://hooks.slack.com/services/…", + help: "Create a Slack app with an incoming webhook and copy its URL.", + }, + { + value: "webhook", + label: "Webhook", + icon: Webhook, + placeholder: "https://example.com/hooks/warmbly", + help: "Receives the event as JSON. Set a secret to have it signed with HMAC-SHA256.", + }, + { + value: "email", + label: "Email", + icon: Mail, + placeholder: "ops@example.com", + help: "Needs a working platform mail transport on this deployment.", + }, +]; + +function typeDef(t: NotifyChannelType) { + return TYPES.find((x) => x.value === t) ?? TYPES[0]; +} + +function newChannel(): NotifyChannel { + return { + // The server assigns a real id on save; this one only has to be unique + // within the unsaved list so React can key it. + id: `new-${Math.random().toString(36).slice(2, 10)}`, + name: "", + type: "discord", + target: "", + secret: "", + events: [], + enabled: true, + }; +} + +export default function NotificationsPage() { + const queryClient = useQueryClient(); + const settings = useQuery({ queryKey: SETTINGS_KEY, queryFn: getInstanceSettings }); + const catalog = useQuery({ queryKey: EVENTS_KEY, queryFn: getNotificationEvents }); + + const [channels, setChannels] = useState(null); + const [testing, setTesting] = useState(null); + + useEffect(() => { + if (settings.data) { + setChannels(settings.data.notifications?.channels ?? []); + } + }, [settings.data]); + + const save = useMutation({ + mutationFn: (next: NotifyChannel[]) => { + const base = settings.data as InstanceSettings; + return putInstanceSettings({ + ...base, + notifications: { channels: next }, + }); + }, + onSuccess: (doc) => { + queryClient.setQueryData(SETTINGS_KEY, doc); + setChannels(doc.notifications?.channels ?? []); + toast.success("Notification channels saved"); + }, + onError: (e: Error) => toast.error(e.message || "Could not save channels"), + }); + + const groups = useMemo(() => { + const events = catalog.data?.events ?? []; + const order: string[] = []; + const byGroup = new Map(); + for (const e of events) { + if (!byGroup.has(e.group)) { + byGroup.set(e.group, []); + order.push(e.group); + } + byGroup.get(e.group)!.push(e); + } + return order.map((g) => ({ group: g, events: byGroup.get(g)! })); + }, [catalog.data]); + + if (settings.isError) { + return settings.refetch()} />; + } + + function update(id: string, patch: Partial) { + setChannels((prev) => (prev ?? []).map((c) => (c.id === id ? { ...c, ...patch } : c))); + } + + function remove(id: string) { + setChannels((prev) => (prev ?? []).filter((c) => c.id !== id)); + } + + function toggleEvent(id: string, key: string, on: boolean) { + setChannels((prev) => + (prev ?? []).map((c) => { + if (c.id !== id) return c; + const next = on ? [...c.events, key] : c.events.filter((e) => e !== key); + return { ...c, events: next }; + }), + ); + } + + async function sendTest(ch: NotifyChannel) { + setTesting(ch.id); + try { + // A saved channel is tested by id so the server uses its stored + // credential; an unsaved one carries its fields inline. + const saved = !ch.id.startsWith("new-"); + await testNotificationChannel( + saved + ? { id: ch.id } + : { type: ch.type, name: ch.name, target: ch.target, secret: ch.secret }, + ); + toast.success("Test alert delivered"); + } catch (e) { + toast.error((e as Error).message || "Delivery failed"); + } finally { + setTesting(null); + } + } + + const list = channels ?? []; + const dirty = + !!settings.data && + JSON.stringify(list) !== JSON.stringify(settings.data.notifications?.channels ?? []); + // Changing a channel's type clears its target on purpose, so a save with + // one still empty would drop the channel server-side. Block it here and + // say which one needs attention. + const incomplete = list.filter((c) => !c.target.trim()); + + return ( +
+ + + + + + {settings.isLoading ? ( + + ) : list.length === 0 ? ( + + + +

No channels yet

+

+ Nothing is being sent anywhere. Add a channel and pick the events it + should receive; leave every event unchecked to receive all of them. +

+ + + + ) : ( +
+ {list.map((ch) => { + const def = typeDef(ch.type); + const Icon = def.icon; + const saved = !ch.id.startsWith("new-"); + return ( + + +
+
+ + + +
+ + {ch.name || def.label} + + + {ch.events.length === 0 + ? "Receives every event" + : `Receives ${ch.events.length} event${ch.events.length === 1 ? "" : "s"}`} + {!saved && " · unsaved"} + +
+
+
+ {!ch.enabled && Off} + update(ch.id, { enabled: v })} + /> + + +
+
+ + +
+
+ +
+ {TYPES.map((t) => ( + + ))} +
+
+
+ + update(ch.id, { name: e.target.value })} + /> +
+
+ + update(ch.id, { target: e.target.value })} + /> + {ch.target.trim() ? ( +

{def.help}

+ ) : ( +

+ {ch.type === "email" + ? "Enter an address before saving." + : "Enter the webhook URL for this transport before saving."} +

+ )} +
+
+ + {ch.type === "webhook" && ( +
+ + update(ch.id, { secret: e.target.value })} + /> +

+ Signs the body as{" "} + X-Warmbly-Signature: t=<unix>,v1=<hex>, the + same scheme customer webhooks use. +

+
+ )} + +
+
+ + +
+ {catalog.isLoading ? ( + + ) : ( +
+ {groups.map(({ group, events }) => ( +
+

+ {group} +

+ {events.map((e) => ( + + ))} +
+ ))} +
+ )} + {ch.events.length === 0 && ( +

+ + Nothing selected, so this channel receives every event. +

+ )} +
+ + + ); + })} +
+ )} +
+ ); +} diff --git a/admin/src/components/layout/Sidebar.tsx b/admin/src/components/layout/Sidebar.tsx index 92f2b619..eb38a919 100644 --- a/admin/src/components/layout/Sidebar.tsx +++ b/admin/src/components/layout/Sidebar.tsx @@ -5,6 +5,7 @@ import { NavLink } from "react-router-dom"; import { + Bell, Activity, BarChart3, Building2, @@ -102,6 +103,12 @@ const GROUPS: NavGroup[] = [ icon: Settings2, perm: AdminPerm.ManageSettings, }, + { + to: "/configuration/notifications", + label: "Notifications", + icon: Bell, + perm: AdminPerm.ManageSettings, + }, { to: "/limits", label: "Effective limits", diff --git a/admin/src/components/layout/Topbar.tsx b/admin/src/components/layout/Topbar.tsx index ffcfade8..d0ea0229 100644 --- a/admin/src/components/layout/Topbar.tsx +++ b/admin/src/components/layout/Topbar.tsx @@ -1,9 +1,11 @@ -// Sticky top bar. Holds the env pill, a search slot (unused for now — -// reserved for the cmd-K palette we'll add later), and the user menu. +// Sticky top bar. Holds the env pill, the version pill (which becomes the +// update indicator), a search slot (unused for now, reserved for the cmd-K +// palette we'll add later), and the user menu. // The 3px admin stripe sits *above* this bar in AppShell so it's the // first thing the eye lands on. import { EnvPill } from "./EnvPill"; +import { UpdatePill } from "./UpdatePill"; import { UserMenu } from "./UserMenu"; export function Topbar() { @@ -12,6 +14,7 @@ export function Topbar() {
+ Connected to /admin/* diff --git a/admin/src/components/layout/UpdateDialog.tsx b/admin/src/components/layout/UpdateDialog.tsx new file mode 100644 index 00000000..c71be78b --- /dev/null +++ b/admin/src/components/layout/UpdateDialog.tsx @@ -0,0 +1,569 @@ +// The update dialog behind the version pill: what is running, what is newest, +// and the one button that pulls, rebuilds and restarts through the updater. +// While a job runs it shows the steps and the live log; when the backend goes +// away for the restart it says so and keeps polling until it is back. + +import { useEffect, useMemo, useRef, useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { + AlertTriangle, + Check, + CheckCircle2, + ExternalLink, + GitBranch, + Loader2, + Package, + RefreshCw, + RotateCw, + XCircle, +} from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { docsUrl } from "@/lib/docs"; +import { cn } from "@/lib/utils"; +import { useAdminPerm } from "@/hooks/useAdminPerm"; +import { AdminPerm } from "@/lib/auth/permissions"; +import { + UPDATE_JOB_KEY, + UPDATE_STATE_KEY, + buildLabel, + isUpdating, + useUpdateJob, + useUpdateState, +} from "@/hooks/useUpdateState"; +import { + applyUpdate, + checkForUpdates, + type UpdateJob, + type UpdateState, +} from "@/lib/api/client/admin/updates"; +import { markUpdateStarted, readUpdateStarted } from "@/lib/updateSession"; + +const DOCS_UPDATES = "/development/updates/"; + +const STEPS: Record = { + compose: ["fetch", "checkout", "build", "restart", "prune", "wait"], + image: ["resolve", "pull", "restart", "prune", "wait"], + command: ["fetch", "checkout", "command", "wait"], +}; + +const STEP_LABELS: Record = { + fetch: "Fetch", + checkout: "Pull", + build: "Build", + resolve: "Pin release", + pull: "Pull images", + restart: "Restart", + prune: "Clean up", + command: "Run script", + wait: "Wait for backend", + starting: "Starting", +}; + +type Phase = "idle" | "running" | "restarting" | "done" | "failed"; + +interface Props { + open: boolean; + onOpenChange: (open: boolean) => void; +} + +export function UpdateDialog({ open, onOpenChange }: Props) { + const qc = useQueryClient(); + const canManage = useAdminPerm(AdminPerm.ManageSettings); + const stateQ = useUpdateState(); + const jobQ = useUpdateJob(open); + const state: UpdateState | undefined = jobQ.data ?? stateQ.data; + const [confirming, setConfirming] = useState(false); + const started = readUpdateStarted(); + + const phase: Phase = useMemo(() => { + if (isUpdating(state)) return "running"; + if (started && (jobQ.isError || stateQ.isError)) return "restarting"; + const last = state?.updater.last_job; + if (started && last && last.status !== "running") { + return last.status === "succeeded" ? "done" : "failed"; + } + return "idle"; + }, [state, started, jobQ.isError, stateQ.isError]); + + const canApply = + canManage && + state?.updater.status === "ok" && + !!state?.update_available && + !state?.updater.checkout?.dirty && + phase === "idle"; + + // Leaving the confirmation open once the update can no longer start + // (a phase change, or the checkout turning dirty) would let a stale + // click start a job that fails. + useEffect(() => { + if (!canApply) setConfirming(false); + }, [canApply]); + + const checkMut = useMutation({ + mutationFn: checkForUpdates, + onSuccess: (data) => { + qc.setQueryData(UPDATE_STATE_KEY, data); + qc.setQueryData(UPDATE_JOB_KEY, data); + toast.success( + data.update_available ? "A newer version is available" : "This instance is up to date", + ); + }, + onError: (err: Error) => toast.error(err.message || "Could not check for updates"), + }); + + const applyMut = useMutation({ + mutationFn: () => { + // The state keeps refreshing while the confirmation is open; a + // checkout that turned dirty meanwhile must not start a job. + if (!canApply) return Promise.reject(new Error("The update can no longer start; check the status above.")); + return applyUpdate("latest"); + }, + onSuccess: (job: UpdateJob) => { + markUpdateStarted(state?.running.version ?? "", state?.running.commit ?? job.from_commit); + setConfirming(false); + toast.success("Update started"); + void qc.invalidateQueries({ queryKey: UPDATE_STATE_KEY }); + void qc.invalidateQueries({ queryKey: UPDATE_JOB_KEY }); + }, + onError: (err: Error) => toast.error(err.message || "Could not start the update"), + }); + + const updater = state?.updater; + const checkout = updater?.checkout; + const job = updater?.job ?? updater?.last_job; + // An image install never builds, so the confirmation must not promise a + // rebuild it will not do. + const imageMode = updater?.mode === "image"; + + return ( + + + + Updates + + {state + ? `Running ${buildLabel(state)}${state.running.commit ? ` (${state.running.commit.slice(0, 7)})` : ""}.` + : "Reading the running version."} + + + + {phase === "idle" && state && ( +
+ + {updater?.status !== "ok" && } + {checkout?.dirty && ( + + The checkout has local modifications. The updater refuses to move it + until they are committed or stashed, or `UPDATER_ALLOW_DIRTY=true`. + + )} + {job && job.status === "failed" && !started && ( + + The last update failed at step {STEP_LABELS[job.step] ?? job.step}:{" "} + {job.error} + + )} + {confirming && ( +
+ +
+ {imageMode + ? "This pulls the release images and restarts every service." + : "This pulls the checkout, rebuilds the images and restarts every service."} +
+
+ Sending and syncing pause for a few minutes and resume on their own; + nothing in flight is lost. Migrations apply when the backend comes + back. This panel reconnects by itself. +
+ +
+ )} +
+ )} + + {(phase === "running" || phase === "restarting") && ( + + )} + + {phase === "done" && state && ( +
+
+ +
+
Updated to {buildLabel(state)}
+ Every service is back and sending has resumed. Reload to pick up the + new admin panel. +
+
+ {job?.log && } +
+ )} + + {phase === "failed" && ( +
+ +
The update failed
+ {job?.error ?? "See the log below."} The previous version is still running + unless the restart step had already begun. + + {job?.log && } +
+ )} + + + + How updates work + + +
+ {phase === "idle" && canManage && ( + + )} + {phase === "idle" && canApply && !confirming && ( + + )} + {phase === "idle" && confirming && ( + <> + + + + )} + {(phase === "done" || phase === "failed") && ( + + )} + {(phase === "running" || phase === "restarting") && ( + + )} +
+ + + + ); +} + +function Overview({ state }: { state: UpdateState }) { + const { latest, updater } = state; + const checkout = updater.checkout; + const release = updater.release; + return ( +
+
Latest release
+
+ {latest ? ( + + {latest.tag} + {latest.published_at && ( + + {new Date(latest.published_at).toLocaleDateString()} + + )} + {latest.html_url && ( + + Release notes + + + )} + + ) : state.check_error ? ( + Could not read releases: {state.check_error} + ) : state.enabled ? ( + No release found for {state.repo} + ) : ( + Release check is off + )} +
+ +
Status
+
+ {state.update_available ? ( + + Update available + + ) : ( + + Up to date + + )} + {state.checked_at && ( + + checked {new Date(state.checked_at).toLocaleTimeString()}, every{" "} + {state.interval} + + )} +
+ + {release && ( + <> +
Installed
+
+ + + {release.prefix}/*:{release.tag} + + + {release.pinned + ? "pinned to this release" + : "following the channel tag"} + +
+ + )} + + {checkout && ( + <> +
Checkout
+
+ + + {checkout.detached ? "pinned" : checkout.branch}@{checkout.commit.slice(0, 7)} + + {!checkout.detached && ( + + {checkout.behind > 0 + ? `${checkout.behind} commit${checkout.behind === 1 ? "" : "s"} behind` + : "matches the remote"} + + )} + {checkout.fetch_error && ( + fetch failed: {checkout.fetch_error} + )} +
+ + )} + +
Updater
+
+ {updater.status === "ok" && ( + + ready + ({updater.mode} mode) + + )} + {updater.status === "off" && not configured} + {updater.status === "unreachable" && unreachable} +
+
+ ); +} + +function UpdaterNotice({ state }: { state: UpdateState }) { + const u = state.updater; + // A clone-free install has no checkout to pull, so the by-hand command is + // the compose one. Which install this is comes from the release block, and + // an unreachable updater reports neither block, so that case names both + // rather than printing one command that may not work. + // + // A pinned install needs the tag edited first: pulling again resolves the + // same fixed version and updates nothing, which is the kind of instruction + // that looks like it worked. + const pinnedTag = u.release?.pinned ? (state.latest?.tag ?? "vX.Y.Z") : null; + const byHand = u.release + ? "docker compose pull && docker compose up -d" + : u.checkout + ? "git pull && make up" + : null; + if (u.status === "unreachable") { + return ( + +
The updater is not answering
+ {u.error} Until it does, update by hand from the install directory: + {pinnedTag && } + {byHand ? {byHand} : } + + ); + } + return ( + +
This panel can only report
+ No updater is configured, so apply updates from a shell on the host: + {pinnedTag && } + {byHand ? {byHand} : } + To get the button, enable the updater compose profile (`make up` and the installer + both do) or run the updater unit on a bare-metal host. + + ); +} + +// An image install pinned to a fixed version resolves the same images on every +// pull, so moving it is an edit to .env and then the pull. Without this the +// command below reports success and changes nothing. +function SetTagFirst({ tag }: { tag: string }) { + return ( + <> +
+ This install is pinned, so set the release in .env first: +
+ {`WARMBLY_TAG=${tag}`} +
then:
+ + ); +} + +// Shown when the updater is unreachable, so nothing says which shape of install +// this is. Naming both beats guessing: the wrong one fails confusingly on an +// install that has no checkout, or no images to pull. +function BothCommands() { + return ( + <> +
From an install.sh install:
+ docker compose pull && docker compose up -d +
From a git checkout:
+ git pull && make up + + ); +} + +function Progress({ + state, + job, + restarting, +}: { + state: UpdateState | undefined; + job: UpdateJob | undefined; + restarting: boolean; +}) { + const mode = state?.updater.mode ?? "compose"; + const steps = STEPS[mode] ?? STEPS.compose; + const current = restarting ? "wait" : (job?.step ?? "starting"); + const currentIdx = Math.max(0, steps.indexOf(current)); + const percent = Math.round(((currentIdx + 0.5) / steps.length) * 100); + return ( +
+
+ +
+
+ + {restarting ? "Restarting services" : `Updating: ${STEP_LABELS[current] ?? current}`} + + {percent}% +
+
+
+
+
+ {restarting + ? "The backend is coming back up. This panel reconnects on its own; keep the tab open or come back later, the result is kept." + : "You can close this dialog; the pill in the top bar keeps following the job."} +
+
+
+
    + {steps.map((s, i) => { + const done = currentIdx > i || (restarting && s !== "wait"); + const active = s === current; + return ( +
  1. + {done ? ( + + ) : active ? ( + + ) : null} + {STEP_LABELS[s] ?? s} +
    2. + ); + })} +
+ {job?.log && job.log.length > 0 && } +
+ ); +} + +function LogPanel({ lines }: { lines: string[] }) { + const ref = useRef(null); + useEffect(() => { + const el = ref.current; + if (el) el.scrollTop = el.scrollHeight; + }, [lines.length]); + return ( + 
+            {lines.join("\n")}
+
+ ); +} + +function Notice({ tone, children }: { tone: "info" | "warning" | "error"; children: React.ReactNode }) { + const styles = { + info: "border-sky-200 bg-sky-50/60 text-sky-900", + warning: "border-amber-200 bg-amber-50/60 text-amber-900", + error: "border-red-200 bg-red-50/60 text-red-900", + }[tone]; + const Icon = tone === "error" ? XCircle : AlertTriangle; + return ( +
+ +
{children}
+
+ ); +} + +function Cmd({ children }: { children: string }) { + return ( + + {children} + + ); +} diff --git a/admin/src/components/layout/UpdatePill.tsx b/admin/src/components/layout/UpdatePill.tsx new file mode 100644 index 00000000..e9cd9fa7 --- /dev/null +++ b/admin/src/components/layout/UpdatePill.tsx @@ -0,0 +1,98 @@ +// The version pill in the top bar. Quiet when the instance is current, amber +// when a newer version exists, a spinner while an update runs or the backend +// restarts. It also closes the loop after a restart: once the backend answers +// with a new build it says "Updated to vX" and refreshes every query. + +import { useEffect, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { ArrowUpCircle, Loader2 } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { useAdminPerm } from "@/hooks/useAdminPerm"; +import { AdminPerm } from "@/lib/auth/permissions"; +import { buildLabel, isUpdating, useUpdateState } from "@/hooks/useUpdateState"; +import { clearUpdateStarted, readUpdateStarted } from "@/lib/updateSession"; +import { UpdateDialog } from "./UpdateDialog"; + +export function UpdatePill() { + const canRead = useAdminPerm(AdminPerm.ViewAnalytics); + const stateQ = useUpdateState({ enabled: canRead }); + const qc = useQueryClient(); + const [open, setOpen] = useState(false); + const state = stateQ.data; + const updating = isUpdating(state); + const started = readUpdateStarted(); + // The poll fails while the backend restarts; an update this browser + // started is the only reason to read that as "restarting" rather than down. + const restarting = !!started && stateQ.isError; + + // The backend is back after an update this browser started: report the + // outcome once, then refresh everything so no page shows stale data. + useEffect(() => { + if (!started || !state || updating) return; + const last = state.updater.last_job; + const moved = + (state.running.commit && state.running.commit !== started.fromCommit) || + (state.running.version && state.running.version !== started.fromVersion); + if (last?.status === "failed") { + clearUpdateStarted(); + toast.error(`The update failed: ${last.error ?? "see the update dialog"}`); + return; + } + if (moved || last?.status === "succeeded") { + clearUpdateStarted(); + toast.success(`Updated to ${buildLabel(state)}`); + void qc.invalidateQueries(); + } + }, [started, state, updating, qc]); + + if (!canRead || (!state && !restarting)) return null; + + let tone = "border-border bg-white text-muted-foreground hover:text-foreground"; + let label = buildLabel(state); + let icon: React.ReactNode = null; + let title = "Up to date"; + + if (restarting) { + tone = "border-sky-200 bg-sky-50 text-sky-700"; + label = "Restarting"; + icon = ; + title = "The backend is restarting after an update"; + } else if (updating) { + tone = "border-sky-200 bg-sky-50 text-sky-700"; + label = "Updating"; + icon = ; + title = `Update in progress: ${state?.updater.job?.step ?? ""}`; + } else if (state?.update_available) { + tone = "border-amber-300 bg-amber-50 text-amber-800 hover:bg-amber-100"; + label = state.latest?.tag && state.reason === "release" ? `Update to ${state.latest.tag}` : "Update available"; + icon = ( + + + + + ); + title = `A newer version is available; running ${buildLabel(state)}`; + } else if (state?.updater.status === "unreachable") { + tone = "border-amber-200 bg-white text-amber-700"; + title = "The updater is not answering"; + } + + return ( + <> + + + + ); +} diff --git a/admin/src/hooks/useUpdateState.ts b/admin/src/hooks/useUpdateState.ts new file mode 100644 index 00000000..5183e2c0 --- /dev/null +++ b/admin/src/hooks/useUpdateState.ts @@ -0,0 +1,45 @@ +import { useQuery } from "@tanstack/react-query"; +import { getUpdateState, type UpdateState } from "@/lib/api/client/admin/updates"; + +export const UPDATE_STATE_KEY = ["admin", "instance", "update"] as const; +export const UPDATE_JOB_KEY = ["admin", "instance", "update", "job"] as const; + +// One cache entry feeds the top-bar pill, the Setup and health card and the +// dialog. It polls every minute, and every few seconds while a job runs, so +// the pill follows an update started from another tab or before a reload. +export function useUpdateState(options?: { enabled?: boolean }) { + return useQuery({ + queryKey: UPDATE_STATE_KEY, + queryFn: () => getUpdateState(false), + refetchInterval: (query) => (isUpdating(query.state.data) ? 3_000 : 60_000), + // While the backend restarts every request fails; keep the last state + // on screen instead of flashing an error. + retry: false, + enabled: options?.enabled ?? true, + }); +} + +// The dialog's view of a running job, with the log. Polled only while open. +export function useUpdateJob(enabled: boolean) { + return useQuery({ + queryKey: UPDATE_JOB_KEY, + queryFn: () => getUpdateState(true), + refetchInterval: (query) => (isUpdating(query.state.data) ? 2_000 : 15_000), + retry: false, + enabled, + }); +} + +export function isUpdating(state: UpdateState | undefined): boolean { + return state?.updater.job?.status === "running"; +} + +// A short, stable label for the running build: the tag when there is one, +// otherwise the commit. +export function buildLabel(state: UpdateState | undefined): string { + if (!state) return ""; + const v = state.running.version; + if (v && v !== "dev") return v; + const c = state.running.commit ?? state.updater.checkout?.commit ?? ""; + return c ? `dev ${c.slice(0, 7)}` : "dev"; +} diff --git a/admin/src/lib/api/client/admin/instance.ts b/admin/src/lib/api/client/admin/instance.ts index 432d4593..19430ecd 100644 --- a/admin/src/lib/api/client/admin/instance.ts +++ b/admin/src/lib/api/client/admin/instance.ts @@ -111,6 +111,16 @@ export interface InstanceSettings { daily_messages_per_mailbox: number; daily_messages_per_org: number; }; + // How long event-level history is kept. Every window bounds personal data: + // opens and clicks carry a client, a device and a location, funnel events + // carry a visitor's path, and the audit trail carries IP addresses, user + // agents and change payloads. None of them affect a count or a routing + // decision; campaign progress keeps its own summary. + retention: { + engagement_event_days: number; + form_event_days: number; + audit_log_days: number; + }; // The sending-domain authentication gate. A mailbox whose domain has been // failing SPF or DMARC for longer than the grace window stops sending cold // mail and warmup mail until the records are fixed. @@ -118,6 +128,71 @@ export interface InstanceSettings { enforce_domain_auth: boolean; auth_grace_hours: number; }; + // Operator notification channels. Targets and secrets are redacted on + // read: a chat webhook URL is a bearer credential, so the server returns a + // recognisable preview and treats the preview (or an empty string) as + // "keep what is stored" on write. + // Optional so a client that does not manage channels (the instance + // settings page) can PUT without them; an absent section keeps the + // stored channels untouched. + notifications?: { + channels: NotifyChannel[]; + }; +} + +export type NotifyChannelType = "discord" | "slack" | "webhook" | "email"; + +export interface NotifyChannel { + id: string; + name: string; + type: NotifyChannelType; + /** Webhook URL, or the address for an email channel. Redacted on read. */ + target: string; + /** HMAC secret for the generic webhook transport. Redacted on read. */ + secret?: string; + /** Subscribed event keys. Empty means every event. */ + events: string[]; + enabled: boolean; +} + +export type NotifySeverity = "info" | "warning" | "urgent"; + +export interface NotifyEventDef { + key: string; + label: string; + description: string; + group: string; + severity: NotifySeverity; + self_host_relevant: boolean; +} + +export interface NotifyEventsResult { + events: NotifyEventDef[]; + self_hosted: boolean; +} + +export function getNotificationEvents(): Promise { + return Request({ + method: "GET", + url: "/admin/instance/notifications/events", + authorization: true, + }); +} + +/** Send a test alert. Pass `id` for a saved channel, or the unsaved fields. */ +export function testNotificationChannel(body: { + id?: string; + type?: NotifyChannelType; + name?: string; + target?: string; + secret?: string; +}): Promise<{ delivered: boolean }> { + return Request({ + method: "POST", + url: "/admin/instance/notifications/test", + data: body, + authorization: true, + }); } export function getInstanceSettings(): Promise { diff --git a/admin/src/lib/api/client/admin/updates.ts b/admin/src/lib/api/client/admin/updates.ts new file mode 100644 index 00000000..84ddbd30 --- /dev/null +++ b/admin/src/lib/api/client/admin/updates.ts @@ -0,0 +1,111 @@ +// /admin/instance/update : what is running, what is newest, and the button +// that applies it through the host-side updater. + +import { Request } from "@/lib/api/client"; + +export interface RunningBuild { + version: string; + commit?: string; + built_at?: string; +} + +export interface LatestRelease { + tag: string; + name?: string; + html_url?: string; + published_at?: string; + channel: string; +} + +export interface UpdaterCheckout { + branch: string; + detached: boolean; + commit: string; + describe: string; + remote_commit: string; + behind: number; + dirty: boolean; + fetched_at: string; + fetch_error?: string; +} + +// An image install has no checkout: what it runs is the tag pinned in its +// .env, which the updater reports instead. +export interface UpdaterRelease { + tag: string; + prefix: string; + // False for a moving channel tag (prod, dev), where re-pulling the same + // name is itself an update. + pinned: boolean; +} + +export type UpdateJobStatus = "running" | "succeeded" | "failed"; + +export interface UpdateJob { + id: string; + status: UpdateJobStatus; + target: string; + step: string; + started_at: string; + finished_at?: string; + error?: string; + from_commit: string; + to_commit?: string; + // Absent on the top-bar poll; present when fetched with log=1. + log?: string[] | null; +} + +export type UpdaterStatus = "off" | "ok" | "unreachable"; + +export interface UpdaterView { + configured: boolean; + status: UpdaterStatus; + error?: string; + mode?: "compose" | "image" | "command"; + repo_dir?: string; + // Exactly one of these: a checkout in compose and command mode, a release + // in image mode. + checkout?: UpdaterCheckout; + release?: UpdaterRelease; + job?: UpdateJob; + last_job?: UpdateJob; +} + +export interface UpdateState { + running: RunningBuild; + latest?: LatestRelease; + update_available: boolean; + reason?: "release" | "commits"; + checked_at?: string; + check_error?: string; + enabled: boolean; + interval: string; + channel: string; + repo: string; + updater: UpdaterView; +} + +export function getUpdateState(withLog = false): Promise { + return Request({ + method: "GET", + url: withLog ? "/admin/instance/update?log=1" : "/admin/instance/update", + authorization: true, + }); +} + +export function checkForUpdates(): Promise { + return Request({ + method: "POST", + url: "/admin/instance/update/check", + authorization: true, + }); +} + +export function applyUpdate(target = "latest"): Promise { + return Request({ + method: "POST", + url: "/admin/instance/update/apply", + data: { target }, + authorization: true, + }); +} diff --git a/admin/src/lib/api/client/admin/warmupContent.ts b/admin/src/lib/api/client/admin/warmupContent.ts index e07b8bee..bccbdb4c 100644 --- a/admin/src/lib/api/client/admin/warmupContent.ts +++ b/admin/src/lib/api/client/admin/warmupContent.ts @@ -151,12 +151,16 @@ export function isJobActive(job: WarmupGenerationJob): boolean { return !TERMINAL_JOB_STATUS.has(job.status); } -/** Only batch jobs that are still in flight can be cancelled. */ +/** + * Only batch jobs that are still in flight can be cancelled. A batch already + * cancelling stays active until the poller ingests what it finished, but a + * second cancel has nothing left to ask for. + */ export function isJobCancellable(job: WarmupGenerationJob): boolean { if (job.mode !== "batch") return false; const bs = job.batch_status ?? ""; if (!bs) return !TERMINAL_JOB_STATUS.has(job.status); - return !TERMINAL_BATCH_STATUS.has(bs); + return bs !== "cancelling" && !TERMINAL_BATCH_STATUS.has(bs); } export interface WarmupAbRow { diff --git a/admin/src/lib/updateSession.ts b/admin/src/lib/updateSession.ts new file mode 100644 index 00000000..7be4a9cc --- /dev/null +++ b/admin/src/lib/updateSession.ts @@ -0,0 +1,44 @@ +// Remembers that this browser started an update, across the backend restart +// and a page reload, so the top bar can say "Updated to vX" (or that it +// failed) once the backend answers again instead of silently going quiet. + +const KEY = "warmbly.admin.update.started"; + +export interface StartedUpdate { + fromVersion: string; + fromCommit: string; + startedAt: number; +} + +export function markUpdateStarted(fromVersion: string, fromCommit: string) { + try { + const v: StartedUpdate = { fromVersion, fromCommit, startedAt: Date.now() }; + sessionStorage.setItem(KEY, JSON.stringify(v)); + } catch { + /* storage unavailable: the dialog still tracks the job while open */ + } +} + +export function readUpdateStarted(): StartedUpdate | null { + try { + const raw = sessionStorage.getItem(KEY); + if (!raw) return null; + const v = JSON.parse(raw) as StartedUpdate; + // An entry older than an hour is a job nobody is waiting on any more. + if (Date.now() - v.startedAt > 60 * 60_000) { + sessionStorage.removeItem(KEY); + return null; + } + return v; + } catch { + return null; + } +} + +export function clearUpdateStarted() { + try { + sessionStorage.removeItem(KEY); + } catch { + /* ignore */ + } +} diff --git a/admin/src/main.tsx b/admin/src/main.tsx index c3761c10..6f55c85a 100644 --- a/admin/src/main.tsx +++ b/admin/src/main.tsx @@ -45,6 +45,7 @@ import SystemStatusPage from "@/app/dashboard/SystemStatusPage"; import HealthPage from "@/app/dashboard/HealthPage"; import ConfigurationPage from "@/app/dashboard/ConfigurationPage"; import InstanceSettingsPage from "@/app/dashboard/InstanceSettingsPage"; +import NotificationsPage from "@/app/dashboard/NotificationsPage"; import LimitsPage from "@/app/dashboard/LimitsPage"; import RealtimeManager from "@/lib/realtime/RealtimeManager"; import { RequirePermission } from "@/components/layout/RequirePermission"; @@ -151,6 +152,17 @@ const router = createBrowserRouter([ ), }, + { + path: "configuration/notifications", + element: ( + + + + ), + }, { path: "limits", element: ( diff --git a/cmd/backend/main.go b/cmd/backend/main.go index d3d00180..ca14ba11 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "log" @@ -25,6 +26,7 @@ import ( "github.com/warmbly/warmbly/internal/app/admin" "github.com/warmbly/warmbly/internal/app/adminoutreach" "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/app/unsublink" "github.com/jackc/pgx/v5/pgxpool" "github.com/warmbly/warmbly/internal/app/advisor" @@ -38,6 +40,7 @@ import ( "github.com/warmbly/warmbly/internal/app/bootstrap" "github.com/warmbly/warmbly/internal/app/campaign" "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/app/cliauth" "github.com/warmbly/warmbly/internal/app/cloudlink" "github.com/warmbly/warmbly/internal/app/compose" "github.com/warmbly/warmbly/internal/app/contact" @@ -67,6 +70,7 @@ import ( "github.com/warmbly/warmbly/internal/app/notification" "github.com/warmbly/warmbly/internal/app/oauth" "github.com/warmbly/warmbly/internal/app/oidcauth" + "github.com/warmbly/warmbly/internal/app/opsnotify" "github.com/warmbly/warmbly/internal/app/organization" orgrisk "github.com/warmbly/warmbly/internal/app/orgrisk" "github.com/warmbly/warmbly/internal/app/orgtransfer" @@ -95,6 +99,7 @@ import ( "github.com/warmbly/warmbly/internal/app/twofa" "github.com/warmbly/warmbly/internal/app/tz" "github.com/warmbly/warmbly/internal/app/unibox" + "github.com/warmbly/warmbly/internal/app/updates" "github.com/warmbly/warmbly/internal/app/user" warmupapp "github.com/warmbly/warmbly/internal/app/warmup" "github.com/warmbly/warmbly/internal/app/warmupcontent" @@ -159,6 +164,7 @@ func main() { var emailService email.EmailService var poolLinkService poollink.Service var cloudLinkService cloudlink.Service + var cliAuthService cliauth.Service var campaignService campaign.CampaignService var analyticsService analytics.AnalyticsService var rateLimitService ratelimit.RateLimitService @@ -179,6 +185,7 @@ func main() { var provisioningPolicyRepo repository.ProvisioningPolicyRepository var tasksService tasks.TasksService var advancedService advanced.Service + var unsubSigner *unsublink.Signer var warmupContentRepo repository.WarmupContentRepository var warmupContentService warmupcontent.Service var creditRepository repository.CreditRepository @@ -232,6 +239,7 @@ func main() { var workerRepoForHandler repository.WorkerRepository var credentialsRepository repository.CredentialsRepository var releasesService *releases.Service + var updatesService *updates.Service // Notifications var emailNotificationService notify.EmailNotificationService @@ -280,6 +288,10 @@ func main() { // instanceSettings and the health registry are built after the handler // dependencies, so the pool is hoisted out of the connection block. var instanceSettings instancesettings.Service + // opsNotifier fans instance-wide operator alerts out to the Discord/Slack/ + // webhook/email channels an admin configured. Nil-safe everywhere: a + // deployment with no channels simply never delivers anything. + var opsNotifier opsnotify.Notifier var instanceChecksDB *pgxpool.Pool var userRepoForHandler repository.UserRepository var organizationRepoForHandler repository.OrganizationRepository @@ -585,6 +597,7 @@ func main() { trackedLinkRepository = repository.NewTrackedLinkRepository(primaryDB.Pool) instanceChecksDB = primaryDB.Pool instanceSettings = instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool)) + bootstrapInstanceSettings(ctx, instanceSettings) if err != nil { sentry.CaptureException(err) log.Fatal(err) @@ -711,6 +724,13 @@ func main() { // trial start (planRepo + creditService, both already constructed above). trialService = trial.NewService(subscriptionRepository, userRepostory, planRepository, creditService) featureGateService = feature.NewService(subscriptionRepository, planRepository) + // An approved daily-send increase must raise what is enforced, not + // only what the dashboard shows. + if g, ok := featureGateService.(interface { + WireLimitOverrides(feature.LimitOverrideReader) + }); ok { + g.WireLimitOverrides(organizationRepository) + } workerAssignmentService = worker.NewAssignmentService(workerRepository, subscriptionRepository, planRepository) subscriptionService = subscription.NewService(subscriptionRepository, planRepository) // dailyThrottleService needs the cache that's constructed @@ -814,6 +834,29 @@ func main() { if organizationService != nil { organizationService.WireInstanceSettings(instanceSettings) } + + // Operator alerts. The channel list lives in the same settings + // document, so this needs nothing else configured to work. + opsNotifier = opsnotify.NewService(instanceSettings, emailNotificationService, config.AppBaseURL()) + authService.WireOperatorNotifier(opsNotifier) + if organizationService != nil { + organizationService.WireOperatorNotifier(opsNotifier) + } + if aware, ok := warmupService.(interface { + WireOperatorNotifier(warmupapp.OperatorNotifier) + }); ok && warmupService != nil { + aware.WireOperatorNotifier(opsNotifier) + } + if aware, ok := orgRiskService.(interface { + WireOperatorNotifier(orgrisk.OperatorNotifier) + }); ok && orgRiskService != nil { + aware.WireOperatorNotifier(opsNotifier) + } + if aware, ok := stripeService.(interface { + WireOperatorNotifier(stripe.OperatorNotifier) + }); ok && stripeService != nil { + aware.WireOperatorNotifier(opsNotifier) + } } log.Printf("Auth policy: login_code=%s registration=%s (DISABLE_REGISTRATION) email_verification=%t sso_auto_provision=%t", authPolicy.LoginCode, authPolicy.Registration, authPolicy.RequireEmailVerification, authPolicy.SSOAutoProvision) @@ -1122,12 +1165,31 @@ func main() { ) releasesService.RunBootCheck(ctx) + // Update indicator and one-click update. The release check is on by + // default (one GitHub API read per interval); applying an update needs + // the host-side updater (UPDATER_URL), which the compose stack ships as + // the "updater" profile. + updateInterval, _ := time.ParseDuration(getenvDefault("UPDATE_CHECK_INTERVAL", "30m")) + updatesService = updates.New(updates.Config{ + Enabled: getenvDefault("UPDATE_CHECK_ENABLED", "true") != "false", + Interval: updateInterval, + Channel: getenvDefault("UPDATE_CHANNEL", "stable"), + GithubRepo: getenvDefault("RELEASES_GITHUB_REPO", "warmbly/warmbly"), + GithubToken: os.Getenv("RELEASES_GITHUB_TOKEN"), + UpdaterURL: os.Getenv("UPDATER_URL"), + UpdaterToken: getenvDefault("UPDATER_TOKEN", os.Getenv("INTERNAL_API_TOKEN")), + }) + updatesService.Start(ctx) + eventsPublisher := events.NewPublisher(bus, s3, codecImpl, cipherService) // apiCfg.Hostname is the bind address, not a reachable base. Building // the mailbox-connect redirect_uri from it sends the provider // "0.0.0.0:8080/addresses/google/callback". oauth2Cfg := config.LoadOauth2(oauthPublicBaseURL(apiCfg.Hostname)) + // Recipient unsubscribe links live on the API origin, signed under the + // auth secret; the same base the OAuth callbacks are built on. + unsubSigner = unsublink.New(authCfg.AuthSecret, oauthPublicBaseURL(apiCfg.Hostname)) emailService = email.NewServiceWithWorker( emailRepostory, cipherService, @@ -1141,10 +1203,9 @@ func main() { ) // Fan out email-account lifecycle events to customer webhooks. emailService.WireWebhooks(webhookService) - // Same wire-after-construct pattern for the daily throttle — - // only the prod backend has a real cache; jobs / tests build - // emailService without one. - emailService.WireThrottle(dailyThrottleService) + // Every connect path checks the workspace's mailbox allowance + // (fair use for paid plans, the free cap otherwise). + emailService.WireMailboxAllowance(organizationService) // Seed Graph delta cursors when the reconciler reloads mailboxes. emailService.WireGraphDelta(repository.NewEmailGraphDeltaRepository(primaryDB)) // The Gmail equivalent: without it a reloaded mailbox re-bootstraps its @@ -1204,6 +1265,11 @@ func main() { if aware, ok := contactService.(contact.SegmentAware); ok { aware.WireSegments(segmentRepository, segmentService) } + // A new contact is an event: customer webhooks and "contact created" + // automations hear about it from the one write path every creator uses. + if aware, ok := contactService.(contact.WebhookAware); ok { + aware.WireWebhooks(webhookServiceForHandler) + } formRepository := repository.NewFormRepository(primaryDB) formEventRepository := repository.NewFormEventRepository(primaryDB) formService = form.NewService(formRepository) @@ -1216,7 +1282,7 @@ func main() { formService.SetLinks(repository.NewFormLinkRepository(primaryDB)) formService.SetEvents(formEventRepository) formService.SetDomains(organizationRepoForHandler) - go jobs.NewFormEventsRetentionJob(formEventRepository).Start(ctx, 12*time.Hour) + go jobs.NewFormEventsRetentionJob(formEventRepository).WireRetention(instanceSettings).Start(ctx, 12*time.Hour) go jobs.NewFormsDomainSweep(organizationRepoForHandler).Start(ctx, time.Hour) // A visibly bad import is filed on the workspace's posture. On its own // it can only reach `watch`, which changes nothing. @@ -1231,6 +1297,9 @@ func main() { leadSyncServiceForHandler = leadsync.NewService(leadSyncRepository, integrationServiceForHandler, contactService) apiKeyService = apikey.NewService(cache, apiKeyRepository) + // `warmbly auth login`: the browser approval mints an ordinary API key + // through the service above, so it has to be built after it. + cliAuthService = cliauth.NewService(repository.NewCLIAuthRepository(primaryDB.Pool), apiKeyService, organizationService, userService, organizationRepository) crmService = crm.NewService(crmRepository) teamRepository := repository.NewTeamRepository(primaryDB.Pool) teamService = team.NewService(teamRepository) @@ -1300,20 +1369,29 @@ func main() { if aware, ok := campaignService.(campaign.ProgressAware); ok { aware.WireProgress(campaignProgressRepository) } + // The wizard's audience-versus-pool estimate counts segment members. + if aware, ok := campaignService.(campaign.SegmentAware); ok { + aware.WireSegments(segmentService) + } // Delete drops attachment objects and duplicate copies them, so the // campaign service needs the store the attachment handler writes to. if aware, ok := campaignService.(campaign.AttachmentAware); ok { aware.WireAttachments(attachmentRepoForHandler, s3ForHandler) } + // Deleting a step cascades its attachment rows away, so the sequence + // service needs the same store to drop the objects behind them. + if aware, ok := sequenceService.(sequence.AttachmentAware); ok { + aware.WireAttachments(attachmentRepoForHandler, s3ForHandler) + } // Attaching a lead to a running campaign has to wake that campaign's // parked send chain, or the lead sits queued until the chain's next // tick. Wired here because contactService is built before the scheduler // and Cloud Tasks client exist. if segmentService != nil { segmentService.SetCampaignWaker(campaignService) - // A completed campaign whose linked segments grow is restarted - // through the full launch checks, never by a raw status flip. - segmentService.SetCampaignStarter(campaignService) + // Sweep enrolments are audited as campaign updates so teammates' + // Leads tabs refresh through the audit spine. + segmentService.SetEnrolmentAuditor(auditService) } if contactService != nil { contactService.SetCampaignWaker(campaignService) @@ -1430,9 +1508,10 @@ func main() { // the advanced/contact/org services exist (the integration service was // constructed earlier). integrationServiceForHandler.SetNativeActions(nativeactions.Adapter{ - Adv: advancedService, - Contacts: contactRepostory, - Orgs: organizationRepository, + Adv: advancedService, + Contacts: contactRepostory, + Orgs: organizationRepository, + ContactSvc: contactService, }) integrationServiceForHandler.SetPublisher(streamingPublisher) // AI automation nodes (ai_step / ai_switch) run over the same provider + @@ -1517,6 +1596,7 @@ func main() { trackedLinkRepository, integrationServiceForHandler, // AutomationRunner for campaign run_automation steps ) + tasksService.SetUnsubscribeLinks(unsubSigner) // Sequence action nodes that pin a contact into or out of a segment, // both on the scheduled path (tasks) and the instant reply path (advanced). if aware, ok := tasksService.(tasks.SegmentAware); ok { @@ -1658,10 +1738,11 @@ func main() { orgTransferScheduler := jobs.NewOrgTransferScheduler(orgTransferJob, 1*time.Hour) go orgTransferScheduler.Start(ctx) - // Prune audit entries past the retention window (90 days). Bounding the - // trail's age also bounds how long PII is retained. auditRepository is + // Prune audit entries past the retention window. Bounding the trail's + // age also bounds how long PII is retained, so the window is an + // instance setting and is read on every pass. auditRepository is // constructed earlier (before authService). - auditRetentionJob := jobs.NewAuditRetentionJob(auditRepository, 90*24*time.Hour) + auditRetentionJob := jobs.NewAuditRetentionJob(auditRepository).WireRetention(instanceSettings) auditRetentionScheduler := jobs.NewAuditRetentionScheduler(auditRetentionJob, 6*time.Hour) go auditRetentionScheduler.Start(ctx) @@ -1838,6 +1919,7 @@ func main() { Policy: authPolicy, DB: instanceChecksDB, Cache: authCache, + Updates: updatesService, }) h := &handler.Handler{ @@ -1853,9 +1935,11 @@ func main() { InstanceRuntime: instanceRuntime, InstanceChecks: instanceChecks, InstanceSettings: instanceSettings, + OpsNotifier: opsNotifier, PoolLinkService: poolLinkService, CloudLinkService: cloudLinkService, + CLIAuthService: cliAuthService, TokenService: tokenService, PasskeyService: passkeyService, @@ -1918,12 +2002,14 @@ func main() { WorkerRepo: workerRepoForHandler, CredentialsRepo: credentialsRepository, ReleasesService: releasesService, + UpdatesService: updatesService, // Notifications EmailNotificationService: emailNotificationService, // Advanced outreach controls - AdvancedService: advancedService, + AdvancedService: advancedService, + UnsubscribeLinks: unsubSigner, // Warmup health WarmupService: warmupService, @@ -2137,3 +2223,33 @@ func (m advisorMembers) MemberPermissions(ctx context.Context, orgID, userID uui } return member.Permissions, nil } + +// bootstrapInstanceSettings applies WARMBLY_SETTINGS_BOOTSTRAP once, on an +// instance whose settings document has never been written. It is how an +// unattended install ships its data-control answers (what is imported, what is +// kept and for how long) with the rest of the environment, so the wizard's +// choices are in place before the first mailbox is connected instead of being +// something the operator has to redo in the panel. +// +// The body is the same partial document PUT /admin/instance/settings takes. +// From the first write onwards the panel is authoritative and this is a no-op, +// so the variable can stay in .env without ever undoing a later edit. +func bootstrapInstanceSettings(ctx context.Context, svc instancesettings.Service) { + raw := strings.TrimSpace(os.Getenv("WARMBLY_SETTINGS_BOOTSTRAP")) + if raw == "" || svc == nil { + return + } + var patch instancesettings.Patch + if err := json.Unmarshal([]byte(raw), &patch); err != nil { + log.Printf("WARMBLY_SETTINGS_BOOTSTRAP is not valid JSON and was ignored: %v", err) + return + } + applied, err := svc.Bootstrap(ctx, patch) + if err != nil { + log.Printf("WARMBLY_SETTINGS_BOOTSTRAP could not be applied: %v", err) + return + } + if applied { + log.Printf("instance settings seeded from WARMBLY_SETTINGS_BOOTSTRAP") + } +} diff --git a/cmd/cli/alias.go b/cmd/cli/alias.go new file mode 100644 index 00000000..de0dd0e6 --- /dev/null +++ b/cmd/cli/alias.go @@ -0,0 +1,112 @@ +package main + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" +) + +// Aliases are plain command lines stored in config.yml. They are expanded +// before cobra sees the arguments, so an alias can carry flags and the user +// can still add more on the end. +func newAliasCmd(f *Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "alias ", + Short: "Shortcuts for command lines you type often", + GroupID: groupSetup, + Long: `Save a command line under a shorter name. + +Anything after the alias on the command line is appended, so an alias can be a +starting point rather than a fixed command.`, + Example: ` $ warmbly alias set hot "campaign list --status active" + $ warmbly hot --json`, + } + + set := &cobra.Command{ + Use: "set ", + Short: "Create or replace an alias", + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + name, expansion := args[0], args[1] + if strings.ContainsAny(name, " \t") { + return fmt.Errorf("an alias name cannot contain spaces") + } + // Shadowing a real command would make it unreachable, and the + // person who did it would have no way to tell why. + for _, existing := range c.Root().Commands() { + if existing.Name() == name { + return fmt.Errorf("%q is already a warmbly command, so an alias for it would hide it", name) + } + } + if _, err := splitArgs(expansion); err != nil { + return err + } + cfg, err := f.Config() + if err != nil { + return err + } + if cfg.Aliases == nil { + cfg.Aliases = map[string]string{} + } + cfg.Aliases[name] = expansion + if err := cfg.Save(); err != nil { + return err + } + f.IO.Errorf("%s %s = %s\n", f.IO.Tick(), name, expansion) + return nil + }, + } + + list := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "Show every alias", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + if len(cfg.Aliases) == 0 { + f.IO.Println(f.IO.Gray("No aliases yet. Try: warmbly alias set hot \"campaign list --status active\"")) + return nil + } + names := make([]string, 0, len(cfg.Aliases)) + for name := range cfg.Aliases { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + f.IO.Printf("%-12s %s\n", name, cfg.Aliases[name]) + } + return nil + }, + } + + del := &cobra.Command{ + Use: "delete ", + Aliases: []string{"rm"}, + Short: "Remove an alias", + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + if _, ok := cfg.Aliases[args[0]]; !ok { + return fmt.Errorf("no alias called %q", args[0]) + } + delete(cfg.Aliases, args[0]) + if err := cfg.Save(); err != nil { + return err + } + f.IO.Errorf("%s removed %s\n", f.IO.Tick(), args[0]) + return nil + }, + } + + cmd.AddCommand(set, list, del) + return cmd +} diff --git a/cmd/cli/api.go b/cmd/cli/api.go new file mode 100644 index 00000000..31f24e04 --- /dev/null +++ b/cmd/cli/api.go @@ -0,0 +1,251 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/output" +) + +// newAPICmd is the escape hatch that makes the typed commands optional: every +// endpoint is reachable on day one, whether or not a noun-verb command for it +// exists yet. +func newAPICmd(f *Factory) *cobra.Command { + var ( + method string + rawField []string + field []string + headers []string + input string + paginate bool + maxPages int + include bool + silent bool + idemKey string + ) + + cmd := &cobra.Command{ + Use: "api ", + Short: "Call any Warmbly API endpoint", + GroupID: groupDevelop, + Long: `Make an authenticated request to the Warmbly REST API. + +The endpoint is relative to /v1 unless it already names a version, so +"/campaigns" and "/v1/campaigns" are the same call. The method defaults to GET, +or POST when any field is supplied. + +Fields build a JSON body. -f keeps the value a string; -F guesses the type, so +true, false, null and numbers arrive as themselves, and @file or @- reads a +value from a file or stdin. Nested keys use key[sub]=value and repeated key[] +builds an array.`, + Example: ` $ warmbly api /me + $ warmbly api "/campaigns?limit=10" --paginate + $ warmbly api /contacts -f email=jane@example.com -f first_name=Jane + $ warmbly api /campaigns/CAMPAIGN_ID -X PATCH -F daily_limit=40 + $ warmbly api /contacts/search -X POST --input filter.json + $ warmbly api /webhooks/WEBHOOK_ID -X DELETE`, + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + client, err := f.Client() + if err != nil { + return err + } + + parsed, err := url.Parse(args[0]) + if err != nil { + return fmt.Errorf("%q is not a usable endpoint: %w", args[0], err) + } + + body, err := bodyFromArg(f.IO.In, input) + if err != nil { + return err + } + fields, err := buildFields(rawField, field, f.IO.In) + if err != nil { + return err + } + if len(fields) > 0 { + if body != nil { + return fmt.Errorf("pass fields or --input, not both") + } + body, err = json.Marshal(fields) + if err != nil { + return err + } + } + if body != nil && !json.Valid(body) { + return fmt.Errorf("the request body is not valid JSON") + } + + if method == "" { + method = http.MethodGet + if body != nil { + method = http.MethodPost + } + } + method = strings.ToUpper(method) + if method == http.MethodGet && body != nil { + return fmt.Errorf("a GET request carries no body. Put parameters in the query string, or pass -X POST.") + } + + req := api.Request{ + Method: method, + Path: parsed.Path, + Query: parsed.Query(), + Body: body, + IdempotencyKey: idemKey, + Headers: map[string]string{}, + } + for _, h := range headers { + name, value, ok := strings.Cut(h, ":") + if !ok { + return fmt.Errorf("headers are name:value, not %q", h) + } + req.Headers[strings.TrimSpace(name)] = strings.TrimSpace(value) + } + + if paginate { + if method != http.MethodGet { + return fmt.Errorf("--paginate only makes sense on a GET") + } + merged, perr := client.Paginate(c.Context(), req, maxPages) + if perr != nil { + return perr + } + if silent { + return nil + } + return (&output.Printer{IO: f.IO, JSON: true}).Print(merged, output.Table{}) + } + + resp, err := client.Do(c.Context(), req) + if resp != nil && include { + f.IO.Printf("HTTP %d\n", resp.Status) + for name, values := range resp.Header { + for _, v := range values { + f.IO.Printf("%s: %s\n", name, v) + } + } + f.IO.Println() + } + if err != nil { + return err + } + if silent { + return nil + } + return (&output.Printer{IO: f.IO, JSON: true, Template: f.Template}).Print(resp.Body, output.Table{}) + }, + } + + cmd.Flags().StringVarP(&method, "method", "X", "", "HTTP method (default GET, or POST when fields are given)") + cmd.Flags().StringArrayVarP(&rawField, "raw-field", "f", nil, "Body field as a string: key=value") + cmd.Flags().StringArrayVarP(&field, "field", "F", nil, "Body field with a guessed type: key=value, key=@file") + cmd.Flags().StringArrayVarP(&headers, "header", "H", nil, "Extra request header: name:value") + cmd.Flags().StringVar(&input, "input", "", "Request body: JSON, @file, or - for stdin") + cmd.Flags().BoolVar(&paginate, "paginate", false, "Follow the cursor and merge every page") + cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Stop after this many pages") + cmd.Flags().BoolVarP(&include, "include", "i", false, "Print the status and response headers too") + cmd.Flags().BoolVar(&silent, "silent", false, "Do not print the response body") + cmd.Flags().StringVar(&idemKey, "idempotency-key", "", "Idempotency-Key header for a safely retryable write") + return cmd +} + +// buildFields turns -f and -F into one JSON object. Order matters only for +// duplicate keys, where the last one wins, as in curl. +func buildFields(raw, typed []string, stdin io.Reader) (map[string]any, error) { + out := map[string]any{} + for _, kv := range raw { + key, value, ok := strings.Cut(kv, "=") + if !ok { + return nil, fmt.Errorf("fields are key=value, not %q", kv) + } + if err := assign(out, key, value); err != nil { + return nil, err + } + } + for _, kv := range typed { + key, value, ok := strings.Cut(kv, "=") + if !ok { + return nil, fmt.Errorf("fields are key=value, not %q", kv) + } + if strings.HasPrefix(value, "@") { + source := value + if value == "@-" { + source = "-" + } + data, err := bodyFromArg(stdin, source) + if err != nil { + return nil, err + } + if err := assign(out, key, strings.TrimRight(string(data), "\n")); err != nil { + return nil, err + } + continue + } + if err := assign(out, key, guessType(value)); err != nil { + return nil, err + } + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +// guessType is the -F conversion: JSON literals become themselves, everything +// else stays a string. +func guessType(v string) any { + switch v { + case "true": + return true + case "false": + return false + case "null": + return nil + } + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + return n + } + if fl, err := strconv.ParseFloat(v, 64); err == nil { + return fl + } + return v +} + +// assign writes one field, understanding key[sub] for nesting and key[] for +// appending to an array. +func assign(obj map[string]any, key string, value any) error { + open := strings.Index(key, "[") + if open < 0 { + obj[key] = value + return nil + } + if !strings.HasSuffix(key, "]") { + return fmt.Errorf("unbalanced brackets in field %q", key) + } + head := key[:open] + inner := key[open+1 : len(key)-1] + if head == "" { + return fmt.Errorf("field %q has no name", key) + } + if inner == "" { + existing, _ := obj[head].([]any) + obj[head] = append(existing, value) + return nil + } + nested, ok := obj[head].(map[string]any) + if !ok { + nested = map[string]any{} + obj[head] = nested + } + return assign(nested, inner, value) +} diff --git a/cmd/cli/auth.go b/cmd/cli/auth.go new file mode 100644 index 00000000..8811a01d --- /dev/null +++ b/cmd/cli/auth.go @@ -0,0 +1,747 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/app/apikey" + "github.com/warmbly/warmbly/internal/app/oauth" + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/models" +) + +func newAuthCmd(f *Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "auth ", + Short: "Sign in, sign out, and see who you are", + GroupID: groupCore, + Long: `Sign the CLI in to a Warmbly instance. + +Signing in through the browser creates one API key named for this machine. It +appears under Settings > API keys and can be revoked there or with +` + "`warmbly auth logout`" + `. Credentials are written to ` + config.HostsPath() + ` +at 0600, and WARMBLY_TOKEN overrides the file without ever being written to it, +which is how CI authenticates with no login step.`, + } + cmd.AddCommand( + newAuthLoginCmd(f), + newAuthStatusCmd(f), + newAuthTokenCmd(f), + newAuthSwitchCmd(f), + newAuthRefreshCmd(f), + newAuthLogoutCmd(f), + ) + return cmd +} + +// parseScopes accepts the two presets plus any comma or space separated list +// of scope names, in either case, so `--scopes read_campaigns,READ_CONTACTS` +// works and a typo is named rather than silently dropped. +func parseScopes(raw string) (uint64, error) { + raw = strings.TrimSpace(raw) + switch strings.ToLower(raw) { + case "": + return models.APIPermFullAccess, nil + case "full", "full-access", "all": + return models.APIPermFullAccess, nil + case "read", "read-only", "readonly": + return models.APIPermReadOnly, nil + } + mask, unknown := oauth.ParseScopes(strings.NewReplacer(",", " ", "+", " ").Replace(raw)) + if len(unknown) > 0 { + return 0, fmt.Errorf("unknown scope %s.\nRun `warmbly key permissions` for the full list, or use the presets: full, read-only.", strings.Join(unknown, ", ")) + } + if mask == 0 { + return 0, fmt.Errorf("--scopes granted nothing. Name at least one scope, or use full or read-only.") + } + return mask, nil +} + +func newAuthLoginCmd(f *Factory) *cobra.Command { + var ( + hostname string + apiURL string + withToken bool + web bool + scopeStr string + force bool + ) + cmd := &cobra.Command{ + Use: "login", + Short: "Sign in to a Warmbly instance", + Long: `Sign in to a Warmbly instance. + +With no flags this asks which instance, then how: a browser approval that +creates a key for this machine, or pasting a key you already have.`, + Example: ` $ warmbly auth login + $ warmbly auth login --hostname warmbly.acme.com + $ warmbly auth login --scopes read-only + $ echo $KEY | warmbly auth login --with-token`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runAuthLogin(c.Context(), f, hostname, apiURL, withToken, web, scopeStr, force) + }, + } + cmd.Flags().StringVar(&hostname, "hostname", "", "Instance to sign in to (default: warmbly.com)") + cmd.Flags().StringVar(&apiURL, "api-url", "", "API base URL, when it is not derivable from the hostname") + cmd.Flags().BoolVar(&withToken, "with-token", false, "Read an API key from stdin instead of using the browser") + cmd.Flags().BoolVarP(&web, "web", "w", false, "Go straight to the browser approval") + cmd.Flags().StringVarP(&scopeStr, "scopes", "s", "", "Scopes to request: full, read-only, or a list of names") + cmd.Flags().BoolVar(&force, "force", false, "Replace an existing sign-in for this host without asking") + return cmd +} + +func runAuthLogin(ctx context.Context, f *Factory, hostname, apiURL string, withToken, web bool, scopeStr string, force bool) error { + io := f.IO + cfg, err := f.Config() + if err != nil { + return err + } + hosts, err := f.Hosts() + if err != nil { + return err + } + + // 1. Which instance. + if hostname == "" && apiURL == "" { + if !io.IsStdinTTY() { + hostname = config.DefaultHost + } else { + idx, serr := io.Select("Where do you want to sign in?", []string{ + "warmbly.com (the hosted service)", + "A self-hosted instance", + }) + if serr != nil { + return serr + } + if idx == 0 { + hostname = config.DefaultHost + } else { + answer, ierr := io.Input("Instance hostname (for example warmbly.acme.com)", "") + if ierr != nil { + return ierr + } + if strings.TrimSpace(answer) == "" { + return fmt.Errorf("a hostname is required to sign in to a self-hosted instance") + } + hostname = answer + } + } + } + if hostname == "" { + hostname = apiURL + } + host := config.NormalizeHost(hostname) + + if existing := hosts[host]; existing != nil && !force { + if !io.IsStdinTTY() { + return fmt.Errorf("already signed in to %s as %s. Pass --force to replace that sign-in.", host, existing.User) + } + ok, cerr := io.Confirm(fmt.Sprintf("Already signed in to %s as %s. Sign in again?", host, existing.User), false) + if cerr != nil { + return cerr + } + if !ok { + return errCancelled + } + } + + // 2. Where its API is. + base, err := resolveAPIBase(ctx, f, host, apiURL) + if err != nil { + return err + } + + // 3. How to authenticate. + useToken := withToken + if !withToken && !web && io.IsStdinTTY() { + idx, serr := io.Select("How do you want to sign in?", []string{ + "Approve in a browser (creates a key for this machine)", + "Paste an API key you already have", + }) + if serr != nil { + return serr + } + useToken = idx == 1 + } + + scopes, err := parseScopes(scopeStr) + if err != nil { + return err + } + + var entry *config.Host + if useToken { + entry, err = loginWithToken(ctx, f, base) + } else { + entry, err = loginWithBrowser(ctx, f, cfg, base, scopes) + } + if err != nil { + return err + } + + hosts[host] = entry + if err := hosts.Save(); err != nil { + return err + } + // Signing in makes that host the active one. Nobody expects to sign in and + // still have the next command talk to somewhere else. + if cfg.ActiveHost != host { + cfg.ActiveHost = host + if err := cfg.Save(); err != nil { + return err + } + } + + io.Errorf("%s Signed in to %s as %s\n", io.Tick(), io.Bold(host), io.Bold(entry.User)) + if entry.Organization != "" { + io.Errorf(" Workspace %s\n", entry.Organization) + } + io.Errorf(" Credentials written to %s\n", config.HostsPath()) + return nil +} + +// resolveAPIBase finds the API for a host, probing the layouts the installer +// writes. Guessing wrong here produces a confusing 404 on every later command, +// so it is settled once, at sign-in, and stored. +func resolveAPIBase(ctx context.Context, f *Factory, host, explicit string) (string, error) { + if explicit != "" { + return strings.TrimRight(explicit, "/"), nil + } + if v := strings.TrimSpace(os.Getenv(config.APIURLEnv)); v != "" { + return strings.TrimRight(v, "/"), nil + } + if host == config.DefaultHost { + return config.DefaultAPIURL(host), nil + } + + candidates := config.CandidateAPIURLs(host) + for _, base := range candidates { + if reachable(ctx, f, base) { + if f.Debug { + f.IO.Errorf("* resolved API base %s\n", base) + } + return base, nil + } + } + return "", fmt.Errorf("could not find a Warmbly API for %s. Tried:\n %s\nPass --api-url with the instance's API base URL (the API_PUBLIC_URL it is configured with).", host, strings.Join(candidates, "\n ")) +} + +// deploymentConfig is the slice of GET /auth/config the CLI uses: enough to +// tell a Warmbly API from any other 200, plus the two URLs a client cannot +// derive on a self-hosted instance. +type deploymentConfig struct { + Registration string `json:"registration"` + AppURL string `json:"app_url"` + WebsocketURL string `json:"websocket_url"` +} + +// fetchDeploymentConfig reads the public deployment facts, or nil when the +// base is not a Warmbly API. /health alone would accept any 200. +func fetchDeploymentConfig(ctx context.Context, f *Factory, base string) *deploymentConfig { + client := api.New(base, "", UserAgent()) + client.HTTP.Timeout = 8 * time.Second + if f.Debug { + client.Debug = f.IO.ErrOut + } + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + resp, err := client.Do(probeCtx, api.Request{Method: http.MethodGet, Path: "/auth/config", Anonymous: true}) + if err != nil || resp == nil || resp.Status != http.StatusOK { + return nil + } + var probe deploymentConfig + if json.Unmarshal(resp.Body, &probe) != nil || probe.Registration == "" { + return nil + } + return &probe +} + +func reachable(ctx context.Context, f *Factory, base string) bool { + return fetchDeploymentConfig(ctx, f, base) != nil +} + +func loginWithToken(ctx context.Context, f *Factory, base string) (*config.Host, error) { + io := f.IO + if io.IsStdinTTY() { + io.Errorln(io.Gray("Create a key under Settings > API keys, then paste it here. It is not echoed.")) + } + token, err := io.Secret("API key") + if err != nil { + return nil, err + } + token = strings.TrimSpace(token) + if token == "" { + return nil, fmt.Errorf("no key was given") + } + if !strings.HasPrefix(token, apikey.KeyPrefix) { + return nil, fmt.Errorf("that does not look like a Warmbly API key: it should start with %q", apikey.KeyPrefix) + } + + entry := &config.Host{APIURL: base, Token: token, AddedAt: time.Now().UTC()} + if err := fillIdentity(ctx, f, entry); err != nil { + return nil, err + } + if cfg := fetchDeploymentConfig(ctx, f, base); cfg != nil { + entry.AppURL = cfg.AppURL + } + return entry, nil +} + +func loginWithBrowser(ctx context.Context, f *Factory, cfg *config.Config, base string, scopes uint64) (*config.Host, error) { + io := f.IO + client := api.New(base, "", UserAgent()) + if f.Debug { + client.Debug = io.ErrOut + } + + machine, _ := os.Hostname() + start, err := startDeviceFlow(ctx, client, machine, scopes) + if err != nil { + return nil, err + } + + target := start.VerificationURLComplete + if target == "" { + target = start.VerificationURL + } + io.Errorf("\n %s %s\n", io.Gray("Your code:"), io.Bold(start.UserCode)) + io.Errorf(" %s %s\n\n", io.Gray("Approve at:"), target) + + if io.IsStdinTTY() { + if ok, cerr := io.Confirm("Open that in your browser now?", true); cerr == nil && ok { + if berr := openBrowser(cfg, target); berr != nil { + io.Errorln(io.Gray("Could not open a browser. Use the link above.")) + } + } + } + + io.Errorf("%s Waiting for approval (the code expires in %d minutes)\n", io.Gray("…"), start.ExpiresIn/60) + result, err := pollDeviceFlow(ctx, client, start) + if err != nil { + return nil, err + } + + entry := &config.Host{ + APIURL: base, + AppURL: appURLFromVerification(start.VerificationURL), + Token: result.Token, + User: result.UserEmail, + UserID: result.UserID, + Organization: result.OrganizationName, + OrganizationID: result.OrganizationID, + Scopes: result.ScopeNames, + APIKeyID: result.APIKeyID, + AddedAt: time.Now().UTC(), + } + if entry.User == "" { + // The approval did not carry an identity; ask the API who we are. + if err := fillIdentity(ctx, f, entry); err != nil { + return nil, err + } + } + return entry, nil +} + +// appURLFromVerification recovers the dashboard origin from the approval link +// the instance just handed us, which is the instance's own APP_URL and so is +// exact where a hostname guess is not. +func appURLFromVerification(raw string) string { + return strings.TrimSuffix(strings.TrimSpace(raw), "/cli") +} + +// fillIdentity calls /me, which both validates the credential and gives the +// host entry the labels `auth status` prints. +func fillIdentity(ctx context.Context, f *Factory, entry *config.Host) error { + client := api.New(entry.APIURL, entry.Token, UserAgent()) + if f.Debug { + client.Debug = f.IO.ErrOut + } + var id models.Identity + if err := client.JSON(ctx, api.Request{Method: http.MethodGet, Path: "/me"}, &id); err != nil { + if api.StatusOf(err) == http.StatusUnauthorized { + return fmt.Errorf("that key was rejected by %s. Check it is a key for this instance and has not been revoked.", entry.APIURL) + } + return err + } + entry.User = id.Email + entry.UserID = id.UserID.String() + entry.Scopes = id.Scopes + entry.Organization = id.OrganizationName + if id.OrganizationID != nil { + entry.OrganizationID = id.OrganizationID.String() + } + return nil +} + +func newAuthStatusCmd(f *Factory) *cobra.Command { + var showToken bool + cmd := &cobra.Command{ + Use: "status", + Short: "Show which hosts you are signed in to", + Long: `Show every signed-in host, who you are on it, and where the credential +came from. Run this first when a command fails with a credential error: an +environment variable nobody remembers exporting is the usual answer.`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runAuthStatus(c.Context(), f, showToken) + }, + } + cmd.Flags().BoolVarP(&showToken, "show-token", "t", false, "Print the token itself") + return cmd +} + +func runAuthStatus(ctx context.Context, f *Factory, showToken bool) error { + io := f.IO + cfg, err := f.Config() + if err != nil { + return err + } + hosts, err := f.Hosts() + if err != nil { + return err + } + + resolved, resolveErr := f.Resolved() + names := hosts.Names() + // A token from the environment points at a host that may not be in the + // file at all; it still deserves a line. + if resolveErr == nil && hosts[resolved.Host] == nil { + names = append(names, resolved.Host) + } + + if len(names) == 0 { + io.Errorf("%s Not signed in anywhere.\n", io.Cross()) + io.Errorln(io.Gray("Run `warmbly auth login` to sign in.")) + return errSilent + } + sort.Strings(names) + + failed := false + for _, name := range names { + active := resolveErr == nil && name == resolved.Host + marker := " " + if active { + marker = io.Green("* ") + } + io.Printf("%s%s\n", marker, io.Bold(name)) + + entry := hosts[name] + token := "" + source := config.HostsPath() + base := "" + if entry != nil { + token, base = entry.Token, entry.APIURL + } + if active { + token, source, base = resolved.Token, resolved.Source, resolved.APIURL + } + + if token == "" { + io.Printf(" %s no credential\n", io.Cross()) + failed = true + continue + } + + probe := &config.Host{APIURL: base, Token: token} + if err := fillIdentity(ctx, f, probe); err != nil { + io.Printf(" %s %s\n", io.Cross(), err.Error()) + failed = true + } else { + io.Printf(" %s signed in as %s\n", io.Tick(), io.Bold(probe.User)) + if probe.Organization != "" { + io.Printf(" - workspace: %s\n", probe.Organization) + } + io.Printf(" - scopes: %s\n", scopeSummary(probe.Scopes)) + } + io.Printf(" - api: %s\n", base) + io.Printf(" - token from: %s\n", source) + if showToken { + io.Printf(" - token: %s\n", token) + } else { + io.Printf(" - token: %s\n", maskToken(token)) + } + } + + if cfg.ActiveHost != "" && len(names) > 1 { + io.Println() + io.Println(io.Gray("The * host is the one commands use. `warmbly auth switch` changes it.")) + } + if failed { + return errSilent + } + return nil +} + +// scopeSummary keeps a full-access key from printing twenty-four lines. +func scopeSummary(scopes []string) string { + if len(scopes) == 0 { + return "none reported" + } + if len(scopes) >= len(models.AllAPIPermissions) { + return fmt.Sprintf("all %d", len(scopes)) + } + if len(scopes) > 6 { + return fmt.Sprintf("%s and %d more", strings.Join(scopes[:6], ", "), len(scopes)-6) + } + return strings.Join(scopes, ", ") +} + +func maskToken(t string) string { + if len(t) <= 12 { + return strings.Repeat("*", len(t)) + } + return t[:8] + strings.Repeat("*", 8) + t[len(t)-4:] +} + +func newAuthTokenCmd(f *Factory) *cobra.Command { + return &cobra.Command{ + Use: "token", + Short: "Print the token the CLI is using", + Long: `Print the active token on stdout and nothing else, so it can be piped +into another tool or exported into a CI environment.`, + Example: ` $ export WARMBLY_TOKEN=$(warmbly auth token) + $ curl -H "Authorization: Bearer $(warmbly auth token)" https://api.warmbly.com/v1/me`, + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + r, err := f.Resolved() + if err != nil { + return err + } + f.IO.Println(r.Token) + return nil + }, + } +} + +func newAuthSwitchCmd(f *Factory) *cobra.Command { + var hostname string + cmd := &cobra.Command{ + Use: "switch", + Short: "Change which signed-in host commands use", + Args: cobra.MaximumNArgs(1), + RunE: func(c *cobra.Command, args []string) error { + if len(args) == 1 { + hostname = args[0] + } + cfg, err := f.Config() + if err != nil { + return err + } + hosts, err := f.Hosts() + if err != nil { + return err + } + names := hosts.Names() + if len(names) == 0 { + return fmt.Errorf("not signed in anywhere. Run `warmbly auth login` first.") + } + if hostname == "" { + if len(names) == 1 { + hostname = names[0] + } else { + labels := make([]string, len(names)) + for i, n := range names { + labels[i] = n + if hosts[n].User != "" { + labels[i] += " " + f.IO.Gray(hosts[n].User) + } + } + idx, serr := f.IO.Select("Use which host?", labels) + if serr != nil { + return serr + } + hostname = names[idx] + } + } + host := config.NormalizeHost(hostname) + if hosts[host] == nil { + return fmt.Errorf("not signed in to %s. Signed in to: %s", host, strings.Join(names, ", ")) + } + cfg.ActiveHost = host + if err := cfg.Save(); err != nil { + return err + } + f.IO.Errorf("%s Now using %s\n", f.IO.Tick(), f.IO.Bold(host)) + return nil + }, + } + cmd.Flags().StringVar(&hostname, "hostname", "", "Host to switch to") + return cmd +} + +func newAuthRefreshCmd(f *Factory) *cobra.Command { + var scopeStr string + cmd := &cobra.Command{ + Use: "refresh", + Short: "Sign in again, usually to add scopes", + Long: `Run the browser sign-in again for the active host. + +A key's scopes are fixed when it is created, so widening what the CLI may do +means a new key. The old one is revoked once the new one works.`, + Example: ` $ warmbly auth refresh --scopes full + $ warmbly auth refresh --scopes read_campaigns,send_campaigns`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + r, err := f.Resolved() + if err != nil { + return err + } + old := r.Entry + if err := runAuthLogin(c.Context(), f, r.Host, r.APIURL, false, true, scopeStr, true); err != nil { + return err + } + // Revoke the key we replaced, so refreshing does not accumulate a + // key per run under Settings > API keys. + if old != nil && old.APIKeyID != "" { + if err := revokeKey(c.Context(), f, old.APIKeyID); err != nil && f.Debug { + f.IO.Errorf("* could not revoke the previous key: %v\n", err) + } + } + return nil + }, + } + cmd.Flags().StringVarP(&scopeStr, "scopes", "s", "", "Scopes to request: full, read-only, or a list of names") + return cmd +} + +func newAuthLogoutCmd(f *Factory) *cobra.Command { + var hostname string + var keepKey bool + cmd := &cobra.Command{ + Use: "logout", + Short: "Sign out and revoke this machine's key", + Long: `Forget a host's credential. + +The key the sign-in created is revoked on the instance too, so signing out on +a machine you are handing back actually ends its access. --keep-key skips the +revocation, for a key you pasted in and use elsewhere.`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runAuthLogout(c.Context(), f, hostname, keepKey) + }, + } + cmd.Flags().StringVar(&hostname, "hostname", "", "Host to sign out of (default: the active one)") + cmd.Flags().BoolVar(&keepKey, "keep-key", false, "Forget the credential locally without revoking it") + return cmd +} + +func runAuthLogout(ctx context.Context, f *Factory, hostname string, keepKey bool) error { + io := f.IO + cfg, err := f.Config() + if err != nil { + return err + } + hosts, err := f.Hosts() + if err != nil { + return err + } + host := config.NormalizeHost(hostname) + if hostname == "" { + r, rerr := f.Resolved() + if rerr != nil { + return rerr + } + host = r.Host + } + entry := hosts[host] + if entry == nil { + return fmt.Errorf("not signed in to %s", host) + } + + if !f.AssumeYes && io.IsStdinTTY() { + ok, cerr := io.Confirm(fmt.Sprintf("Sign out of %s as %s?", host, entry.User), false) + if cerr != nil { + return cerr + } + if !ok { + return errCancelled + } + } + + // Revoke first: if it fails, the credential is still in the file and the + // user can retry, which is better than a live key nobody can reach. + revoked := false + if !keepKey && entry.APIKeyID != "" { + f.hosts = hosts + if err := revokeKeyWith(ctx, f, entry, entry.APIKeyID); err != nil { + io.Errorf("%s Could not revoke the key on %s: %v\n", io.Yellow("!"), host, err) + io.Errorln(io.Gray("Revoke it by hand under Settings > API keys.")) + } else { + revoked = true + } + } + + delete(hosts, host) + if err := hosts.Save(); err != nil { + return err + } + if cfg.ActiveHost == host { + cfg.ActiveHost = "" + if names := hosts.Names(); len(names) == 1 { + cfg.ActiveHost = names[0] + } + if err := cfg.Save(); err != nil { + return err + } + } + + io.Errorf("%s Signed out of %s\n", io.Tick(), io.Bold(host)) + if revoked { + io.Errorln(io.Gray("The key this machine was using has been revoked.")) + } + return nil +} + +func revokeKey(ctx context.Context, f *Factory, keyID string) error { + r, err := f.Resolved() + if err != nil { + return err + } + return revokeKeyWith(ctx, f, r.Entry, keyID) +} + +// revokeKeyWith ends a key using the credential in entry. +// +// It asks the instance to revoke the calling credential first: self-revocation +// needs no scope, so it works for a read-only sign-in, where deleting by id +// would be refused. The by-id path is the fallback for an instance that +// predates /api-keys/self, and for revoking a key that is not the caller. +func revokeKeyWith(ctx context.Context, f *Factory, entry *config.Host, keyID string) error { + if entry == nil { + return fmt.Errorf("no credential to revoke with") + } + client := api.New(entry.APIURL, entry.Token, UserAgent()) + if f.Debug { + client.Debug = f.IO.ErrOut + } + + if keyID == "" || keyID == entry.APIKeyID { + _, err := client.Do(ctx, api.Request{Method: http.MethodDelete, Path: "/api-keys/self"}) + if err == nil { + return nil + } + if status := api.StatusOf(err); status != http.StatusNotFound && status != http.StatusMethodNotAllowed { + return err + } + if keyID == "" { + return err + } + } + + _, err := client.Do(ctx, api.Request{Method: http.MethodDelete, Path: "/api-keys/" + keyID}) + if err != nil && api.StatusOf(err) == http.StatusNotFound { + // Already gone is the outcome we wanted. + return nil + } + return err +} diff --git a/cmd/cli/browse.go b/cmd/cli/browse.go new file mode 100644 index 00000000..9c1c7e89 --- /dev/null +++ b/cmd/cli/browse.go @@ -0,0 +1,144 @@ +package main + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/config" +) + +// browseTargets maps a noun to its dashboard path, so `warmbly browse +// campaigns` does not require anyone to remember the URL layout. +var browseTargets = map[string]string{ + "campaigns": "/app/campaigns", + "campaign": "/app/campaigns", + "contacts": "/app/contacts", + "contact": "/app/contacts", + "mailboxes": "/app/emails", + "mailbox": "/app/emails", + "emails": "/app/emails", + "inbox": "/app/unibox", + "unibox": "/app/unibox", + "analytics": "/app/analytics", + "automations": "/app/automations", + "forms": "/app/forms", + "templates": "/app/templates", + "crm": "/app/crm", + "audit": "/app/audit", + "keys": "/app/api-keys", + "api-keys": "/app/api-keys", + "settings": "/app/settings/profile", + "webhooks": "/app/settings/webhooks", + "billing": "/app/settings/billing", + "members": "/app/settings/members", + "deliverability": "/app/deliverability", +} + +// browseDetail is the subset that has a per-record page, for `warmbly browse +// campaign `. +var browseDetail = map[string]string{ + "campaign": "/app/campaigns", + "campaigns": "/app/campaigns", + "contact": "/app/contacts", + "contacts": "/app/contacts", + "mailbox": "/app/emails", + "mailboxes": "/app/emails", + "automation": "/app/automations", + "form": "/app/forms", + "forms": "/app/forms", +} + +func newBrowseCmd(f *Factory) *cobra.Command { + var printOnly bool + cmd := &cobra.Command{ + Use: "browse [
] []", + Short: "Open the dashboard in a browser", + GroupID: groupCore, + Long: `Open the Warmbly dashboard for the host you are signed in to. + +With a section it goes straight there; with a section and an id it opens that +record. --no-browser prints the URL instead, which is what you want over SSH.`, + Example: ` $ warmbly browse + $ warmbly browse campaigns + $ warmbly browse campaign 6f1c... + $ warmbly browse inbox --no-browser`, + Args: cobra.MaximumNArgs(2), + RunE: func(c *cobra.Command, args []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + r, err := f.Resolved() + if err != nil { + return err + } + + path := "/app/emails" + if len(args) > 0 { + section := strings.ToLower(args[0]) + if len(args) > 1 { + detail, ok := browseDetail[section] + if !ok { + return fmt.Errorf("%q has no detail page to open by id. Sections with one: %s", args[0], keysOf(browseDetail)) + } + path = detail + "/" + args[1] + } else { + target, ok := browseTargets[section] + if !ok { + return fmt.Errorf("nothing to browse called %q. Try one of: %s", args[0], keysOf(browseTargets)) + } + path = target + } + } + + url := dashboardURL(r) + path + if printOnly || !f.IO.IsStdoutTTY() { + f.IO.Println(url) + return nil + } + f.IO.Errorf("%s Opening %s\n", f.IO.Gray("→"), url) + if err := openBrowser(cfg, url); err != nil { + f.IO.Println(url) + } + return nil + }, + } + cmd.Flags().BoolVar(&printOnly, "no-browser", false, "Print the URL instead of opening it") + return cmd +} + +// dashboardURL is where this host's dashboard lives. The instance reports its +// own APP_URL at sign-in, which is exact; the derivation below is the fallback +// for a credential that came from the environment and never signed in. +func dashboardURL(r *config.Resolved) string { + if r.Entry != nil && strings.TrimSpace(r.Entry.AppURL) != "" { + return strings.TrimRight(r.Entry.AppURL, "/") + } + return appBaseURL(r.Host) +} + +// appBaseURL is the dashboard for a host, following the layout the installer +// writes: app. for a real deployment, the host itself for a local one. +func appBaseURL(host string) string { + host = config.NormalizeHost(host) + if strings.HasPrefix(host, "localhost") || strings.HasPrefix(host, "127.0.0.1") { + return "http://" + host + } + if strings.Contains(host, ":") { + return "http://" + host + } + return "https://app." + host +} + +// keysOf lists a target map's names, sorted, for an error message. +func keysOf(m map[string]string) string { + out := make([]string, 0, len(m)) + for name := range m { + out = append(out, name) + } + sort.Strings(out) + return strings.Join(out, ", ") +} diff --git a/cmd/cli/cli_test.go b/cmd/cli/cli_test.go new file mode 100644 index 00000000..8878cb16 --- /dev/null +++ b/cmd/cli/cli_test.go @@ -0,0 +1,301 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/models" +) + +func TestFillPath(t *testing.T) { + specs := []argSpec{{Name: "id"}, {Name: "step"}} + got, err := fillPath("/campaigns/{id}/steps/{step}", specs, []string{"abc", "def"}) + if err != nil { + t.Fatalf("fill: %v", err) + } + if got != "/campaigns/abc/steps/def" { + t.Errorf("path = %q", got) + } + if _, err := fillPath("/campaigns/{id}", specs[:1], []string{" "}); err == nil { + t.Error("an empty argument must be rejected rather than producing /campaigns/") + } + // A tool name can contain characters that need escaping in a path. + got, err = fillPath("/ai/tools/{name}/call", []argSpec{{Name: "name"}}, []string{"a b"}) + if err != nil { + t.Fatalf("fill: %v", err) + } + if !strings.Contains(got, "a%20b") { + t.Errorf("path segment was not escaped: %q", got) + } +} + +// Every spec has to produce a runnable command: one {} per positional argument +// and no leftovers, or the command is dead on arrival at runtime. +func TestEverySpecPathMatchesItsArguments(t *testing.T) { + seen := map[string]bool{} + for _, r := range resourceSpecs() { + if seen[r.Name] { + t.Errorf("two resources are called %q", r.Name) + } + seen[r.Name] = true + + endpoints := map[string]bool{} + for _, e := range r.Endpoints { + if endpoints[e.Name] { + t.Errorf("%s has two %q commands", r.Name, e.Name) + } + endpoints[e.Name] = true + + if e.Method == "" || e.Path == "" || e.Short == "" { + t.Errorf("%s %s is missing a method, path or summary", r.Name, e.Name) + } + if !strings.HasPrefix(e.Path, "/") { + t.Errorf("%s %s path %q must be /v1-relative", r.Name, e.Name, e.Path) + } + placeholders := strings.Count(e.Path, "{") + if placeholders != len(e.Args) { + t.Errorf("%s %s has %d placeholders and %d arguments", r.Name, e.Name, placeholders, len(e.Args)) + } + args := make([]string, len(e.Args)) + for i := range e.Args { + args[i] = "x" + if e.Args[i].Help == "" { + t.Errorf("%s %s argument %q has no help", r.Name, e.Name, e.Args[i].Name) + } + } + if _, err := fillPath(e.Path, e.Args, args); err != nil { + t.Errorf("%s %s: %v", r.Name, e.Name, err) + } + if e.Method == http.MethodGet && e.Body != bodyNone { + t.Errorf("%s %s is a GET with a body", r.Name, e.Name) + } + flags := map[string]bool{} + for _, fl := range e.Flag { + if flags[fl.Name] { + t.Errorf("%s %s declares --%s twice", r.Name, e.Name, fl.Name) + } + flags[fl.Name] = true + if fl.Help == "" { + t.Errorf("%s %s flag --%s has no help", r.Name, e.Name, fl.Name) + } + // -h is cobra's help shorthand; claiming it breaks the command. + if fl.Short == "h" { + t.Errorf("%s %s cannot use -h for --%s", r.Name, e.Name, fl.Name) + } + } + } + } +} + +// Building the whole command tree catches the failures cobra reports by +// panicking: a duplicate shorthand, a bad group id. +func TestCommandTreeBuilds(t *testing.T) { + f := NewFactory() + root := newRootCmd(f) + if len(root.Commands()) == 0 { + t.Fatal("no commands registered") + } + for _, c := range root.Commands() { + if c.GroupID == "" && c.Name() != "help" && c.Name() != "completion" { + t.Errorf("%s has no group, so it falls out of the grouped help", c.Name()) + } + for _, sub := range c.Commands() { + if sub.Short == "" { + t.Errorf("%s %s has no summary", c.Name(), sub.Name()) + } + } + } +} + +func TestBuildFields(t *testing.T) { + body, err := buildFields( + []string{"name=Jane", "note=true"}, + []string{"limit=40", "active=true", "missing=null", "tags[]=a", "tags[]=b", "nested[key]=v"}, + strings.NewReader(""), + ) + if err != nil { + t.Fatalf("build: %v", err) + } + if body["name"] != "Jane" { + t.Errorf("-f name should stay a string, got %#v", body["name"]) + } + if body["note"] != "true" { + t.Errorf("-f keeps values literal, got %#v", body["note"]) + } + if body["limit"] != int64(40) { + t.Errorf("-F limit should be a number, got %#v", body["limit"]) + } + if body["active"] != true { + t.Errorf("-F active should be a bool, got %#v", body["active"]) + } + if body["missing"] != nil { + t.Errorf("-F null should be null, got %#v", body["missing"]) + } + tags, _ := body["tags"].([]any) + if len(tags) != 2 { + t.Errorf("key[] should build an array, got %#v", body["tags"]) + } + nested, _ := body["nested"].(map[string]any) + if nested["key"] != "v" { + t.Errorf("key[sub] should nest, got %#v", body["nested"]) + } + + if _, err := buildFields([]string{"broken"}, nil, strings.NewReader("")); err == nil { + t.Error("a field with no = must be rejected") + } +} + +func TestSplitArgs(t *testing.T) { + got, err := splitArgs(`campaign list --status "in progress" --q 'x y'`) + if err != nil { + t.Fatalf("split: %v", err) + } + want := []string{"campaign", "list", "--status", "in progress", "--q", "x y"} + if len(got) != len(want) { + t.Fatalf("got %#v, want %#v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %#v, want %#v", got, want) + } + } + if _, err := splitArgs(`unbalanced "`); err == nil { + t.Error("an unbalanced quote must be an error, not a silent truncation") + } + // An empty quoted argument is still an argument. + if got, _ := splitArgs(`a "" b`); len(got) != 3 { + t.Errorf("empty quoted argument was dropped: %#v", got) + } +} + +func TestExpandAliases(t *testing.T) { + dir := t.TempDir() + t.Setenv(config.DirEnv, dir) + cfg := &config.Config{Aliases: map[string]string{"hot": "campaign list --status active"}} + if err := cfg.Save(); err != nil { + t.Fatalf("save: %v", err) + } + + f := NewFactory() + got := expandAliases(f, []string{"hot", "--json"}) + want := []string{"campaign", "list", "--status", "active", "--json"} + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("expanded to %v, want %v", got, want) + } + // A non-alias is untouched. + if got := expandAliases(f, []string{"campaign", "list"}); got[0] != "campaign" { + t.Errorf("a real command was rewritten: %v", got) + } +} + +func TestParseScopes(t *testing.T) { + if mask, err := parseScopes(""); err != nil || mask != models.APIPermFullAccess { + t.Errorf("empty should mean full access, got %d %v", mask, err) + } + if mask, err := parseScopes("read-only"); err != nil || mask != models.APIPermReadOnly { + t.Errorf("read-only preset = %d %v", mask, err) + } + mask, err := parseScopes("read_campaigns,SEND_CAMPAIGNS") + if err != nil { + t.Fatalf("named scopes: %v", err) + } + if mask&models.APIPermReadCampaigns == 0 || mask&models.APIPermSendCampaigns == 0 { + t.Errorf("named scopes did not resolve: %d", mask) + } + if _, err := parseScopes("not_a_scope"); err == nil { + t.Error("an unknown scope must be named, not dropped") + } +} + +// The device flow is the whole sign-in, so it gets an end-to-end test against +// a server that behaves like the real one: pending, then approved once. +func TestDeviceFlow(t *testing.T) { + polls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/auth/cli/code": + var req models.CLIAuthStartRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if req.CLIVersion == "" || req.Scopes == 0 { + t.Errorf("the CLI must identify itself and name its scopes: %+v", req) + } + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, `{"device_code":"dc","user_code":"ABCD-EFGH","verification_uri":"https://app.example/cli","verification_uri_complete":"https://app.example/cli?code=ABCD-EFGH","expires_in":600,"interval":1}`) + case "/v1/auth/cli/poll": + polls++ + if polls < 2 { + fmt.Fprint(w, `{"status":"pending"}`) + return + } + fmt.Fprint(w, `{"status":"approved","token":"wmbly_minted","user_email":"jane@example.com","organization_name":"Acme","api_key_id":"key-1","scope_names":["READ_CAMPAIGNS"]}`) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + defer srv.Close() + + client := api.New(srv.URL, "", "test") + start, err := startDeviceFlow(context.Background(), client, "laptop", models.APIPermReadOnly) + if err != nil { + t.Fatalf("start: %v", err) + } + if start.UserCode != "ABCD-EFGH" { + t.Errorf("user code = %q", start.UserCode) + } + + result, err := pollDeviceFlow(context.Background(), client, start) + if err != nil { + t.Fatalf("poll: %v", err) + } + if polls < 2 { + t.Errorf("the client stopped polling before approval") + } + if result.Token != "wmbly_minted" || result.UserEmail != "jane@example.com" { + t.Errorf("approval payload lost: %+v", result) + } +} + +func TestDeviceFlowStopsOnDenial(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"denied"}`) + })) + defer srv.Close() + + client := api.New(srv.URL, "", "test") + _, err := pollDeviceFlow(context.Background(), client, &deviceStart{DeviceCode: "dc", UserCode: "X", Interval: 1, ExpiresIn: 60}) + if err == nil || !strings.Contains(err.Error(), "declined") { + t.Errorf("a denial must end the wait, got %v", err) + } +} + +func TestAppURLFromVerification(t *testing.T) { + // The instance hands back its own APP_URL with /cli on the end, which is + // exact where a hostname guess is not. + if got := appURLFromVerification("https://app.acme.dev/cli"); got != "https://app.acme.dev" { + t.Errorf("got %q", got) + } + if got := appURLFromVerification("http://localhost:5173/cli"); got != "http://localhost:5173" { + t.Errorf("got %q", got) + } + if got := appURLFromVerification(""); got != "" { + t.Errorf("got %q, want empty", got) + } +} + +func TestDashboardURLPrefersWhatTheInstanceReported(t *testing.T) { + r := &config.Resolved{Host: "acme.dev", Entry: &config.Host{AppURL: "https://warmbly.acme.dev/"}} + if got := dashboardURL(r); got != "https://warmbly.acme.dev" { + t.Errorf("got %q, want the reported origin", got) + } + // With nothing reported, fall back to the layout the installer writes. + if got := dashboardURL(&config.Resolved{Host: "acme.dev"}); got != "https://app.acme.dev" { + t.Errorf("fallback = %q", got) + } +} diff --git a/cmd/cli/config.go b/cmd/cli/config.go new file mode 100644 index 00000000..a07dd679 --- /dev/null +++ b/cmd/cli/config.go @@ -0,0 +1,142 @@ +package main + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/config" +) + +func newConfigCmd(f *Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "config ", + Short: "Read and write the CLI's own settings", + GroupID: groupSetup, + Long: `Read and write ` + config.ConfigPath() + `. + +These are preferences, not credentials: credentials live in hosts.yml and are +managed with ` + "`warmbly auth`" + `.`, + } + + get := &cobra.Command{ + Use: "get ", + Short: "Print one setting", + Args: cobra.ExactArgs(1), + RunE: func(*cobra.Command, []string) error { return nil }, + } + get.RunE = func(c *cobra.Command, args []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + if !knownKey(args[0]) { + return unknownKey(args[0]) + } + f.IO.Println(cfg.Get(args[0])) + return nil + } + + set := &cobra.Command{ + Use: "set ", + Short: "Change one setting", + Example: " $ warmbly config set output json\n $ warmbly config set confirm always", + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + if err := cfg.Set(args[0], args[1]); err != nil { + return err + } + if err := cfg.Save(); err != nil { + return err + } + f.IO.Errorf("%s %s = %s\n", f.IO.Tick(), args[0], args[1]) + return nil + }, + } + + list := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "Show every setting and what it does", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + for _, k := range config.Keys { + value := cfg.Get(k.Name) + if value == "" { + value = f.IO.Gray("(unset, " + k.Default + ")") + } + f.IO.Printf("%-14s %s\n", k.Name, value) + f.IO.Printf("%-14s %s\n", "", f.IO.Gray(k.Help)) + } + f.IO.Println() + f.IO.Printf("%s %s\n", f.IO.Gray("config:"), config.ConfigPath()) + f.IO.Printf("%s %s\n", f.IO.Gray("hosts: "), config.HostsPath()) + return nil + }, + } + + clear := &cobra.Command{ + Use: "clear ", + Short: "Reset one setting to its default", + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + cfg, err := f.Config() + if err != nil { + return err + } + if !knownKey(args[0]) { + return unknownKey(args[0]) + } + // Set validates values, so clearing goes through the zero value + // directly rather than through a value it would reject. + switch args[0] { + case "active_host": + cfg.ActiveHost = "" + case "output": + cfg.Output = "" + case "confirm": + cfg.Confirm = "" + case "pager": + cfg.Pager = "" + case "browser": + cfg.Browser = "" + } + if err := cfg.Save(); err != nil { + return err + } + f.IO.Errorf("%s %s reset\n", f.IO.Tick(), args[0]) + return nil + }, + } + + cmd.AddCommand(get, set, list, clear) + return cmd +} + +func knownKey(name string) bool { + for _, k := range config.Keys { + if k.Name == name { + return true + } + } + return false +} + +func unknownKey(name string) error { + names := make([]string, 0, len(config.Keys)) + for _, k := range config.Keys { + names = append(names, k.Name) + } + sort.Strings(names) + return fmt.Errorf("unknown config key %q. Settable keys: %s", name, strings.Join(names, ", ")) +} diff --git a/cmd/cli/device.go b/cmd/cli/device.go new file mode 100644 index 00000000..eec70b54 --- /dev/null +++ b/cmd/cli/device.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/version" +) + +// The browser half of `warmbly auth login`, RFC 8628 shaped: ask for a code, +// show it, wait for a member to approve it in the browser, collect the key. + +type deviceStart struct { + DeviceCode string `json:"device_code"` + UserCode string `json:"user_code"` + VerificationURL string `json:"verification_uri"` + VerificationURLComplete string `json:"verification_uri_complete"` + ExpiresIn int `json:"expires_in"` + Interval int `json:"interval"` +} + +type devicePoll struct { + Status string `json:"status"` + Token string `json:"token"` + APIKeyID string `json:"api_key_id"` + ScopeNames []string `json:"scope_names"` + UserID string `json:"user_id"` + UserEmail string `json:"user_email"` + UserName string `json:"user_name"` + OrganizationID string `json:"organization_id"` + OrganizationName string `json:"organization_name"` +} + +// startDeviceFlow opens the handshake. The client is anonymous: there is +// nothing to authenticate with yet. +func startDeviceFlow(ctx context.Context, client *api.Client, hostname string, scopes uint64) (*deviceStart, error) { + body, err := json.Marshal(models.CLIAuthStartRequest{ + ClientName: "Warmbly CLI", + Hostname: hostname, + CLIVersion: version.String(), + Scopes: scopes, + }) + if err != nil { + return nil, err + } + resp, err := client.Do(ctx, api.Request{Method: http.MethodPost, Path: "/auth/cli/code", Body: body, Anonymous: true}) + if err != nil { + if api.StatusOf(err) == http.StatusNotFound || api.StatusOf(err) == http.StatusNotImplemented { + return nil, fmt.Errorf("%s does not support browser sign-in for the CLI.\nUse `warmbly auth login --with-token` with an API key from Settings > API keys instead.", client.BaseURL) + } + return nil, err + } + var out deviceStart + if err := json.Unmarshal(resp.Body, &out); err != nil { + return nil, fmt.Errorf("the sign-in handshake returned something unexpected: %w", err) + } + if out.DeviceCode == "" || out.UserCode == "" { + return nil, errors.New("the sign-in handshake returned no code") + } + if out.Interval <= 0 { + out.Interval = 3 + } + if out.ExpiresIn <= 0 { + out.ExpiresIn = 600 + } + return &out, nil +} + +// pollDeviceFlow waits for the browser half. It stops on approval, denial, +// expiry, or the context being cancelled, and never faster than the interval +// the server asked for. +func pollDeviceFlow(ctx context.Context, client *api.Client, start *deviceStart) (*devicePoll, error) { + body, err := json.Marshal(map[string]string{"device_code": start.DeviceCode}) + if err != nil { + return nil, err + } + interval := time.Duration(start.Interval) * time.Second + deadline := time.Now().Add(time.Duration(start.ExpiresIn) * time.Second) + + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(interval): + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("the code %s expired before it was approved. Run `warmbly auth login` again.", start.UserCode) + } + + resp, err := client.Do(ctx, api.Request{Method: http.MethodPost, Path: "/auth/cli/poll", Body: body, Anonymous: true}) + if err != nil { + // The code is gone: expired, or someone else claimed it. + if api.StatusOf(err) == http.StatusNotFound { + return nil, fmt.Errorf("the code %s is no longer valid. Run `warmbly auth login` again.", start.UserCode) + } + // A rate limit or a blip must not end a sign-in someone is + // standing at; back off and keep waiting. + if api.StatusOf(err) == http.StatusTooManyRequests || api.StatusOf(err) == 0 { + interval += time.Second + continue + } + return nil, err + } + + var out devicePoll + if err := json.Unmarshal(resp.Body, &out); err != nil { + return nil, err + } + switch out.Status { + case string(models.CLIAuthCodeApproved): + if out.Token == "" { + return nil, errors.New("the approval returned no token. Run `warmbly auth login` again.") + } + return &out, nil + case string(models.CLIAuthCodeDenied): + return nil, errors.New("the request was declined in the browser. Nothing was created.") + case string(models.CLIAuthCodeClaimed): + return nil, errors.New("that code was already used. Run `warmbly auth login` again.") + } + } +} + +// openBrowser opens a URL, honouring the browser config key and then BROWSER. +// A failure is never fatal: the URL has already been printed. +func openBrowser(cfg *config.Config, url string) error { + if custom := strings.TrimSpace(cfg.Browser); custom != "" { + parts, err := splitArgs(custom) + if err != nil || len(parts) == 0 { + return fmt.Errorf("the browser config key is not a usable command") + } + return exec.Command(parts[0], append(parts[1:], url)...).Start() + } + switch runtime.GOOS { + case "darwin": + return exec.Command("open", url).Start() + case "windows": + return exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() + default: + return exec.Command("xdg-open", url).Start() + } +} diff --git a/cmd/cli/events.go b/cmd/cli/events.go new file mode 100644 index 00000000..cee6ed11 --- /dev/null +++ b/cmd/cli/events.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gorilla/websocket" + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/cli/iostreams" +) + +// `warmbly events tail` is the terminal view of the developer WebSocket: the +// same stream the dashboard runs on, printed as it happens. It is the fastest +// way to see whether an integration is receiving what you think it is, without +// standing up a public webhook URL first. +// +// The socket speaks the Phoenix channel protocol, serializer 1.0.0: every +// frame is [join_ref, ref, topic, event, payload]. + +func newEventsCmd(f *Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "events ", + Short: "Watch live events as they happen", + GroupID: groupDevelop, + Long: `Subscribe to the workspace's live event stream. + +This needs a key with the REALTIME_SUBSCRIBE scope. Sign in again with +` + "`warmbly auth refresh --scopes full`" + ` if the key you have does not carry it.`, + } + cmd.AddCommand(newEventsTailCmd(f)) + return cmd +} + +func newEventsTailCmd(f *Factory) *cobra.Command { + var ( + intents []string + wsURL string + orgID string + compact bool + maxCount int + ) + cmd := &cobra.Command{ + Use: "tail", + Short: "Stream live events to the terminal", + Long: `Print events as Warmbly publishes them: sends, opens, clicks, replies, +inbox arrivals, campaign state, and the custom events your automations fire. + +Filter with --intent, which matches the event type as a case-insensitive +substring, so --intent EMAIL takes EMAIL_SENT, EMAIL_OPENED and EMAIL_RECEIVED. +Intents reduce traffic; they are not a permission boundary, and the key's +scopes still decide what reaches you at all.`, + Example: ` $ warmbly events tail + $ warmbly events tail --intent EMAIL --intent CAMPAIGN + $ warmbly events tail --json | jq 'select(.event_type == "EMAIL_REPLIED")'`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runEventsTail(c.Context(), f, wsURL, orgID, intents, compact, maxCount) + }, + } + cmd.Flags().StringArrayVar(&intents, "intent", nil, "Only these event families, for example EMAIL or CAMPAIGN") + cmd.Flags().StringVar(&wsURL, "url", "", "WebSocket URL, when the instance does not advertise one") + cmd.Flags().StringVar(&orgID, "org", "", "Organization to subscribe to (default: the signed-in one)") + cmd.Flags().BoolVar(&compact, "compact", false, "One line per event, even on a terminal") + cmd.Flags().IntVar(&maxCount, "count", 0, "Stop after this many events") + return cmd +} + +func runEventsTail(ctx context.Context, f *Factory, wsURL, orgID string, intents []string, compact bool, maxCount int) error { + io := f.IO + r, err := f.Resolved() + if err != nil { + return err + } + + if orgID == "" && r.Entry != nil { + orgID = r.Entry.OrganizationID + } + if orgID == "" { + // The key knows its own organization even when the config file does not. + client, cerr := f.Client() + if cerr != nil { + return cerr + } + var id struct { + OrganizationID string `json:"organization_id"` + } + if jerr := client.JSON(ctx, api.Request{Method: http.MethodGet, Path: "/me"}, &id); jerr != nil { + return jerr + } + orgID = id.OrganizationID + } + if orgID == "" { + return fmt.Errorf("this credential is not scoped to a workspace, so there is no org channel to join") + } + + endpoint, err := resolveSocketURL(ctx, f, r, wsURL) + if err != nil { + return err + } + + target := endpoint + if !strings.Contains(target, "?") { + target += "?" + } else { + target += "&" + } + target += "vsn=1.0.0&token=" + url.QueryEscape(r.Token) + + if f.Debug { + io.Errorf("* connecting to %s\n", endpoint) + } + dialer := websocket.Dialer{HandshakeTimeout: 20 * time.Second} + conn, resp, err := dialer.DialContext(ctx, target, nil) + if resp != nil && resp.Body != nil { + // The handshake response body carries the rejection reason and nothing + // the stream needs; the socket itself is what stays open. + defer resp.Body.Close() + } + if err != nil { + if resp != nil && resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("the realtime gateway refused this key.\nIt needs the REALTIME_SUBSCRIBE scope: `warmbly auth refresh --scopes full`.") + } + return fmt.Errorf("could not connect to the realtime gateway at %s: %w\nPass --url if this instance serves it somewhere else.", endpoint, err) + } + defer conn.Close() + + topic := "org:" + orgID + payload := map[string]any{} + if len(intents) > 0 { + payload["intents"] = intents + } + if err := conn.WriteJSON([]any{"1", "1", topic, "phx_join", payload}); err != nil { + return err + } + + io.Errorf("%s Listening on %s%s\n", io.Gray("…"), io.Bold(topic), intentSuffix(io, intents)) + io.Errorln(io.Gray("Ctrl-C to stop.")) + + // Heartbeats are client-initiated. The join reply carries the cadence the + // server wants; until it arrives, the documented default is safe. + heartbeat := time.NewTicker(25 * time.Second) + defer heartbeat.Stop() + done := make(chan error, 1) + + go func() { + seen := 0 + for { + var frame []json.RawMessage + if err := conn.ReadJSON(&frame); err != nil { + done <- err + return + } + if len(frame) < 5 { + continue + } + var event string + var body json.RawMessage + _ = json.Unmarshal(frame[3], &event) + body = frame[4] + + switch event { + case "phx_reply": + if hb := handleJoinReply(io, body, heartbeat); hb != nil { + done <- hb + return + } + continue + case "phx_close", "phx_error": + done <- fmt.Errorf("the channel closed. Rejoin with `warmbly events tail`.") + return + case "phx_join", "heartbeat": + continue + } + + printEvent(f, event, body, compact) + seen++ + if maxCount > 0 && seen >= maxCount { + done <- nil + return + } + } + }() + + ref := 2 + for { + select { + case <-ctx.Done(): + _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + return nil + case err := <-done: + if err != nil && !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + return err + } + return nil + case <-heartbeat.C: + ref++ + if err := conn.WriteJSON([]any{nil, fmt.Sprint(ref), "phoenix", "heartbeat", map[string]any{}}); err != nil { + return err + } + } + } +} + +// handleJoinReply reads the HELLO the org channel answers a join with: it +// carries the heartbeat cadence, so the client does not hardcode one, and it +// is where a rejected join is reported. +func handleJoinReply(io *iostreams.IOStreams, body json.RawMessage, heartbeat *time.Ticker) error { + var reply struct { + Status string `json:"status"` + Response struct { + Role string `json:"role"` + HeartbeatIntervalMS int `json:"heartbeat_interval_ms"` + Seq int64 `json:"seq"` + Reason string `json:"reason"` + } `json:"response"` + } + if err := json.Unmarshal(body, &reply); err != nil { + return nil + } + if reply.Status == "error" { + reason := reply.Response.Reason + if reason == "" { + reason = "the gateway refused the join" + } + return fmt.Errorf("could not join the channel: %s", reason) + } + if reply.Response.HeartbeatIntervalMS > 1000 { + heartbeat.Reset(time.Duration(reply.Response.HeartbeatIntervalMS) * time.Millisecond) + } + if reply.Response.Role != "" { + io.Errorf("%s\n", io.Gray("Joined as "+reply.Response.Role+".")) + } + return nil +} + +func printEvent(f *Factory, event string, body json.RawMessage, compact bool) { + io := f.IO + if f.JSONOut || !io.IsStdoutTTY() { + io.Println(strings.TrimSpace(string(body))) + return + } + + var fields map[string]any + _ = json.Unmarshal(body, &fields) + stamp := time.Now().Format("15:04:05") + + io.Printf("%s %s %s\n", io.Gray(stamp), eventColour(io, event), io.Gray(summarize(fields))) + if compact { + return + } + // The interesting ids, on one indented line, so a terminal stays readable + // while still carrying enough to look something up. + var parts []string + for _, key := range []string{"campaign_id", "contact_id", "email_id", "thread_id", "email_account_id", "name", "entity_type", "action"} { + if v, ok := fields[key]; ok && v != nil && fmt.Sprint(v) != "" { + parts = append(parts, fmt.Sprintf("%s=%v", key, v)) + } + } + if len(parts) > 0 { + io.Printf(" %s\n", io.Gray(strings.Join(parts, " "))) + } +} + +func eventColour(io *iostreams.IOStreams, event string) string { + switch { + case strings.Contains(event, "FAILED"), strings.Contains(event, "BOUNCE"), strings.Contains(event, "ERROR"): + return io.Red(event) + case strings.Contains(event, "REPLIED"), strings.Contains(event, "BOOKED"): + return io.Green(event) + case strings.Contains(event, "OPENED"), strings.Contains(event, "CLICKED"): + return io.Cyan(event) + default: + return io.Bold(event) + } +} + +// summarize is the short human tail of an event line: whichever descriptive +// field the event happens to carry. +func summarize(fields map[string]any) string { + for _, key := range []string{"subject", "email", "to", "contact_email", "campaign_name", "message", "status", "name"} { + if v, ok := fields[key]; ok { + if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "" { + if len(s) > 70 { + s = s[:69] + "…" + } + return s + } + } + } + return "" +} + +// intentSuffix names the filter in the "listening" line, so a stream that goes +// quiet does not look broken when it is only filtered. +func intentSuffix(io *iostreams.IOStreams, intents []string) string { + if len(intents) == 0 { + return "" + } + return io.Gray(" (" + strings.Join(intents, ", ") + " only)") +} + +// resolveSocketURL finds the realtime gateway: the flag, then what the +// instance advertises on /auth/config, then the layouts the installer writes. +func resolveSocketURL(ctx context.Context, f *Factory, r *config.Resolved, explicit string) (string, error) { + if explicit != "" { + return explicit, nil + } + client := api.New(r.APIURL, "", UserAgent()) + if f.Debug { + client.Debug = f.IO.ErrOut + } + var cfg struct { + WebsocketURL string `json:"websocket_url"` + } + if err := client.JSON(ctx, api.Request{Method: http.MethodGet, Path: "/auth/config", Anonymous: true}, &cfg); err == nil && cfg.WebsocketURL != "" { + return cfg.WebsocketURL, nil + } + + host := config.NormalizeHost(r.Host) + if host == config.DefaultHost { + return "wss://realtime." + config.DefaultHost + "/socket/websocket", nil + } + if strings.HasPrefix(host, "localhost") || strings.HasPrefix(host, "127.0.0.1") { + return "ws://" + strings.Split(host, ":")[0] + ":4000/socket/websocket", nil + } + // The installer's proxy and Caddy layouts both put it on ws.. + return "wss://ws." + host + "/socket/websocket", nil +} diff --git a/cmd/cli/factory.go b/cmd/cli/factory.go new file mode 100644 index 00000000..e69ba645 --- /dev/null +++ b/cmd/cli/factory.go @@ -0,0 +1,168 @@ +package main + +import ( + "fmt" + "io" + "os" + "runtime" + "strings" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/cli/iostreams" + "github.com/warmbly/warmbly/internal/cli/output" + "github.com/warmbly/warmbly/internal/version" +) + +// Factory is what every command is handed: the terminal, the two config files, +// and a way to build an authenticated client. Building it lazily matters, +// because `warmbly auth login` and `warmbly version` must work before there is +// anything to authenticate with. +type Factory struct { + IO *iostreams.IOStreams + + // Global flags, bound once on the root command. + HostFlag string + JSONOut bool + Template string + Fields []string + AssumeYes bool + NoColor bool + Debug bool + + cfg *config.Config + hosts config.Hosts +} + +func NewFactory() *Factory { + return &Factory{IO: iostreams.System()} +} + +// UserAgent identifies the CLI in API usage logs, which is how an operator +// tells a CLI call from a script's. +func UserAgent() string { + return fmt.Sprintf("warmbly-cli/%s (%s/%s)", version.String(), runtime.GOOS, runtime.GOARCH) +} + +func (f *Factory) Config() (*config.Config, error) { + if f.cfg != nil { + return f.cfg, nil + } + cfg, err := config.Load() + if err != nil { + return nil, err + } + f.cfg = cfg + return cfg, nil +} + +func (f *Factory) Hosts() (config.Hosts, error) { + if f.hosts != nil { + return f.hosts, nil + } + hosts, err := config.LoadHosts() + if err != nil { + return nil, err + } + f.hosts = hosts + return hosts, nil +} + +// Resolved answers which host and token this invocation uses. +func (f *Factory) Resolved() (*config.Resolved, error) { + cfg, err := f.Config() + if err != nil { + return nil, err + } + hosts, err := f.Hosts() + if err != nil { + return nil, err + } + return config.Resolve(cfg, hosts, f.HostFlag) +} + +// Client builds an authenticated client, or explains how to get one. +func (f *Factory) Client() (*api.Client, error) { + r, err := f.Resolved() + if err != nil { + return nil, err + } + c := api.New(r.APIURL, r.Token, UserAgent()) + if f.Debug { + c.Debug = f.IO.ErrOut + } + return c, nil +} + +// Printer is the renderer for this invocation, honouring the config default +// and then the flags. +func (f *Factory) Printer() *output.Printer { + jsonOut := f.JSONOut + if !jsonOut { + if cfg, err := f.Config(); err == nil && cfg.Get("output") == "json" { + jsonOut = true + } + } + return &output.Printer{IO: f.IO, JSON: jsonOut, Template: f.Template, Fields: f.Fields} +} + +// ConfirmSend is the gate in front of anything that puts real mail on the +// wire. Without a terminal it refuses rather than sending, because a script +// that forgot --yes must not discover the omission by mailing strangers. +func (f *Factory) ConfirmSend(what string) error { + if f.AssumeYes { + return nil + } + if !f.IO.IsStdinTTY() { + return fmt.Errorf("%s sends real mail, and there is no terminal to confirm on.\nPass --yes to go ahead.", what) + } + ok, err := f.IO.Confirm(f.IO.Yellow("! ")+what+" sends real mail. Continue?", false) + if err != nil { + return err + } + if !ok { + return errCancelled + } + return nil +} + +// ConfirmMutation guards a destructive but non-sending change. It only asks +// when the user opted in with `warmbly config set confirm always`, because +// prompting on every write makes a CLI tiring to use. +func (f *Factory) ConfirmMutation(what string) error { + if f.AssumeYes { + return nil + } + cfg, err := f.Config() + if err != nil || cfg.Get("confirm") != "always" { + return nil + } + if !f.IO.IsStdinTTY() { + return nil + } + ok, cerr := f.IO.Confirm(what+"?", false) + if cerr != nil { + return cerr + } + if !ok { + return errCancelled + } + return nil +} + +// bodyFromArg turns a --input value into a request body: a JSON literal, `-` +// for stdin, or @path for a file, the conventions curl taught everyone. +func bodyFromArg(in io.Reader, data string) ([]byte, error) { + data = strings.TrimSpace(data) + if data == "" { + return nil, nil + } + switch { + case data == "-": + return io.ReadAll(in) + case strings.HasPrefix(data, "@"): + return os.ReadFile(strings.TrimPrefix(data, "@")) + default: + return []byte(data), nil + } +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go new file mode 100644 index 00000000..d1539c83 --- /dev/null +++ b/cmd/cli/main.go @@ -0,0 +1,101 @@ +// warmbly is the command line interface to Warmbly. +// +// It is the customer's CLI, not the operator's: it signs in as a person, holds +// one credential per host in ~/.config/warmbly, and speaks only the public +// REST API, so it drives the hosted service and any self-hosted instance the +// caller can reach. Everything it can do is bounded by the scopes the sign-in +// approved, and it never serves HTTP. +// +// The other CLI, warmblyctl, is the operator's: it talks to Postgres directly, +// runs inside the backend container, and exists for recovery and accounts. If +// you are asking "what is wrong with this install", that is the one you want. +// +// warmbly auth login +// warmbly campaign list +// warmbly api "/campaigns?limit=10" +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/cli/iostreams" +) + +// errCancelled is a user saying no at a prompt. It exits 1 with no error line, +// because the person already knows what happened. +var errCancelled = errors.New("cancelled") + +// errSilent lets a command print its own failure and still exit non-zero. +var errSilent = errors.New("") + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + f := NewFactory() + root := newRootCmd(f) + root.SetArgs(expandAliases(f, os.Args[1:])) + + err := root.ExecuteContext(ctx) + + // After the command, never before: the reminder is not worth delaying a + // result for, and it must not appear instead of an error. + if err == nil { + nudgeAboutUpdates(ctx, f) + } + + if err != nil { + os.Exit(reportError(f.IO, err)) + } +} + +// reportError turns whatever came back into one line a person can act on, and +// the exit code a script can branch on: +// +// 1 the command failed +// 2 usage was wrong +// 4 not signed in, or the credential was rejected +func reportError(io *iostreams.IOStreams, err error) int { + if errors.Is(err, errCancelled) { + fmt.Fprintln(io.ErrOut, io.Gray("Cancelled.")) + return 1 + } + if errors.Is(err, errSilent) { + return 1 + } + + var noToken *config.ErrNoToken + if errors.As(err, &noToken) { + fmt.Fprintf(io.ErrOut, "%s %s\n", io.Cross(), noToken.Error()) + return 4 + } + + var apiErr *api.Error + if errors.As(err, &apiErr) { + fmt.Fprintf(io.ErrOut, "%s %s\n", io.Cross(), apiErr.Error()) + switch { + case apiErr.IsUnauthorized(): + fmt.Fprintln(io.ErrOut, io.Gray("The credential was rejected. Run `warmbly auth status` to see which one was used, or `warmbly auth login` to replace it.")) + return 4 + case apiErr.Status == 403: + fmt.Fprintln(io.ErrOut, io.Gray("The key is missing a scope for this call. `warmbly auth refresh` re-runs the sign-in and can ask for more.")) + return 4 + } + return 1 + } + + var noTTY *iostreams.ErrNoTTY + if errors.As(err, &noTTY) { + fmt.Fprintf(io.ErrOut, "%s %s\n", io.Cross(), noTTY.Error()) + return 2 + } + + fmt.Fprintf(io.ErrOut, "%s %s\n", io.Cross(), err.Error()) + return 1 +} diff --git a/cmd/cli/resources.go b/cmd/cli/resources.go new file mode 100644 index 00000000..f43f9c6b --- /dev/null +++ b/cmd/cli/resources.go @@ -0,0 +1,338 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/output" +) + +// The typed commands are a table. Dispatch, flags, help, the request and the +// rendering all come from one row, so covering a new endpoint is adding a row +// rather than writing a command. `warmbly api` covers anything the table does +// not, which is what keeps the table from having to be exhaustive to be useful. + +type bodyMode int + +const ( + bodyNone bodyMode = iota // the endpoint takes no body + bodyOptional // a body may be sent + bodyRequired // a body must be sent +) + +type flagKind int + +const ( + flagString flagKind = iota + flagInt + flagBool + flagStrings +) + +// argSpec is one positional argument, filling the next {} in the path. +type argSpec struct { + Name string + Help string +} + +// flagSpec is one flag. Query flags become query parameters; the rest become +// body fields, so `campaign create --name X` needs no JSON. +type flagSpec struct { + Name string + Short string + Help string + Kind flagKind + Query bool + // Key overrides the body field or query parameter name, which otherwise + // is the flag name with dashes turned into underscores. + Key string +} + +func (f flagSpec) key() string { + if f.Key != "" { + return f.Key + } + return strings.ReplaceAll(f.Name, "-", "_") +} + +type endpoint struct { + Name string + Aliases []string + Short string + Long string + Example string + + Method string + // Path is /v1-relative and carries one {name} per positional argument. + Path string + Args []argSpec + Flag []flagSpec + Body bodyMode + + // Sends marks a command that puts real mail on the wire. Those confirm. + Sends bool + // Paginate offers --all, which walks the cursor. + Paginate bool + // Idempotent offers --idempotency-key. + Idempotent bool + + Table output.Table + // Success is what to say on a terminal when there is nothing to tabulate. + Success string +} + +type resource struct { + Name string + Aliases []string + Short string + Long string + Group string + Endpoints []endpoint +} + +func resourceCommands(f *Factory) []*cobra.Command { + specs := resourceSpecs() + out := make([]*cobra.Command, 0, len(specs)) + for _, r := range specs { + out = append(out, buildResource(f, r)) + } + return out +} + +func buildResource(f *Factory, r resource) *cobra.Command { + cmd := &cobra.Command{ + Use: r.Name + " ", + Aliases: r.Aliases, + Short: r.Short, + Long: r.Long, + GroupID: r.Group, + } + for _, e := range r.Endpoints { + cmd.AddCommand(buildEndpoint(f, r, e)) + } + return cmd +} + +func buildEndpoint(f *Factory, r resource, e endpoint) *cobra.Command { + use := e.Name + for _, a := range e.Args { + use += " <" + a.Name + ">" + } + + long := e.Long + if long == "" { + long = e.Short + "." + } + if len(e.Args) > 0 { + var lines []string + for _, a := range e.Args { + lines = append(lines, fmt.Sprintf(" <%s> %s", a.Name, a.Help)) + } + long += "\n\nArguments:\n" + strings.Join(lines, "\n") + } + if e.Sends { + long += "\n\nThis command sends real mail. It asks before doing so; --yes skips the question." + } + + cmd := &cobra.Command{ + Use: use, + Aliases: e.Aliases, + Short: e.Short, + Long: long, + Example: e.Example, + Args: cobra.ExactArgs(len(e.Args)), + } + + // Flag values are held here so the runner reads whatever cobra parsed. + strs := map[string]*string{} + ints := map[string]*int{} + bools := map[string]*bool{} + slices := map[string]*[]string{} + for _, fl := range e.Flag { + switch fl.Kind { + case flagInt: + ints[fl.Name] = cmd.Flags().IntP(fl.Name, fl.Short, 0, fl.Help) + case flagBool: + bools[fl.Name] = cmd.Flags().BoolP(fl.Name, fl.Short, false, fl.Help) + case flagStrings: + slices[fl.Name] = cmd.Flags().StringSliceP(fl.Name, fl.Short, nil, fl.Help) + default: + strs[fl.Name] = cmd.Flags().StringP(fl.Name, fl.Short, "", fl.Help) + } + } + + var ( + input string + rawField []string + typField []string + all bool + maxPages int + idemKey string + ) + if e.Body != bodyNone { + cmd.Flags().StringVar(&input, "input", "", "Request body: JSON, @file, or - for stdin") + cmd.Flags().StringArrayVarP(&rawField, "raw-field", "f", nil, "Body field as a string: key=value") + cmd.Flags().StringArrayVarP(&typField, "field", "F", nil, "Body field with a guessed type: key=value") + } + if e.Paginate { + cmd.Flags().BoolVar(&all, "all", false, "Fetch every page, not just the first") + cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Stop after this many pages when --all is set") + } + if e.Idempotent { + cmd.Flags().StringVar(&idemKey, "idempotency-key", "", "Idempotency-Key header for a safely retryable write") + } + + cmd.RunE = func(c *cobra.Command, args []string) error { + path, err := fillPath(e.Path, e.Args, args) + if err != nil { + return err + } + + query := url.Values{} + body := map[string]any{} + for _, fl := range e.Flag { + if !c.Flags().Changed(fl.Name) { + continue + } + var value any + switch fl.Kind { + case flagInt: + value = *ints[fl.Name] + case flagBool: + value = *bools[fl.Name] + case flagStrings: + value = *slices[fl.Name] + default: + value = *strs[fl.Name] + } + if fl.Query { + query.Set(fl.key(), queryString(value)) + continue + } + if err := assign(body, fl.key(), value); err != nil { + return err + } + } + + raw, err := bodyFromArg(f.IO.In, input) + if err != nil { + return err + } + fields, err := buildFields(rawField, typField, f.IO.In) + if err != nil { + return err + } + for k, v := range fields { + body[k] = v + } + + var payload []byte + switch { + case raw != nil && len(body) > 0: + // Merging a literal body with flags would silently pick a winner. + return fmt.Errorf("pass --input or the field flags, not both") + case raw != nil: + if !json.Valid(raw) { + return fmt.Errorf("the request body is not valid JSON") + } + payload = raw + case len(body) > 0: + payload, err = json.Marshal(body) + if err != nil { + return err + } + case e.Body == bodyRequired: + return fmt.Errorf("%s %s needs a body.\nSupply one with the flags above, with -f key=value, or with --input @file.json", r.Name, e.Name) + case e.Body == bodyOptional && e.Method != http.MethodGet: + payload = []byte("{}") + } + + if e.Sends { + if err := f.ConfirmSend(fmt.Sprintf("`warmbly %s %s`", r.Name, e.Name)); err != nil { + return err + } + } else if e.Method == http.MethodDelete { + if err := f.ConfirmMutation(fmt.Sprintf("Run `warmbly %s %s`", r.Name, e.Name)); err != nil { + return err + } + } + + client, err := f.Client() + if err != nil { + return err + } + req := api.Request{ + Method: e.Method, + Path: path, + Query: query, + Body: payload, + IdempotencyKey: idemKey, + } + + printer := f.Printer() + if all { + merged, perr := client.Paginate(c.Context(), req, maxPages) + if perr != nil { + return perr + } + return printer.Print(merged, e.Table) + } + + resp, err := client.Do(c.Context(), req) + if err != nil { + return err + } + // Nothing to tabulate and a terminal to talk to: say what happened + // rather than printing an empty object. + if len(e.Table.Columns) == 0 && e.Success != "" && !printer.JSON && printer.Template == "" && f.IO.IsStdoutTTY() { + f.IO.Printf("%s %s\n", f.IO.Tick(), e.Success) + return nil + } + return printer.Print(resp.Body, e.Table) + } + + return cmd +} + +// fillPath substitutes positional arguments into the {} markers, in order. +func fillPath(path string, specs []argSpec, args []string) (string, error) { + for i, spec := range specs { + marker := "{" + spec.Name + "}" + if !strings.Contains(path, marker) { + return "", fmt.Errorf("internal: path %q has no %s", path, marker) + } + value := strings.TrimSpace(args[i]) + if value == "" { + return "", fmt.Errorf("<%s> cannot be empty", spec.Name) + } + path = strings.ReplaceAll(path, marker, url.PathEscape(value)) + } + if strings.Contains(path, "{") { + return "", fmt.Errorf("internal: path %q still has an unfilled placeholder", path) + } + return path, nil +} + +func queryString(v any) string { + switch t := v.(type) { + case string: + return t + case bool: + if t { + return "true" + } + return "false" + case int: + return fmt.Sprint(t) + case []string: + return strings.Join(t, ",") + default: + return fmt.Sprint(t) + } +} diff --git a/cmd/cli/root.go b/cmd/cli/root.go new file mode 100644 index 00000000..1a4b36c6 --- /dev/null +++ b/cmd/cli/root.go @@ -0,0 +1,147 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" +) + +// Command groups, so `warmbly --help` reads as a product rather than an +// alphabetical dump of forty nouns. +const ( + groupCore = "core" + groupWork = "work" + groupData = "data" + groupDevelop = "develop" + groupSetup = "setup" +) + +func newRootCmd(f *Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "warmbly [flags]", + Short: "Warmbly from the command line", + Long: `Work with Warmbly from your terminal. + +Sign in once with ` + "`warmbly auth login`" + `, then drive campaigns, contacts, +mailboxes and the inbox as yourself. Everything the CLI can do is bounded by +the scopes you approved, on the hosted service or on your own instance.`, + Example: ` $ warmbly auth login + $ warmbly campaign list + $ warmbly mailbox list --json + $ warmbly inbox list --unseen + $ warmbly api "/campaigns?limit=10"`, + SilenceUsage: true, + SilenceErrors: true, + // A bare `warmbly` is a request for the help, not an error. + RunE: func(c *cobra.Command, args []string) error { + if len(args) == 0 { + return c.Help() + } + return fmt.Errorf("unknown command %q. Run `warmbly --help` for the full list.", args[0]) + }, + } + + p := cmd.PersistentFlags() + p.StringVar(&f.HostFlag, "host", "", "Signed-in host to use (default: the active one)") + p.BoolVar(&f.JSONOut, "json", false, "Print the API response as JSON") + p.StringVar(&f.Template, "template", "", "Format the response with a Go template") + p.StringSliceVar(&f.Fields, "fields", nil, "Table columns to keep, comma separated") + p.BoolVar(&f.AssumeYes, "yes", false, "Answer every prompt with yes, including sends") + p.BoolVar(&f.NoColor, "no-color", false, "Never colourise output") + p.BoolVar(&f.Debug, "debug", false, "Print each request to stderr") + + cmd.PersistentPreRun = func(*cobra.Command, []string) { + if f.NoColor { + f.IO.SetColor(false) + } + } + + cmd.AddGroup( + &cobra.Group{ID: groupCore, Title: "Getting started"}, + &cobra.Group{ID: groupWork, Title: "Doing the work"}, + &cobra.Group{ID: groupData, Title: "Your data"}, + &cobra.Group{ID: groupDevelop, Title: "Building on Warmbly"}, + &cobra.Group{ID: groupSetup, Title: "Setting up the CLI"}, + ) + + cmd.AddCommand(newAuthCmd(f)) + cmd.AddCommand(newStatusCmd(f)) + cmd.AddCommand(newBrowseCmd(f)) + cmd.AddCommand(newAPICmd(f)) + cmd.AddCommand(newEventsCmd(f)) + cmd.AddCommand(newConfigCmd(f)) + cmd.AddCommand(newAliasCmd(f)) + cmd.AddCommand(newVersionCmd(f)) + cmd.AddCommand(newUpgradeCmd(f)) + for _, rc := range resourceCommands(f) { + cmd.AddCommand(rc) + } + + cmd.SetOut(f.IO.Out) + cmd.SetErr(f.IO.ErrOut) + // The generated completion command has its own group so it does not sit + // under "Getting started" pretending to be a first step. + cmd.SetHelpCommandGroupID(groupSetup) + cmd.SetCompletionCommandGroupID(groupSetup) + return cmd +} + +// expandAliases rewrites the argument list when the first word is a user +// alias. Aliases are plain command lines, so `warmbly alias set hot 'campaign +// list --status active'` makes `warmbly hot --json` work. +func expandAliases(f *Factory, args []string) []string { + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + return args + } + cfg, err := f.Config() + if err != nil || len(cfg.Aliases) == 0 { + return args + } + expansion, ok := cfg.Aliases[args[0]] + if !ok { + return args + } + parts, err := splitArgs(expansion) + if err != nil || len(parts) == 0 { + return args + } + return append(parts, args[1:]...) +} + +// splitArgs is shell-ish word splitting: quotes group, nothing else is +// special. An alias is a command line, not a shell script. +func splitArgs(in string) ([]string, error) { + var out []string + var cur strings.Builder + var quote rune + started := false + for _, r := range in { + switch { + case quote != 0: + if r == quote { + quote = 0 + continue + } + cur.WriteRune(r) + case r == '\'' || r == '"': + quote = r + started = true + case r == ' ' || r == '\t': + if started || cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + started = false + } + default: + cur.WriteRune(r) + } + } + if quote != 0 { + return nil, fmt.Errorf("unbalanced quote in %q", in) + } + if started || cur.Len() > 0 { + out = append(out, cur.String()) + } + return out, nil +} diff --git a/cmd/cli/specs.go b/cmd/cli/specs.go new file mode 100644 index 00000000..662f2188 --- /dev/null +++ b/cmd/cli/specs.go @@ -0,0 +1,1925 @@ +package main + +import ( + "net/http" + + "github.com/warmbly/warmbly/internal/cli/output" +) + +// Every typed command in the CLI. One row per endpoint; the framework in +// resources.go turns each into a cobra command with its flags, help, request +// and table. Paths are /v1-relative and match internal/api/routes.go. + +// Column helpers, so a table reads as a list of columns rather than a list of +// struct literals. +func col(header, path string) output.Column { return output.Column{Header: header, Path: path} } +func colf(header, path, format string) output.Column { + return output.Column{Header: header, Path: path, Format: format} +} +func colt(header, path string, width int) output.Column { + return output.Column{Header: header, Path: path, Truncate: width} +} + +// Query flags every list endpoint shares. +var pageFlags = []flagSpec{ + {Name: "limit", Short: "L", Help: "How many to fetch (max 100)", Kind: flagInt, Query: true}, + {Name: "cursor", Help: "Opaque cursor from a previous page", Query: true}, +} + +func withPaging(extra ...flagSpec) []flagSpec { + return append(append([]flagSpec{}, pageFlags...), extra...) +} + +func resourceSpecs() []resource { + out := []resource{} + out = append(out, campaignSpec(), contactSpec(), suppressionSpec(), mailboxSpec(), inboxSpec()) + out = append(out, segmentSpec(), templateSpec(), automationSpec(), formSpec()) + out = append(out, dealSpec(), pipelineSpec(), taskSpec()) + out = append(out, analyticsSpec(), auditSpec(), advisorSpec()) + out = append(out, webhookSpec(), keySpec(), oauthAppSpec(), toolSpec()) + out = append(out, orgSpec(), teamSpec(), settingsSpec(), warmupRoutingSpec(), integrationSpec()) + return out +} + +func campaignSpec() resource { + campaignTable := output.Table{ + Root: "data", + Columns: []output.Column{ + col("ID", "id"), + colt("NAME", "name", 40), + col("STATUS", "status"), + col("DAILY", "daily_limit"), + colf("CREATED", "created_at", "time"), + }, + Empty: "No campaigns yet. Create one with `warmbly campaign create --name \"My campaign\"`.", + } + return resource{ + Name: "campaign", + Aliases: []string{"campaigns"}, + Short: "Create, run and inspect campaigns", + Group: groupWork, + Long: `Work with campaigns: the sequences, the audience, the senders, and +starting and stopping them. + +Starting a campaign and sending a test both put real mail on the wire, so both +ask before they do it.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List campaigns", + Method: http.MethodGet, Path: "/campaigns", Paginate: true, + Example: " $ warmbly campaign list\n $ warmbly campaign list --status active --limit 50\n $ warmbly campaign list --all --json", + Flag: withPaging( + flagSpec{Name: "query", Short: "q", Help: "Search campaign names", Query: true, Key: "q"}, + flagSpec{Name: "status", Help: "Filter by status: draft, active, paused, completed", Query: true}, + flagSpec{Name: "folder", Help: "Filter by folder", Query: true}, + ), + Table: campaignTable, + }, + { + Name: "view", Aliases: []string{"get", "show"}, Short: "Show one campaign", + Method: http.MethodGet, Path: "/campaigns/{id}", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Columns: []output.Column{ + col("ID", "id"), colt("NAME", "name", 40), col("STATUS", "status"), + col("DAILY", "daily_limit"), col("TIMEZONE", "timezone"), colf("CREATED", "created_at", "time"), + }}, + }, + { + Name: "overview", Short: "Status and folder counts across every campaign", + Method: http.MethodGet, Path: "/campaigns-overview", + }, + { + Name: "create", Short: "Create a campaign", + Method: http.MethodPost, Path: "/campaigns", Body: bodyRequired, Idempotent: true, + Example: " $ warmbly campaign create --name \"Q3 outbound\"\n $ warmbly campaign create --name \"Q3\" --daily-limit 40 --stop-on-reply", + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Campaign name"}, + {Name: "description", Help: "What the campaign is for"}, + {Name: "daily-limit", Help: "Emails per day across the campaign", Kind: flagInt}, + {Name: "timezone", Help: "Sending timezone, for example Europe/London"}, + {Name: "stop-on-reply", Help: "Stop a contact's sequence when they reply", Kind: flagBool}, + {Name: "open-tracking", Help: "Track opens", Kind: flagBool}, + {Name: "link-tracking", Help: "Track link clicks", Kind: flagBool}, + {Name: "text-only", Help: "Send plain text only", Kind: flagBool}, + }, + Table: output.Table{Columns: []output.Column{col("ID", "id"), col("NAME", "name"), col("STATUS", "status")}}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a campaign", + Method: http.MethodPatch, Path: "/campaigns/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Example: " $ warmbly campaign edit CAMPAIGN_ID --daily-limit 30\n $ warmbly campaign edit CAMPAIGN_ID --name \"Renamed\"", + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Campaign name"}, + {Name: "description", Help: "What the campaign is for"}, + {Name: "daily-limit", Help: "Emails per day across the campaign", Kind: flagInt}, + {Name: "timezone", Help: "Sending timezone"}, + {Name: "start-date", Help: "When sending may begin (RFC 3339)"}, + {Name: "end-date", Help: "When sending must stop (RFC 3339)"}, + {Name: "stop-on-reply", Help: "Stop a contact's sequence when they reply", Kind: flagBool}, + }, + Success: "Campaign updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a campaign", + Method: http.MethodDelete, Path: "/campaigns/{id}", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Success: "Campaign deleted.", + }, + { + Name: "duplicate", Short: "Copy a campaign, including its steps", + Method: http.MethodPost, Path: "/campaigns/{id}/duplicate", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The campaign to copy"}}, + Table: output.Table{Columns: []output.Column{col("ID", "id"), col("NAME", "name"), col("STATUS", "status")}}, + }, + { + Name: "steps", Short: "List the campaign's sequence steps", + Method: http.MethodGet, Path: "/campaigns/{id}/steps", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), colt("SUBJECT", "subject", 44), col("WAIT", "wait_after"), col("POSITION", "position"), + }, Empty: "This campaign has no steps yet."}, + }, + { + Name: "add-step", Short: "Add a sequence step", + Method: http.MethodPost, Path: "/campaigns/{id}/steps", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Example: " $ warmbly campaign add-step CAMPAIGN_ID --subject \"Quick question\" --body-html \"

Hi {{first_name}}

\"", + Flag: []flagSpec{ + {Name: "subject", Help: "Subject line"}, + {Name: "body-html", Help: "HTML body"}, + {Name: "body-plain", Help: "Plain text body"}, + {Name: "wait-after", Help: "Days to wait before this step", Kind: flagInt}, + }, + }, + { + Name: "edit-step", Short: "Change a sequence step", + Method: http.MethodPatch, Path: "/campaigns/{id}/steps/{step}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}, {Name: "step", Help: "The step's id"}}, + Flag: []flagSpec{ + {Name: "subject", Help: "Subject line"}, + {Name: "body-html", Help: "HTML body"}, + {Name: "body-plain", Help: "Plain text body"}, + {Name: "wait-after", Help: "Days to wait before this step", Kind: flagInt}, + }, + Success: "Step updated.", + }, + { + Name: "delete-step", Short: "Delete a sequence step", + Method: http.MethodDelete, Path: "/campaigns/{id}/steps/{step}", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}, {Name: "step", Help: "The step's id"}}, + Success: "Step deleted.", + }, + { + Name: "senders", Short: "Show the campaign's sender pool", + Method: http.MethodGet, Path: "/campaigns/{id}/senders", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("MAILBOX", "email"), col("WEIGHT", "weight"), col("STATUS", "status"), + }, Empty: "No explicit sender pool; the campaign uses the workspace default."}, + }, + { + Name: "set-senders", Short: "Replace the campaign's sender pool", + Method: http.MethodPut, Path: "/campaigns/{id}/senders", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Example: " $ warmbly campaign set-senders CAMPAIGN_ID --input '{\"senders\":[{\"email_account_id\":\"...\",\"weight\":1}]}'", + Success: "Sender pool replaced.", + }, + { + Name: "segments", Short: "List the segments feeding the campaign", + Method: http.MethodGet, Path: "/campaigns/{id}/segments", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("NAME", "name"), col("CONTACTS", "contact_count"), + }, Empty: "No segments are linked to this campaign."}, + }, + { + Name: "set-segments", Short: "Replace the segments feeding the campaign", + Method: http.MethodPut, Path: "/campaigns/{id}/segments", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Success: "Segments replaced.", + }, + { + Name: "attachments", Short: "List the campaign's attachments", + Method: http.MethodGet, Path: "/campaigns/{id}/attachments", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("FILE", "filename"), col("SIZE", "size_bytes"), colf("ADDED", "created_at", "time"), + }, Empty: "This campaign carries no attachments."}, + }, + { + Name: "delete-attachment", Short: "Remove an attachment", + Method: http.MethodDelete, Path: "/campaigns/{id}/attachments/{attachment}", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}, {Name: "attachment", Help: "The attachment's id"}}, + Success: "Attachment removed.", + }, + { + Name: "advanced", Short: "Show the campaign's advanced settings", + Method: http.MethodGet, Path: "/campaigns/{id}/advanced", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "set-advanced", Short: "Change the campaign's advanced settings", + Method: http.MethodPatch, Path: "/campaigns/{id}/advanced", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Success: "Advanced settings updated.", + }, + { + Name: "variants", Short: "List the campaign's A/B variants", + Method: http.MethodGet, Path: "/campaigns/{id}/ab-variants", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("NAME", "name"), colt("SUBJECT", "subject", 40), col("WEIGHT", "weight"), + }, Empty: "This campaign has no A/B variants."}, + }, + { + Name: "ab-analysis", Short: "Compare the campaign's A/B variants", + Method: http.MethodGet, Path: "/campaigns/{id}/ab-analysis", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "preflight", Short: "Run the pre-send checks without sending", + Method: http.MethodPost, Path: "/campaigns/{id}/preflight", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Long: `Run every check the campaign has to pass before it can send, and +report what would stop it. Nothing is sent.`, + }, + { + Name: "test", Aliases: []string{"test-email"}, Short: "Send the campaign as a test to an address you name", + Method: http.MethodPost, Path: "/campaigns/{id}/test-email", Body: bodyRequired, Sends: true, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Example: " $ warmbly campaign test CAMPAIGN_ID --to you@example.com", + Flag: []flagSpec{ + {Name: "to", Help: "Where to send the test"}, + {Name: "step", Help: "Which step to send", Kind: flagInt, Key: "step_id"}, + }, + Success: "Test email sent.", + }, + { + Name: "start", Short: "Start the campaign", + Method: http.MethodPost, Path: "/campaigns/{id}/start", Body: bodyOptional, Sends: true, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Success: "Campaign started.", + }, + { + Name: "stop", Aliases: []string{"pause"}, Short: "Stop the campaign", + Method: http.MethodPost, Path: "/campaigns/{id}/stop", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Success: "Campaign stopped.", + }, + { + Name: "logs", Short: "The campaign's send log", + Method: http.MethodGet, Path: "/campaigns/{id}/logs", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("EVENT", "type"), colt("CONTACT", "contact_email", 32), colt("DETAIL", "message", 50), + }, Empty: "Nothing in this campaign's log yet."}, + }, + { + Name: "forms", Short: "Form performance for this campaign's recipients", + Method: http.MethodGet, Path: "/campaigns/{id}/forms", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "verify-tracking-domain", Short: "Re-check the campaign's tracking domain", + Method: http.MethodPost, Path: "/campaigns/{id}/tracking-domain/verify", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "estimate", Short: "Estimate how long a campaign will take to send", + Method: http.MethodPost, Path: "/campaigns-estimate", Body: bodyRequired, + }, + }, + } +} + +func contactSpec() resource { + contactColumns := []output.Column{ + col("ID", "id"), + colt("EMAIL", "email", 34), + colt("NAME", "first_name", 16), + colt("COMPANY", "company", 24), + col("SUBSCRIBED", "subscribed"), + colf("ADDED", "created_at", "time"), + } + return resource{ + Name: "contact", + Aliases: []string{"contacts"}, + Short: "Add, find and update contacts", + Group: groupWork, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls", "search"}, Short: "List or search contacts", + Method: http.MethodPost, Path: "/contacts/search", Body: bodyOptional, Paginate: true, + Long: `List contacts, optionally filtered. + +The filter is a JSON body, so anything the dashboard's search can express is +available here through --input.`, + Example: " $ warmbly contact list\n $ warmbly contact list --limit 100 --all\n $ warmbly contact list --input '{\"query\":\"acme.com\"}'", + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: contactColumns, Empty: "No contacts yet. Add one with `warmbly contact create --email jane@example.com`."}, + }, + { + Name: "view", Aliases: []string{"get", "show"}, Short: "Show one contact", + Method: http.MethodGet, Path: "/contacts/{id}", + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Table: output.Table{Columns: contactColumns}, + }, + { + Name: "lookup", Short: "Find a contact by email address", + Method: http.MethodGet, Path: "/contacts/lookup", + Long: `Resolve an email address to a contact. + +A display-name form works too, so a raw From header can be passed straight in.`, + Example: " $ warmbly contact lookup --email jane@example.com", + Flag: []flagSpec{{Name: "email", Short: "e", Help: "The address to look up", Query: true}}, + Table: output.Table{Root: "contact", Columns: contactColumns, Empty: "No contact with that address."}, + }, + { + Name: "create", Aliases: []string{"add"}, Short: "Create a contact", + Method: http.MethodPost, Path: "/contacts", Body: bodyRequired, Idempotent: true, + Example: " $ warmbly contact create --email jane@example.com --first-name Jane --company Acme", + Flag: []flagSpec{ + {Name: "email", Short: "e", Help: "Email address"}, + {Name: "first-name", Help: "First name"}, + {Name: "last-name", Help: "Last name"}, + {Name: "company", Help: "Company"}, + {Name: "phone", Help: "Phone number"}, + }, + Table: output.Table{Columns: contactColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a contact", + Method: http.MethodPatch, Path: "/contacts/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Flag: []flagSpec{ + {Name: "email", Short: "e", Help: "Email address"}, + {Name: "first-name", Help: "First name"}, + {Name: "last-name", Help: "Last name"}, + {Name: "company", Help: "Company"}, + {Name: "phone", Help: "Phone number"}, + {Name: "subscribed", Help: "Whether the contact may receive campaign mail", Kind: flagBool}, + }, + Success: "Contact updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a contact", + Method: http.MethodDelete, Path: "/contacts/{id}", + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Success: "Contact deleted.", + }, + { + Name: "timeline", Short: "Everything that happened to a contact, newest first", + Method: http.MethodGet, Path: "/contacts/{id}/timeline", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("EVENT", "type"), colt("DETAIL", "description", 60), + }, Empty: "Nothing has happened to this contact yet."}, + }, + { + Name: "emails", Short: "Emails sent to a contact", + Method: http.MethodGet, Path: "/contacts/{id}/emails", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), colt("SUBJECT", "subject", 46), col("STATUS", "status"), + }, Empty: "No mail has gone to this contact."}, + }, + { + Name: "campaigns", Short: "Campaigns a contact is in", + Method: http.MethodGet, Path: "/contacts/{id}/campaigns", + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "campaign_id"), colt("CAMPAIGN", "name", 40), col("STATUS", "status"), col("SENT", "sent"), + }, Empty: "This contact is not in any campaign."}, + }, + { + Name: "notes", Short: "List a contact's notes", + Method: http.MethodGet, Path: "/contacts/{id}/notes", + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), colt("NOTE", "content", 60), colf("ADDED", "created_at", "time"), + }, Empty: "No notes on this contact."}, + }, + { + Name: "add-note", Short: "Add a note to a contact", + Method: http.MethodPost, Path: "/contacts/{id}/notes", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Flag: []flagSpec{{Name: "content", Short: "m", Help: "The note"}}, + Success: "Note added.", + }, + { + Name: "activities", Short: "A contact's recorded activities", + Method: http.MethodGet, Path: "/contacts/{id}/activities", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("TYPE", "activity_type"), + }, Empty: "No activities recorded."}, + }, + { + Name: "custom-fields", Short: "The custom field keys in use", + Method: http.MethodGet, Path: "/contacts/custom-fields", + }, + { + Name: "import-preview", Short: "Preview a bulk import without writing anything", + Method: http.MethodPost, Path: "/contacts/import/preview", Body: bodyRequired, + }, + { + Name: "import", Short: "Commit a previewed bulk import", + Method: http.MethodPost, Path: "/contacts/import/commit", Body: bodyRequired, Idempotent: true, + }, + { + Name: "export", Short: "Export contacts", + Method: http.MethodPost, Path: "/contacts/export", Body: bodyOptional, + }, + { + Name: "verify", Short: "Queue address verification for contacts", + Method: http.MethodPost, Path: "/contacts/verification", Body: bodyRequired, + }, + { + Name: "verification-status", Short: "How address verification is going", + Method: http.MethodGet, Path: "/contacts/verification", + }, + { + Name: "research", Short: "Run AI research on a contact", + Method: http.MethodPost, Path: "/contacts/{id}/research", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + }, + { + Name: "research-result", Short: "Show a contact's research", + Method: http.MethodGet, Path: "/contacts/{id}/research", + Args: []argSpec{{Name: "id", Help: "The contact's id"}}, + }, + }, + } +} + +func suppressionSpec() resource { + return resource{ + Name: "suppression", + Aliases: []string{"suppressions"}, + Short: "Addresses and domains that get no campaign mail", + Group: groupData, + Long: `The workspace suppression list. + +Anything on it is skipped by every campaign, which is what keeps an +unsubscribe or a complaint from being undone by the next import.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List suppressed addresses and domains", + Method: http.MethodGet, Path: "/suppressions", Paginate: true, + Flag: withPaging(flagSpec{Name: "query", Short: "q", Help: "Search the list", Query: true, Key: "q"}), + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("VALUE", "value"), col("KIND", "kind"), col("REASON", "reason"), colf("ADDED", "created_at", "time"), + }, Empty: "Nothing is suppressed."}, + }, + { + Name: "add", Short: "Suppress an address or a domain", + Method: http.MethodPost, Path: "/suppressions", Body: bodyRequired, + Example: " $ warmbly suppression add --input '{\"values\":[\"jane@example.com\"],\"reason\":\"manual\"}'", + }, + { + Name: "remove", Aliases: []string{"rm"}, Short: "Lift a suppression", + Method: http.MethodDelete, Path: "/suppressions/{id}", + Args: []argSpec{{Name: "id", Help: "The suppression entry's id"}}, + Success: "Suppression lifted.", + }, + }, + } +} + +func mailboxSpec() resource { + mailboxColumns := []output.Column{ + col("ID", "id"), + colt("MAILBOX", "email", 34), + col("PROVIDER", "provider"), + col("STATUS", "status"), + col("DAILY", "campaign_limit"), + colf("SYNCED", "last_synced_at", "time"), + } + return resource{ + Name: "mailbox", + Aliases: []string{"mailboxes", "email", "emails"}, + Short: "Connected sending mailboxes, their health and their warmup", + Group: groupWork, + Long: `Work with the mailboxes this workspace sends from. + +Connecting a new mailbox needs a browser (OAuth consent or a credential form), +so that stays in the dashboard: ` + "`warmbly browse mailboxes`" + ` opens it. +Everything after connection is here.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List connected mailboxes", + Method: http.MethodGet, Path: "/emails", Paginate: true, + Flag: withPaging(flagSpec{Name: "query", Short: "q", Help: "Search addresses", Query: true, Key: "q"}), + Table: output.Table{Root: "data", Columns: mailboxColumns, Empty: "No mailboxes connected. Connect one with `warmbly browse mailboxes`."}, + }, + { + Name: "view", Aliases: []string{"get", "show"}, Short: "Show one mailbox", + Method: http.MethodGet, Path: "/emails/{id}", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Table: output.Table{Columns: mailboxColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a mailbox's settings", + Method: http.MethodPatch, Path: "/emails/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Long: `Change a mailbox's sending settings. + +The daily cap is the safety control that matters most: 50 a day is the product +default and the top of the normal band for cold outreach. Above 100 the +dashboard warns, and it should.`, + Example: " $ warmbly mailbox edit MAILBOX_ID --daily-limit 40\n $ warmbly mailbox edit MAILBOX_ID --min-wait 900", + Flag: []flagSpec{ + {Name: "name", Help: "Display name on outgoing mail"}, + {Name: "daily-limit", Help: "Campaign emails per day from this mailbox", Kind: flagInt, Key: "campaign_limit"}, + {Name: "min-wait", Help: "Minimum seconds between sends", Kind: flagInt, Key: "min_wait_time"}, + {Name: "reply-to", Help: "Reply-To address"}, + {Name: "signature", Help: "Plain text signature", Key: "signature_plain"}, + {Name: "timezone", Help: "The mailbox's timezone"}, + }, + Success: "Mailbox updated.", + }, + { + Name: "remove", Aliases: []string{"rm", "delete"}, Short: "Disconnect a mailbox", + Method: http.MethodDelete, Path: "/emails/{id}", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Mailbox disconnected.", + }, + { + Name: "check", Aliases: []string{"auth-check"}, Short: "Show the mailbox's SPF, DKIM and DMARC", + Method: http.MethodGet, Path: "/emails/{id}/auth-check", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Table: output.Table{Columns: []output.Column{ + col("STATE", "auth_state"), col("SPF", "auth_spf"), col("DKIM", "auth_dkim"), + col("DMARC", "auth_dmarc"), col("POLICY", "auth_dmarc_policy"), colf("CHECKED", "auth_checked_at", "time"), + }}, + }, + { + Name: "recheck", Short: "Re-run the authentication check now", + Method: http.MethodPost, Path: "/emails/{id}/auth-check", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "sync", Short: "The mailbox's sync state and backfill progress", + Method: http.MethodGet, Path: "/emails/{id}/sync", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "behavior", Short: "The mailbox's human-sending ranges", + Method: http.MethodGet, Path: "/emails/{id}/behavior", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "set-behavior", Short: "Change the mailbox's sending behaviour", + Method: http.MethodPut, Path: "/emails/{id}/behavior", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Sending behaviour updated.", + }, + { + Name: "behavior-plan", Short: "What the behaviour settings mean in practice", + Method: http.MethodGet, Path: "/emails/{id}/behavior/plan", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "verify", Short: "Verify an email address without sending to it", + Method: http.MethodPost, Path: "/emails/verify", Body: bodyRequired, + Flag: []flagSpec{{Name: "email", Short: "e", Help: "The address to verify"}}, + }, + { + Name: "send", Short: "Send one email from this mailbox", + Method: http.MethodPost, Path: "/emails/{id}/send", Body: bodyRequired, Sends: true, Idempotent: true, + Args: []argSpec{{Name: "id", Help: "The mailbox to send from"}}, + Example: " $ warmbly mailbox send MAILBOX_ID --to jane@example.com --subject Hello --body \"Hi Jane\"", + Flag: []flagSpec{ + {Name: "to", Help: "Recipient address"}, + {Name: "subject", Help: "Subject line"}, + {Name: "body", Help: "Message body", Key: "body_html"}, + {Name: "cc", Help: "CC addresses", Kind: flagStrings}, + {Name: "bcc", Help: "BCC addresses", Kind: flagStrings}, + }, + Success: "Email sent.", + }, + { + Name: "hold", Short: "Hold the mailbox out of campaign sending", + Method: http.MethodPost, Path: "/emails/{id}/hold", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Mailbox held out of campaign sending.", + }, + { + Name: "release", Short: "Put a held mailbox back into campaign sending", + Method: http.MethodPost, Path: "/emails/{id}/release", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Mailbox released back into campaign sending.", + }, + { + Name: "warmup-start", Short: "Start warming the mailbox", + Method: http.MethodPost, Path: "/emails/{id}/warmup/start", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Long: `Start warmup for this mailbox. + +Warmup ramps gradually from the product default of ten a day; it is not a +switch that produces volume today, and it should keep running once campaigns +begin rather than being stopped the moment the mailbox looks ready.`, + Success: "Warmup started.", + }, + { + Name: "warmup-pause", Short: "Pause warmup", + Method: http.MethodPost, Path: "/emails/{id}/warmup/pause", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Warmup paused.", + }, + { + Name: "warmup-resume", Short: "Resume warmup", + Method: http.MethodPost, Path: "/emails/{id}/warmup/resume", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Warmup resumed.", + }, + { + Name: "warmup-stop", Short: "Stop warmup", + Method: http.MethodPost, Path: "/emails/{id}/warmup/stop", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Success: "Warmup stopped.", + }, + { + Name: "warmup-status", Short: "The mailbox's warmup pool standing", + Method: http.MethodGet, Path: "/emails/{id}/warmup/ban-status", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "warmup-appeal", Short: "Appeal a warmup pool block", + Method: http.MethodPost, Path: "/emails/{id}/warmup/appeal", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "tracking", Short: "The mailbox's tracking domain", + Method: http.MethodGet, Path: "/emails/{id}/track", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "set-tracking", Short: "Set the mailbox's tracking domain", + Method: http.MethodPatch, Path: "/emails/{id}/track", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + Flag: []flagSpec{{Name: "domain", Help: "The tracking domain", Key: "tracking_domain"}}, + Success: "Tracking domain set.", + }, + { + Name: "verify-tracking", Short: "Check the tracking domain's DNS", + Method: http.MethodPost, Path: "/emails/{id}/track/verify", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "tags", Short: "Change tags across mailboxes", + Method: http.MethodPatch, Path: "/emails/tags", Body: bodyRequired, + Success: "Tags updated.", + }, + }, + } +} + +func inboxSpec() resource { + messageColumns := []output.Column{ + col("ID", "id"), + colt("FROM", "from_addr", 28), + colt("SUBJECT", "subject", 44), + col("SEEN", "seen"), + colf("WHEN", "internal_date", "time"), + } + return resource{ + Name: "inbox", + Aliases: []string{"unibox"}, + Short: "Read and reply to mail across every mailbox", + Group: groupWork, + Long: `The unified inbox: every connected mailbox in one stream. + +Replying and composing put real mail on the wire, so both ask before they do.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List inbox messages", + Method: http.MethodGet, Path: "/unibox", Paginate: true, + Example: " $ warmbly inbox list\n $ warmbly inbox list --unseen\n $ warmbly inbox list --from acme.com --limit 50", + Flag: withPaging( + flagSpec{Name: "address", Help: "Conversations with this address, either direction", Query: true}, + flagSpec{Name: "direction", Help: "sent or received", Query: true}, + flagSpec{Name: "folder", Help: "inbox, sent, drafts, archive, spam or trash", Query: true}, + flagSpec{Name: "from", Help: "Filter by sender", Query: true}, + flagSpec{Name: "subject", Help: "Filter by subject", Query: true}, + flagSpec{Name: "unseen", Help: "Only unread messages", Kind: flagBool, Query: true}, + flagSpec{Name: "awaiting-reply", Help: "Only threads waiting on a reply", Kind: flagBool, Query: true, Key: "awaiting_reply"}, + flagSpec{Name: "since", Help: "Only messages after this time (RFC 3339)", Query: true}, + flagSpec{Name: "until", Help: "Only messages before this time (RFC 3339)", Query: true}, + ), + Table: output.Table{Root: "data", Columns: messageColumns, Empty: "Nothing in the inbox."}, + }, + { + Name: "view", Aliases: []string{"get", "show"}, Short: "Show one message", + Method: http.MethodGet, Path: "/unibox/{id}", + Args: []argSpec{{Name: "id", Help: "The message's id"}}, + }, + { + Name: "count", Short: "The unread count", + Method: http.MethodGet, Path: "/unibox/count", + }, + { + Name: "overview", Short: "Per-mailbox and per-tag inbox rollup", + Method: http.MethodGet, Path: "/unibox/overview", + }, + { + Name: "thread", Short: "One conversation, oldest first", + Method: http.MethodGet, Path: "/unibox/thread", Paginate: true, + Long: `Read one conversation. + +--thread-id is required and is the thread_id on any message from ` + "`warmbly inbox list`" + `. +Without --mailbox the thread is read across every mailbox in the workspace, +which is the unified view.`, + Example: " $ warmbly inbox thread --thread-id THREAD_ID", + Flag: withPaging( + flagSpec{Name: "thread-id", Help: "The thread's id (required)", Query: true, Key: "thread_id"}, + flagSpec{Name: "mailbox", Help: "Narrow to one mailbox", Query: true, Key: "email_id"}, + ), + Table: output.Table{Root: "data", Columns: messageColumns, Empty: "No messages in that thread."}, + }, + { + Name: "read", Short: "Mark messages read", + Method: http.MethodPatch, Path: "/unibox/seen", Body: bodyRequired, + Example: " $ warmbly inbox read --input '{\"ids\":[\"...\"],\"seen\":true}'", + Success: "Marked.", + }, + { + Name: "reply", Short: "Reply in a thread", + Method: http.MethodPost, Path: "/unibox/reply", Body: bodyRequired, Sends: true, Idempotent: true, + Example: " $ warmbly inbox reply --input '{\"email_id\":\"...\",\"body_html\":\"

Thanks

\"}'", + Success: "Reply sent.", + }, + { + Name: "compose", Short: "Send a new email", + Method: http.MethodPost, Path: "/unibox/compose", Body: bodyRequired, Sends: true, Idempotent: true, + Flag: []flagSpec{ + {Name: "from", Help: "The mailbox to send from", Key: "email_account_id"}, + {Name: "to", Help: "Recipient addresses", Kind: flagStrings}, + {Name: "subject", Help: "Subject line"}, + {Name: "body", Help: "Message body", Key: "body_html"}, + }, + Success: "Email sent.", + }, + { + Name: "drafts", Short: "List your saved drafts", + Method: http.MethodGet, Path: "/unibox/drafts", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), colt("SUBJECT", "subject", 44), colf("SAVED", "updated_at", "time"), + }, Empty: "No drafts saved."}, + }, + { + Name: "delete-draft", Short: "Delete a saved draft", + Method: http.MethodDelete, Path: "/unibox/drafts/{id}", + Args: []argSpec{{Name: "id", Help: "The draft's id"}}, + Success: "Draft deleted.", + }, + { + Name: "agent-drafts", Short: "Replies the inbox agent drafted for review", + Method: http.MethodGet, Path: "/unibox/agent-drafts", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), colt("SUBJECT", "subject", 40), colt("TO", "to_addr", 28), colf("DRAFTED", "created_at", "time"), + }, Empty: "The inbox agent has nothing waiting."}, + }, + { + Name: "approve-draft", Short: "Approve an agent draft, which sends it", + Method: http.MethodPost, Path: "/unibox/agent-drafts/{id}/approve", Body: bodyOptional, Sends: true, + Args: []argSpec{{Name: "id", Help: "The draft's id"}}, + Success: "Draft approved and sent.", + }, + { + Name: "discard-draft", Short: "Discard an agent draft", + Method: http.MethodPost, Path: "/unibox/agent-drafts/{id}/discard", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The draft's id"}}, + Success: "Draft discarded.", + }, + { + Name: "scheduled", Short: "List scheduled sends", + Method: http.MethodGet, Path: "/unibox/scheduled", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("TASK", "task_id"), colt("SUBJECT", "subject", 40), colf("SENDS", "scheduled_for", "date"), + }, Empty: "Nothing is scheduled."}, + }, + { + Name: "cancel-scheduled", Short: "Cancel a scheduled send", + Method: http.MethodDelete, Path: "/unibox/scheduled/{task}", + Args: []argSpec{{Name: "task", Help: "The scheduled task's id"}}, + Success: "Scheduled send cancelled.", + }, + { + Name: "snooze", Short: "Snooze a thread until later", + Method: http.MethodPost, Path: "/unibox/snooze", Body: bodyRequired, + Success: "Thread snoozed.", + }, + { + Name: "unsnooze", Short: "Bring a snoozed thread back now", + Method: http.MethodDelete, Path: "/unibox/snooze", Body: bodyRequired, + Success: "Thread unsnoozed.", + }, + { + Name: "snoozes", Short: "List snoozed threads", + Method: http.MethodGet, Path: "/unibox/snoozes", + }, + { + Name: "labels", Short: "A thread's labels", + Method: http.MethodGet, Path: "/unibox/thread/labels", + Flag: []flagSpec{{Name: "thread-id", Help: "The thread to read labels for (required)", Query: true, Key: "thread_id"}}, + }, + { + Name: "set-labels", Short: "Replace a thread's labels", + Method: http.MethodPut, Path: "/unibox/thread/labels", Body: bodyRequired, + Success: "Labels updated.", + }, + }, + } +} + +func segmentSpec() resource { + segmentColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 34), col("MATCH", "match"), + col("CONTACTS", "contact_count"), colf("CREATED", "created_at", "time"), + } + return resource{ + Name: "segment", + Aliases: []string{"segments"}, + Short: "Live contact audiences built from conditions", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List segments", + Method: http.MethodGet, Path: "/segments", + Table: output.Table{Root: "data", Columns: segmentColumns, Empty: "No segments yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one segment", + Method: http.MethodGet, Path: "/segments/{id}", + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + Table: output.Table{Columns: segmentColumns}, + }, + { + Name: "create", Short: "Create a segment", + Method: http.MethodPost, Path: "/segments", Body: bodyRequired, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Segment name"}, + {Name: "description", Help: "What the segment is for"}, + {Name: "match", Help: "all or any"}, + }, + Table: output.Table{Columns: segmentColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a segment", + Method: http.MethodPatch, Path: "/segments/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Segment name"}, + {Name: "description", Help: "What the segment is for"}, + {Name: "match", Help: "all or any"}, + }, + Success: "Segment updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a segment", + Method: http.MethodDelete, Path: "/segments/{id}", + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + Success: "Segment deleted.", + }, + { + Name: "fields", Short: "Every field a condition can match on", + Method: http.MethodGet, Path: "/segments/fields", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("FIELD", "field"), col("LABEL", "label"), col("GROUP", "group"), col("KIND", "kind"), + }, Empty: "No segment fields reported."}, + }, + { + Name: "preview", Short: "Count what a set of conditions would match", + Method: http.MethodPost, Path: "/segments/preview", Body: bodyRequired, + }, + { + Name: "members", Short: "Add or remove explicit members", + Method: http.MethodPost, Path: "/segments/{id}/members", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + Success: "Membership updated.", + }, + { + Name: "overrides", Short: "Contacts forced in or out of the segment", + Method: http.MethodGet, Path: "/segments/{id}/overrides", + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + }, + { + Name: "add-to-campaign", Short: "Enrol the segment's contacts in a campaign", + Method: http.MethodPost, Path: "/segments/{id}/add-to-campaign", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The segment's id"}}, + Flag: []flagSpec{{Name: "campaign", Help: "The campaign to add them to", Key: "campaign_id"}}, + }, + }, + } +} + +func templateSpec() resource { + templateColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 30), colt("SUBJECT", "subject", 40), colf("UPDATED", "updated_at", "time"), + } + return resource{ + Name: "template", + Aliases: []string{"templates"}, + Short: "Reply templates", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List templates", + Method: http.MethodGet, Path: "/templates", + Table: output.Table{Root: "data", Columns: templateColumns, Empty: "No templates yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one template", + Method: http.MethodGet, Path: "/templates/{id}", + Args: []argSpec{{Name: "id", Help: "The template's id"}}, + Table: output.Table{Columns: templateColumns}, + }, + { + Name: "create", Short: "Create a template", + Method: http.MethodPost, Path: "/templates", Body: bodyRequired, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Template name"}, + {Name: "subject", Help: "Subject line"}, + {Name: "body-html", Help: "HTML body"}, + {Name: "body-plain", Help: "Plain text body"}, + }, + Table: output.Table{Columns: templateColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a template", + Method: http.MethodPatch, Path: "/templates/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The template's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Template name"}, + {Name: "subject", Help: "Subject line"}, + {Name: "body-html", Help: "HTML body"}, + {Name: "body-plain", Help: "Plain text body"}, + }, + Success: "Template updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a template", + Method: http.MethodDelete, Path: "/templates/{id}", + Args: []argSpec{{Name: "id", Help: "The template's id"}}, + Success: "Template deleted.", + }, + { + Name: "duplicate", Short: "Copy a template", + Method: http.MethodPost, Path: "/templates/{id}/duplicate", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The template's id"}}, + Table: output.Table{Columns: templateColumns}, + }, + { + Name: "render", Short: "Render a template with variables filled in", + Method: http.MethodPost, Path: "/templates/{id}/render", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The template's id"}}, + }, + { + Name: "score", Short: "Score a draft for deliverability and tone", + Method: http.MethodPost, Path: "/templates/score", Body: bodyRequired, + }, + { + Name: "reorder", Short: "Change the order templates appear in", + Method: http.MethodPatch, Path: "/templates/reorder", Body: bodyRequired, + Success: "Templates reordered.", + }, + }, + } +} + +func automationSpec() resource { + automationColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 34), col("STATUS", "status"), colf("UPDATED", "updated_at", "time"), + } + return resource{ + Name: "automation", + Aliases: []string{"automations"}, + Short: "Event-driven automations", + Group: groupWork, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List automations", + Method: http.MethodGet, Path: "/automations", + Table: output.Table{Root: "data", Columns: automationColumns, Empty: "No automations yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one automation", + Method: http.MethodGet, Path: "/automations/{id}", + Args: []argSpec{{Name: "id", Help: "The automation's id"}}, + Table: output.Table{Columns: automationColumns}, + }, + { + Name: "create", Short: "Create an automation", + Method: http.MethodPost, Path: "/automations", Body: bodyRequired, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Automation name"}}, + Table: output.Table{Columns: automationColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change an automation", + Method: http.MethodPatch, Path: "/automations/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The automation's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Automation name"}, + {Name: "status", Help: "active or paused"}, + }, + Success: "Automation updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete an automation", + Method: http.MethodDelete, Path: "/automations/{id}", + Args: []argSpec{{Name: "id", Help: "The automation's id"}}, + Success: "Automation deleted.", + }, + { + Name: "test", Short: "Run an automation once against test input", + Method: http.MethodPost, Path: "/automations/{id}/test", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The automation's id"}}, + }, + { + Name: "runs", Short: "Recent runs of an automation", + Method: http.MethodGet, Path: "/automations/{id}/runs", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The automation's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("STATUS", "status"), colf("STARTED", "created_at", "time"), colt("DETAIL", "error", 40), + }, Empty: "This automation has not run yet."}, + }, + }, + } +} + +func formSpec() resource { + formColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 30), col("STATUS", "status"), + col("VIEWS", "views_count"), col("SUBMISSIONS", "submissions_count"), + } + return resource{ + Name: "form", + Aliases: []string{"forms"}, + Short: "Lead capture forms", + Group: groupWork, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List forms", + Method: http.MethodGet, Path: "/forms", + Table: output.Table{Root: "data", Columns: formColumns, Empty: "No forms yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one form", + Method: http.MethodGet, Path: "/forms/{id}", + Args: []argSpec{{Name: "id", Help: "The form's id"}}, + Table: output.Table{Columns: formColumns}, + }, + { + Name: "create", Short: "Create a form", + Method: http.MethodPost, Path: "/forms", Body: bodyRequired, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Form name"}}, + Table: output.Table{Columns: formColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a form", + Method: http.MethodPatch, Path: "/forms/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The form's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Form name"}, + {Name: "status", Help: "draft or published"}, + {Name: "redirect-url", Help: "Where to send a visitor after submitting"}, + }, + Success: "Form updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a form", + Method: http.MethodDelete, Path: "/forms/{id}", + Args: []argSpec{{Name: "id", Help: "The form's id"}}, + Success: "Form deleted.", + }, + { + Name: "submissions", Short: "A form's submissions", + Method: http.MethodGet, Path: "/forms/{id}/submissions", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The form's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), colt("EMAIL", "email", 34), colf("WHEN", "created_at", "time"), + }, Empty: "No submissions yet."}, + }, + { + Name: "stats", Short: "A form's view and completion numbers", + Method: http.MethodGet, Path: "/forms/{id}/stats", + Args: []argSpec{{Name: "id", Help: "The form's id"}}, + }, + { + Name: "config", Short: "The workspace's form configuration", + Method: http.MethodGet, Path: "/forms/config", + }, + { + Name: "domain", Short: "The domain forms are served from", + Method: http.MethodGet, Path: "/forms/domain", + }, + { + Name: "set-domain", Short: "Set the domain forms are served from", + Method: http.MethodPut, Path: "/forms/domain", Body: bodyRequired, + Flag: []flagSpec{{Name: "domain", Help: "The custom forms domain"}}, + Success: "Forms domain set.", + }, + { + Name: "verify-domain", Short: "Check the forms domain's DNS", + Method: http.MethodPost, Path: "/forms/domain/verify", Body: bodyOptional, + }, + }, + } +} + +func dealSpec() resource { + dealColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 32), col("STATUS", "status"), + col("VALUE", "value"), col("CURRENCY", "currency"), colf("CLOSES", "expected_close_date", "date"), + } + return resource{ + Name: "deal", + Aliases: []string{"deals"}, + Short: "CRM deals", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List deals", + Method: http.MethodGet, Path: "/crm/deals", Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: dealColumns, Empty: "No deals yet."}, + }, + { + Name: "search", Short: "Search deals with a filter body", + Method: http.MethodPost, Path: "/crm/deals/search", Body: bodyOptional, Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: dealColumns, Empty: "Nothing matched."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one deal", + Method: http.MethodGet, Path: "/crm/deals/{id}", + Args: []argSpec{{Name: "id", Help: "The deal's id"}}, + Table: output.Table{Columns: dealColumns}, + }, + { + Name: "create", Short: "Create a deal", + Method: http.MethodPost, Path: "/crm/deals", Body: bodyRequired, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Deal name"}, + {Name: "pipeline", Help: "The pipeline it belongs to", Key: "pipeline_id"}, + {Name: "stage", Help: "The stage it starts in", Key: "stage_id"}, + {Name: "contact", Help: "The contact it is with", Key: "contact_id"}, + {Name: "value", Help: "Deal value", Kind: flagInt}, + {Name: "currency", Help: "Currency code"}, + }, + Table: output.Table{Columns: dealColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a deal", + Method: http.MethodPatch, Path: "/crm/deals/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The deal's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Deal name"}, + {Name: "stage", Help: "Move it to this stage", Key: "stage_id"}, + {Name: "status", Help: "open, won or lost"}, + {Name: "value", Help: "Deal value", Kind: flagInt}, + }, + Success: "Deal updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a deal", + Method: http.MethodDelete, Path: "/crm/deals/{id}", + Args: []argSpec{{Name: "id", Help: "The deal's id"}}, + Success: "Deal deleted.", + }, + { + Name: "summary", Short: "Deal totals by stage and status", + Method: http.MethodPost, Path: "/crm/deals/summary", Body: bodyOptional, + }, + }, + } +} + +func pipelineSpec() resource { + return resource{ + Name: "pipeline", + Aliases: []string{"pipelines"}, + Short: "CRM pipelines and their stages", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List pipelines with their stages", + Method: http.MethodGet, Path: "/crm/pipelines", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("NAME", "name"), col("POSITION", "position"), + }, Empty: "No pipelines yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one pipeline", + Method: http.MethodGet, Path: "/crm/pipelines/{id}", + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}}, + }, + { + Name: "create", Short: "Create a pipeline", + Method: http.MethodPost, Path: "/crm/pipelines", Body: bodyRequired, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Pipeline name"}}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Rename a pipeline", + Method: http.MethodPatch, Path: "/crm/pipelines/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}}, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Pipeline name"}}, + Success: "Pipeline updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a pipeline", + Method: http.MethodDelete, Path: "/crm/pipelines/{id}", + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}}, + Success: "Pipeline deleted.", + }, + { + Name: "add-stage", Short: "Add a stage to a pipeline", + Method: http.MethodPost, Path: "/crm/pipelines/{id}/stages", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Stage name"}, + {Name: "color", Help: "Stage colour"}, + }, + }, + { + Name: "edit-stage", Short: "Change a stage", + Method: http.MethodPatch, Path: "/crm/pipelines/{id}/stages/{stage}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}, {Name: "stage", Help: "The stage's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Stage name"}, + {Name: "color", Help: "Stage colour"}, + }, + Success: "Stage updated.", + }, + { + Name: "delete-stage", Short: "Delete a stage", + Method: http.MethodDelete, Path: "/crm/pipelines/{id}/stages/{stage}", + Args: []argSpec{{Name: "id", Help: "The pipeline's id"}, {Name: "stage", Help: "The stage's id"}}, + Success: "Stage deleted.", + }, + }, + } +} + +func taskSpec() resource { + taskColumns := []output.Column{ + col("ID", "id"), colt("TITLE", "title", 40), col("STATUS", "status"), colf("DUE", "due_at", "date"), + } + return resource{ + Name: "task", + Aliases: []string{"tasks"}, + Short: "CRM tasks and follow-ups", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List tasks", + Method: http.MethodGet, Path: "/crm/tasks", Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: taskColumns, Empty: "No tasks."}, + }, + { + Name: "search", Short: "Search tasks with a filter body", + Method: http.MethodPost, Path: "/crm/tasks/search", Body: bodyOptional, Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: taskColumns, Empty: "Nothing matched."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one task", + Method: http.MethodGet, Path: "/crm/tasks/{id}", + Args: []argSpec{{Name: "id", Help: "The task's id"}}, + Table: output.Table{Columns: taskColumns}, + }, + { + Name: "create", Short: "Create a task", + Method: http.MethodPost, Path: "/crm/tasks", Body: bodyRequired, + Flag: []flagSpec{ + {Name: "title", Short: "t", Help: "What needs doing"}, + {Name: "contact", Help: "The contact it is about", Key: "contact_id"}, + {Name: "due", Help: "When it is due (RFC 3339)", Key: "due_at"}, + }, + Table: output.Table{Columns: taskColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a task", + Method: http.MethodPatch, Path: "/crm/tasks/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The task's id"}}, + Flag: []flagSpec{ + {Name: "title", Short: "t", Help: "What needs doing"}, + {Name: "status", Help: "open or done"}, + {Name: "due", Help: "When it is due (RFC 3339)", Key: "due_at"}, + }, + Success: "Task updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a task", + Method: http.MethodDelete, Path: "/crm/tasks/{id}", + Args: []argSpec{{Name: "id", Help: "The task's id"}}, + Success: "Task deleted.", + }, + { + Name: "summary", Short: "Task counts by status", + Method: http.MethodPost, Path: "/crm/tasks/summary", Body: bodyOptional, + }, + { + Name: "types", Short: "The task types in use", + Method: http.MethodGet, Path: "/crm/task-types", + }, + }, + } +} + +func analyticsSpec() resource { + return resource{ + Name: "analytics", + Aliases: []string{"stats"}, + Short: "The numbers: sends, opens, replies, deliverability, warmup", + Group: groupData, + Endpoints: []endpoint{ + {Name: "dashboard", Short: "The headline numbers", Method: http.MethodGet, Path: "/analytics/dashboard"}, + {Name: "deliverability", Short: "Bounces, complaints and placement", Method: http.MethodGet, Path: "/analytics/deliverability"}, + { + Name: "warmup", Short: "Warmup analytics over a date range", + Method: http.MethodGet, Path: "/analytics/warmup", + Example: " $ warmbly analytics warmup --from 2026-01-01 --to 2026-01-31", + Flag: []flagSpec{ + {Name: "from", Help: "Start of the range, YYYY-MM-DD", Query: true}, + {Name: "to", Help: "End of the range, YYYY-MM-DD", Query: true}, + {Name: "mailbox", Help: "Only this mailbox", Query: true, Key: "email_id"}, + }, + }, + { + Name: "mailboxes", Aliases: []string{"accounts"}, Short: "Per-mailbox analytics", + Method: http.MethodGet, Path: "/analytics/accounts", + Table: output.Table{Root: "data", Columns: []output.Column{ + colt("MAILBOX", "email", 32), col("SENT", "sent"), col("OPENED", "opened"), + col("REPLIED", "replied"), col("BOUNCED", "bounced"), + }, Empty: "No mailbox analytics yet."}, + }, + { + Name: "mailbox", Aliases: []string{"account"}, Short: "One mailbox's analytics", + Method: http.MethodGet, Path: "/analytics/accounts/{id}", + Args: []argSpec{{Name: "id", Help: "The mailbox's id"}}, + }, + { + Name: "campaign", Short: "One campaign's analytics", + Method: http.MethodGet, Path: "/analytics/campaigns/{id}", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "campaign-daily", Short: "One campaign's daily series", + Method: http.MethodGet, Path: "/analytics/campaigns/{id}/daily", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "campaign-hourly", Short: "One campaign's hourly series", + Method: http.MethodGet, Path: "/analytics/campaigns/{id}/hourly", + Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, + }, + { + Name: "compare", Short: "Compare campaigns side by side over a date range", + Method: http.MethodGet, Path: "/analytics/campaigns/compare", + Example: " $ warmbly analytics compare --ids ID_A,ID_B --from 2026-01-01 --to 2026-01-31", + Flag: []flagSpec{ + {Name: "ids", Help: "Campaign ids to compare (required)", Kind: flagStrings, Query: true}, + {Name: "from", Help: "Start of the range, YYYY-MM-DD (required)", Query: true}, + {Name: "to", Help: "End of the range, YYYY-MM-DD (required)", Query: true}, + }, + }, + {Name: "usage", Short: "API and plan usage", Method: http.MethodGet, Path: "/analytics/usage"}, + }, + } +} + +func auditSpec() resource { + return resource{ + Name: "audit", + Short: "The workspace's audit trail", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls", "log"}, Short: "List audit entries, newest first", + Method: http.MethodGet, Path: "/audit-logs", Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("ACTION", "action"), col("ENTITY", "entity_type"), + colt("ACTOR", "actor_email", 30), colt("IP", "ip_address", 18), + }, Empty: "Nothing in the audit log yet."}, + }, + }, + } +} + +func advisorSpec() resource { + return resource{ + Name: "advisor", + Short: "Warmbly's recommendations for this workspace", + Group: groupData, + Endpoints: []endpoint{ + {Name: "summary", Short: "What the advisor thinks overall", Method: http.MethodGet, Path: "/advisor/summary"}, + { + Name: "list", Aliases: []string{"ls"}, Short: "List open recommendations", + Method: http.MethodGet, Path: "/advisor/recommendations", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("SEVERITY", "severity"), colt("TITLE", "title", 50), col("STATUS", "status"), + }, Empty: "Nothing to advise on. That is the good outcome."}, + }, + {Name: "settings", Short: "How the advisor is configured", Method: http.MethodGet, Path: "/advisor/settings"}, + {Name: "refresh", Short: "Recompute the recommendations now", Method: http.MethodPost, Path: "/advisor/refresh", Body: bodyOptional}, + { + Name: "apply", Short: "Apply a recommendation", + Method: http.MethodPost, Path: "/advisor/recommendations/{id}/apply", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The recommendation's id"}}, + Success: "Recommendation applied.", + }, + { + Name: "dismiss", Short: "Dismiss a recommendation", + Method: http.MethodPost, Path: "/advisor/recommendations/{id}/dismiss", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The recommendation's id"}}, + Success: "Recommendation dismissed.", + }, + { + Name: "snooze", Short: "Snooze a recommendation", + Method: http.MethodPost, Path: "/advisor/recommendations/{id}/snooze", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The recommendation's id"}}, + Success: "Recommendation snoozed.", + }, + { + Name: "undo", Short: "Undo an applied recommendation", + Method: http.MethodPost, Path: "/advisor/recommendations/{id}/undo", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The recommendation's id"}}, + Success: "Recommendation undone.", + }, + }, + } +} + +func webhookSpec() resource { + webhookColumns := []output.Column{ + col("ID", "id"), colt("URL", "url", 44), col("ENABLED", "enabled"), + col("FAILURES", "consecutive_failures"), colf("LAST OK", "last_success_at", "time"), + } + return resource{ + Name: "webhook", + Aliases: []string{"webhooks"}, + Short: "Webhook endpoints and their deliveries", + Group: groupDevelop, + Long: `Manage the HTTPS endpoints Warmbly posts events to. + +Every delivery is HMAC signed with the endpoint's secret, so rotating a secret +means updating the receiver at the same time.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List webhook endpoints", + Method: http.MethodGet, Path: "/webhooks", + Table: output.Table{Root: "data", Columns: webhookColumns, Empty: "No webhook endpoints yet."}, + }, + { + Name: "create", Short: "Create a webhook endpoint", + Method: http.MethodPost, Path: "/webhooks", Body: bodyRequired, + Example: " $ warmbly webhook create --url https://example.com/hooks/warmbly --events EMAIL_REPLIED,EMAIL_BOUNCED", + Flag: []flagSpec{ + {Name: "url", Help: "Where to POST events"}, + {Name: "description", Help: "What this endpoint is for"}, + {Name: "events", Help: "Event types to subscribe to", Kind: flagStrings, Key: "event_types"}, + }, + Table: output.Table{Columns: webhookColumns}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a webhook endpoint", + Method: http.MethodPatch, Path: "/webhooks/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The endpoint's id"}}, + Flag: []flagSpec{ + {Name: "url", Help: "Where to POST events"}, + {Name: "description", Help: "What this endpoint is for"}, + {Name: "events", Help: "Event types to subscribe to", Kind: flagStrings, Key: "event_types"}, + {Name: "enabled", Help: "Whether it receives events", Kind: flagBool}, + }, + Success: "Webhook updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a webhook endpoint", + Method: http.MethodDelete, Path: "/webhooks/{id}", + Args: []argSpec{{Name: "id", Help: "The endpoint's id"}}, + Success: "Webhook deleted.", + }, + { + Name: "verify", Short: "Send a verification ping to the endpoint", + Method: http.MethodPost, Path: "/webhooks/{id}/verify", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The endpoint's id"}}, + }, + { + Name: "rotate-secret", Short: "Rotate the endpoint's signing secret", + Method: http.MethodPost, Path: "/webhooks/{id}/rotate-secret", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The endpoint's id"}}, + Long: `Rotate the HMAC secret this endpoint's deliveries are signed with. + +The new secret is in the response and nowhere else. Update the receiver before +the next event fires, or its signature check will fail.`, + }, + { + Name: "deliveries", Short: "Recent deliveries across endpoints", + Method: http.MethodGet, Path: "/webhooks/deliveries", Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("EVENT", "event_type"), col("STATUS", "response_status"), colt("ERROR", "error", 40), + }, Empty: "Nothing has been delivered yet."}, + }, + { + Name: "redeliver", Short: "Send one delivery again", + Method: http.MethodPost, Path: "/webhooks/deliveries/{delivery}/redeliver", Body: bodyOptional, + Args: []argSpec{{Name: "delivery", Help: "The delivery's id"}}, + Success: "Redelivery queued.", + }, + { + Name: "event-types", Short: "Every event type a webhook can subscribe to", + Method: http.MethodGet, Path: "/webhooks/event-types", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("TYPE", "type"), col("CATEGORY", "category"), colt("DESCRIPTION", "description", 60), + }, Empty: "No event types reported."}, + }, + { + Name: "drops", Short: "Events dropped because the endpoint was throttled", + Method: http.MethodGet, Path: "/webhooks/throttle-drops", + }, + }, + } +} + +func keySpec() resource { + keyColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 34), col("PREFIX", "key_prefix"), + col("STATUS", "status"), colf("LAST USED", "last_used_at", "time"), + } + return resource{ + Name: "key", + Aliases: []string{"keys", "api-key", "api-keys"}, + Short: "API keys for this workspace", + Group: groupDevelop, + Long: `Manage the workspace's API keys. + +The key this CLI is signed in with is one of these, named for the machine that +created it. A key's secret exists only in the create response.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List API keys", + Method: http.MethodGet, Path: "/api-keys", Paginate: true, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: keyColumns, Empty: "No API keys yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one API key", + Method: http.MethodGet, Path: "/api-keys/{id}", + Args: []argSpec{{Name: "id", Help: "The key's id"}}, + Table: output.Table{Columns: keyColumns}, + }, + { + Name: "create", Short: "Create an API key", + Method: http.MethodPost, Path: "/api-keys", Body: bodyRequired, + Long: `Create an API key. + +The secret is in this response and is never retrievable again. Store it before +you close the terminal.`, + Example: " $ warmbly key create --name \"CI deploy\" --permissions 8388607", + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "What the key is for"}, + {Name: "description", Help: "Longer note"}, + {Name: "permissions", Help: "Scope bitmask; see `warmbly key permissions`", Kind: flagInt}, + {Name: "rate-limit", Help: "Requests per minute for this key", Kind: flagInt, Key: "rate_limit_per_minute"}, + {Name: "allowed-ips", Help: "Restrict the key to these IPs or CIDRs", Kind: flagStrings, Key: "allowed_ips"}, + }, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a key's name, scopes or restrictions", + Method: http.MethodPatch, Path: "/api-keys/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The key's id"}}, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "What the key is for"}, + {Name: "description", Help: "Longer note"}, + {Name: "permissions", Help: "Scope bitmask", Kind: flagInt}, + {Name: "rate-limit", Help: "Requests per minute for this key", Kind: flagInt, Key: "rate_limit_per_minute"}, + }, + Success: "API key updated.", + }, + { + Name: "revoke", Aliases: []string{"rm", "delete"}, Short: "Revoke an API key", + Method: http.MethodDelete, Path: "/api-keys/{id}", + Args: []argSpec{{Name: "id", Help: "The key's id"}}, + Success: "API key revoked.", + }, + { + Name: "permissions", Aliases: []string{"scopes"}, Short: "Every grantable scope and its bit value", + Method: http.MethodGet, Path: "/api-keys/permissions", + }, + { + Name: "logs", Short: "One key's request log", + Method: http.MethodGet, Path: "/api-keys/{id}/logs", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The key's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("METHOD", "method"), colt("ENDPOINT", "endpoint", 40), + col("STATUS", "response_code"), col("MS", "response_time_ms"), + }, Empty: "This key has not been used."}, + }, + { + Name: "analytics", Short: "One key's usage over time", + Method: http.MethodGet, Path: "/api-keys/{id}/analytics", + Args: []argSpec{{Name: "id", Help: "The key's id"}}, + Flag: []flagSpec{{Name: "interval", Help: "hour or day", Query: true}}, + }, + { + Name: "usage", Short: "API key usage across the workspace", + Method: http.MethodGet, Path: "/api-keys/usage/summary", + }, + }, + } +} + +func oauthAppSpec() resource { + appColumns := []output.Column{ + col("ID", "id"), colt("NAME", "name", 30), col("CLIENT", "client_id"), colf("CREATED", "created_at", "time"), + } + return resource{ + Name: "oauth-app", + Aliases: []string{"oauth-apps"}, + Short: "OAuth applications you publish", + Group: groupDevelop, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List OAuth applications", + Method: http.MethodGet, Path: "/oauth/applications", + Table: output.Table{Root: "data", Columns: appColumns, Empty: "No OAuth applications yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one application", + Method: http.MethodGet, Path: "/oauth/applications/{id}", + Args: []argSpec{{Name: "id", Help: "The application's id"}}, + Table: output.Table{Columns: appColumns}, + }, + { + Name: "create", Short: "Create an OAuth application", + Method: http.MethodPost, Path: "/oauth/applications", Body: bodyRequired, + Flag: []flagSpec{ + {Name: "name", Short: "n", Help: "Application name"}, + {Name: "redirect-uris", Help: "Allowed redirect URIs", Kind: flagStrings, Key: "redirect_uris"}, + }, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change an application", + Method: http.MethodPatch, Path: "/oauth/applications/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The application's id"}}, + Success: "Application updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete an application", + Method: http.MethodDelete, Path: "/oauth/applications/{id}", + Args: []argSpec{{Name: "id", Help: "The application's id"}}, + Success: "Application deleted.", + }, + { + Name: "rotate-secret", Short: "Rotate the application's client secret", + Method: http.MethodPost, Path: "/oauth/applications/{id}/rotate-secret", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The application's id"}}, + }, + }, + } +} + +func toolSpec() resource { + return resource{ + Name: "tool", + Aliases: []string{"tools"}, + Short: "The AI tool registry, callable over REST", + Group: groupDevelop, + Long: `The same tool registry the dashboard agent and MCP use, exposed as +plain REST for function-calling agents that do not speak MCP. + +Everything a tool can do is bounded by the signed-in key's scopes.`, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List the tools this key may call", + Method: http.MethodGet, Path: "/ai/tools", + Flag: []flagSpec{{Name: "format", Help: "openai emits function-calling manifests", Query: true}}, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("NAME", "name"), colt("DESCRIPTION", "description", 60), + }, Empty: "No tools available to this key."}, + }, + { + Name: "call", Short: "Run one tool", + Method: http.MethodPost, Path: "/ai/tools/{name}/call", Body: bodyOptional, + Args: []argSpec{{Name: "name", Help: "The tool's name, not a uuid"}}, + Example: " $ warmbly tool call list_campaigns\n $ warmbly tool call search_contacts -f query=acme.com", + }, + }, + } +} + +// orgSpec is deliberately one command. Every /organization/* route is JWT +// only: members, roles, invitations, exports and the danger zone all depend on +// a human-bound session and refuse an API key, which is the only credential +// this CLI holds. Shipping commands that can never succeed would be worse than +// not shipping them, so what is left is what GET /me answers, and the rest is +// a link to the dashboard. +func orgSpec() resource { + return resource{ + Name: "org", + Aliases: []string{"organization", "workspace"}, + Short: "Which workspace you are signed in to", + Group: groupData, + Long: `Show the workspace this credential belongs to. + +Members, roles, invitations, billing and workspace exports are session-only on +the API and cannot be driven with a key. ` + "`warmbly browse settings`" + ` opens +them in the dashboard.`, + Endpoints: []endpoint{ + { + Name: "view", Aliases: []string{"current", "me"}, Short: "Show the workspace and who you are in it", + Method: http.MethodGet, Path: "/me", + Table: output.Table{Columns: []output.Column{ + col("USER", "email"), + col("WORKSPACE", "organization_name"), + col("WORKSPACE ID", "organization_id"), + col("AUTH", "auth_type"), + }}, + }, + }, + } +} + +func teamSpec() resource { + return resource{ + Name: "team", + Aliases: []string{"teams"}, + Short: "Named groups of members, for CRM ownership and routing", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List teams", + Method: http.MethodGet, Path: "/teams", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("NAME", "name"), col("MEMBERS", "member_count"), + }, Empty: "No teams yet."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one team", + Method: http.MethodGet, Path: "/teams/{id}", + Args: []argSpec{{Name: "id", Help: "The team's id"}}, + }, + { + Name: "create", Short: "Create a team", + Method: http.MethodPost, Path: "/teams", Body: bodyRequired, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Team name"}}, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Rename a team", + Method: http.MethodPatch, Path: "/teams/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The team's id"}}, + Flag: []flagSpec{{Name: "name", Short: "n", Help: "Team name"}}, + Success: "Team updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a team", + Method: http.MethodDelete, Path: "/teams/{id}", + Args: []argSpec{{Name: "id", Help: "The team's id"}}, + Success: "Team deleted.", + }, + { + Name: "add-member", Short: "Add a member to a team", + Method: http.MethodPost, Path: "/teams/{id}/members", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The team's id"}}, + Flag: []flagSpec{{Name: "user", Help: "The member's user id", Key: "user_id"}}, + Success: "Member added.", + }, + { + Name: "remove-member", Short: "Remove a member from a team", + Method: http.MethodDelete, Path: "/teams/{id}/members/{user}", + Args: []argSpec{{Name: "id", Help: "The team's id"}, {Name: "user", Help: "The member's user id"}}, + Success: "Member removed.", + }, + }, + } +} + +func settingsSpec() resource { + return resource{ + Name: "settings", + Short: "Workspace-wide sending and suppression settings", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "view", Aliases: []string{"get", "outreach"}, Short: "Show the outreach settings", + Method: http.MethodGet, Path: "/outreach/settings", + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change the outreach settings", + Method: http.MethodPatch, Path: "/outreach/settings", Body: bodyRequired, + Long: `Change the workspace's outreach settings. + +These apply across every campaign, so they are the right place for a policy +decision (auto-suppress on bounce, honour unsubscribes globally) and the wrong +place for a per-campaign one.`, + Success: "Outreach settings updated.", + }, + }, + } +} + +func warmupRoutingSpec() resource { + return resource{ + Name: "warmup-routing", + Aliases: []string{"routing"}, + Short: "Rules deciding which mailboxes warm with which", + Group: groupData, + Endpoints: []endpoint{ + { + Name: "list", Aliases: []string{"ls"}, Short: "List routing rules", + Method: http.MethodGet, Path: "/warmup/routing", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("NAME", "name"), col("ENABLED", "enabled"), + }, Empty: "No warmup routing rules; the default pool policy applies."}, + }, + { + Name: "create", Short: "Create a routing rule", + Method: http.MethodPost, Path: "/warmup/routing", Body: bodyRequired, + }, + { + Name: "edit", Aliases: []string{"update"}, Short: "Change a routing rule", + Method: http.MethodPatch, Path: "/warmup/routing/{id}", Body: bodyRequired, + Args: []argSpec{{Name: "id", Help: "The rule's id"}}, + Success: "Routing rule updated.", + }, + { + Name: "delete", Aliases: []string{"rm"}, Short: "Delete a routing rule", + Method: http.MethodDelete, Path: "/warmup/routing/{id}", + Args: []argSpec{{Name: "id", Help: "The rule's id"}}, + Success: "Routing rule deleted.", + }, + }, + } +} + +func integrationSpec() resource { + return resource{ + Name: "integration", + Aliases: []string{"integrations"}, + Short: "Third-party connections", + Group: groupDevelop, + Long: `Manage connections to third-party tools. + +Connecting one usually needs a browser (OAuth consent), so creating a +connection stays in the dashboard. Everything after that is here.`, + Endpoints: []endpoint{ + { + Name: "catalog", Short: "Every integration this instance offers", + Method: http.MethodGet, Path: "/integrations/catalog", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("KEY", "key"), col("NAME", "name"), col("CATEGORY", "category"), + }, Empty: "No integrations available."}, + }, + { + Name: "list", Aliases: []string{"ls", "connections"}, Short: "List this workspace's connections", + Method: http.MethodGet, Path: "/integrations/connections", + Table: output.Table{Root: "data", Columns: []output.Column{ + col("ID", "id"), col("PROVIDER", "provider"), col("STATUS", "status"), colf("CONNECTED", "created_at", "time"), + }, Empty: "Nothing connected."}, + }, + { + Name: "view", Aliases: []string{"get"}, Short: "Show one connection", + Method: http.MethodGet, Path: "/integrations/connections/{id}", + Args: []argSpec{{Name: "id", Help: "The connection's id"}}, + }, + { + Name: "test", Short: "Check a connection still works", + Method: http.MethodPost, Path: "/integrations/connections/{id}/test", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The connection's id"}}, + }, + { + Name: "runs", Short: "A connection's recent runs", + Method: http.MethodGet, Path: "/integrations/connections/{id}/runs", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The connection's id"}}, + Flag: withPaging(), + Table: output.Table{Root: "data", Columns: []output.Column{ + colf("WHEN", "created_at", "time"), col("STATUS", "status"), colt("DETAIL", "error", 44), + }, Empty: "This connection has not run yet."}, + }, + { + Name: "push", Short: "Push data through a connection now", + Method: http.MethodPost, Path: "/integrations/connections/{id}/push", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The connection's id"}}, + }, + { + Name: "disconnect", Aliases: []string{"rm"}, Short: "Disconnect an integration", + Method: http.MethodDelete, Path: "/integrations/connections/{id}", + Args: []argSpec{{Name: "id", Help: "The connection's id"}}, + Success: "Integration disconnected.", + }, + }, + } +} diff --git a/cmd/cli/status.go b/cmd/cli/status.go new file mode 100644 index 00000000..e246d368 --- /dev/null +++ b/cmd/cli/status.go @@ -0,0 +1,241 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/api" + "github.com/warmbly/warmbly/internal/cli/iostreams" + "github.com/warmbly/warmbly/internal/models" +) + +// `warmbly status` is the one screen answer to "what is happening in my +// workspace right now". It is several calls composed into one view, and every +// section degrades on its own: a key without analytics scope still gets the +// mailbox and inbox lines rather than one failure for the whole command. + +func newStatusCmd(f *Factory) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "What is happening in your workspace right now", + GroupID: groupCore, + Long: `A single screen: who you are, which mailboxes need attention, what is +sending today, and what is waiting for a reply. + +Sections you have no scope for are skipped rather than failing the command.`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runStatus(c.Context(), f) + }, + } +} + +func runStatus(ctx context.Context, f *Factory) error { + io := f.IO + client, err := f.Client() + if err != nil { + return err + } + + // --json is one document rather than a rendered screen, so a script gets + // the same information without parsing prose. + bundle := map[string]any{} + collect := func(key, path string, query url.Values) json.RawMessage { + resp, derr := client.Do(ctx, api.Request{Method: http.MethodGet, Path: path, Query: query}) + if derr != nil { + if f.Debug { + io.Errorf("* %s: %v\n", path, derr) + } + return nil + } + var parsed any + if json.Unmarshal(resp.Body, &parsed) == nil { + bundle[key] = parsed + } + return resp.Body + } + + identity := collect("me", "/me", nil) + mailboxes := collect("mailboxes", "/emails", url.Values{"limit": []string{"100"}}) + campaigns := collect("campaigns", "/campaigns", url.Values{"limit": []string{"100"}}) + unread := collect("inbox", "/unibox/count", nil) + dashboard := collect("analytics", "/analytics/dashboard", nil) + + if f.JSONOut || !io.IsStdoutTTY() { + raw, merr := json.MarshalIndent(bundle, "", " ") + if merr != nil { + return merr + } + io.Println(string(raw)) + return nil + } + + printIdentity(io, identity) + printMailboxes(io, mailboxes) + printCampaigns(io, campaigns) + printInbox(io, unread, dashboard) + + if len(bundle) == 0 { + return fmt.Errorf("nothing could be read with this credential. Run `warmbly auth status` to check it.") + } + return nil +} + +func printIdentity(io *iostreams.IOStreams, raw json.RawMessage) { + if raw == nil { + return + } + var id struct { + Email string `json:"email"` + OrganizationName string `json:"organization_name"` + } + if json.Unmarshal(raw, &id) != nil { + return + } + line := io.Bold(id.Email) + if id.OrganizationName != "" { + line += io.Gray(" in ") + io.Bold(id.OrganizationName) + } + io.Printf("%s\n\n", line) +} + +func printMailboxes(io *iostreams.IOStreams, raw json.RawMessage) { + if raw == nil { + return + } + var list struct { + Data []struct { + Email string `json:"email"` + Status string `json:"status"` + AuthState string `json:"auth_state"` + CampaignLimit int `json:"campaign_limit"` + Warmup *string `json:"warmup"` + } `json:"data"` + } + if json.Unmarshal(raw, &list) != nil { + return + } + + total, warming, capacity, unchecked := len(list.Data), 0, 0, 0 + var trouble []string + for _, m := range list.Data { + capacity += m.CampaignLimit + if m.Warmup != nil { + warming++ + } + // "unknown" means never checked and never gates sending, so it is not + // a problem to report; only a failing check is. + switch { + case m.Status != "" && m.Status != "active": + trouble = append(trouble, fmt.Sprintf("%s %s", m.Email, io.Red(m.Status))) + case m.AuthState == models.AuthStateFailing: + trouble = append(trouble, fmt.Sprintf("%s %s", m.Email, io.Yellow("SPF, DKIM or DMARC failing"))) + case m.AuthState == models.AuthStateUnknown: + unchecked++ + } + } + + io.Printf("%s\n", io.Gray("MAILBOXES")) + if total == 0 { + io.Printf(" none connected. `warmbly browse mailboxes` opens the dashboard.\n\n") + return + } + io.Printf(" %d connected, %d warming, %d emails/day of campaign capacity\n", total, warming, capacity) + for _, t := range trouble { + io.Printf(" %s %s\n", io.Cross(), t) + } + if len(trouble) == 0 { + io.Printf(" %s all healthy\n", io.Tick()) + } + if unchecked > 0 { + io.Printf(" %s\n", io.Gray(fmt.Sprintf("%d never had their authentication checked: `warmbly mailbox recheck `", unchecked))) + } + io.Println() +} + +func printCampaigns(io *iostreams.IOStreams, raw json.RawMessage) { + if raw == nil { + return + } + var list struct { + Data []struct { + Name string `json:"name"` + Status string `json:"status"` + } `json:"data"` + } + if json.Unmarshal(raw, &list) != nil { + return + } + + counts := map[string]int{} + var active []string + for _, c := range list.Data { + counts[c.Status]++ + if c.Status == "active" || c.Status == "running" { + active = append(active, c.Name) + } + } + + io.Printf("%s\n", io.Gray("CAMPAIGNS")) + if len(list.Data) == 0 { + io.Printf(" none yet. `warmbly campaign create --name \"My campaign\"` starts one.\n\n") + return + } + var parts []string + for _, status := range []string{"active", "paused", "draft", "completed"} { + if n := counts[status]; n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n, status)) + } + } + io.Printf(" %s\n", strings.Join(parts, ", ")) + for i, name := range active { + if i == 5 { + io.Printf(" %s\n", io.Gray(fmt.Sprintf("and %d more sending", len(active)-5))) + break + } + io.Printf(" %s %s\n", io.Green("▸"), name) + } + io.Println() +} + +func printInbox(io *iostreams.IOStreams, unread, dashboard json.RawMessage) { + io.Printf("%s\n", io.Gray("INBOX")) + if unread != nil { + var count struct { + Count int `json:"count"` + Total int `json:"total"` + } + if json.Unmarshal(unread, &count) == nil { + n := count.Count + if n == 0 { + n = count.Total + } + if n > 0 { + io.Printf(" %s unread\n", io.Bold(fmt.Sprint(n))) + } else { + io.Printf(" %s nothing unread\n", io.Tick()) + } + } + } + if dashboard != nil { + var stats map[string]any + if json.Unmarshal(dashboard, &stats) == nil { + var parts []string + for _, key := range []string{"sent", "opened", "clicked", "replied", "bounced"} { + if v, ok := stats[key]; ok { + parts = append(parts, fmt.Sprintf("%v %s", v, key)) + } + } + if len(parts) > 0 { + io.Printf("\n%s\n %s\n", io.Gray("RECENT ACTIVITY"), strings.Join(parts, ", ")) + } + } + } + io.Println() +} diff --git a/cmd/cli/upgrade.go b/cmd/cli/upgrade.go new file mode 100644 index 00000000..301f95a0 --- /dev/null +++ b/cmd/cli/upgrade.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/cli/config" + "github.com/warmbly/warmbly/internal/cli/update" + "github.com/warmbly/warmbly/internal/version" +) + +func newUpgradeCmd(f *Factory) *cobra.Command { + var check bool + cmd := &cobra.Command{ + Use: "upgrade", + Aliases: []string{"update", "self-update"}, + Short: "Update the CLI to the newest release", + GroupID: groupSetup, + Long: `Replace this binary with the newest published release. + +When the CLI came from a package manager it says which command to run instead, +because overwriting a file Homebrew or Scoop owns produces a version that +reverts on their next upgrade.`, + Example: ` $ warmbly upgrade + $ warmbly upgrade --check`, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { + return runUpgrade(c.Context(), f, check) + }, + } + cmd.Flags().BoolVar(&check, "check", false, "Only report whether a newer release exists") + return cmd +} + +func runUpgrade(ctx context.Context, f *Factory, checkOnly bool) error { + io := f.IO + current := version.String() + + io.Errorf("%s\n", io.Gray("Checking for a newer release")) + latest, err := update.LatestVersion(ctx, 15*time.Second) + if err != nil { + return fmt.Errorf("could not reach the release feed: %w", err) + } + + // The state file is refreshed here too, so an explicit check silences the + // automatic reminder for the next day. + state := config.LoadState() + state.LastUpdateCheck = time.Now().UTC() + state.LatestVersion = latest + _ = state.Save() + + if !update.IsNewer(current, latest) { + if current == latest { + io.Printf("%s warmbly %s is the newest release\n", io.Tick(), io.Bold(current)) + } else { + // A dev build is not behind, it is simply not one of ours. + io.Printf("%s running %s; the newest release is %s\n", io.Tick(), io.Bold(current), io.Bold(latest)) + } + return nil + } + + io.Printf("%s %s is available (you have %s)\n", io.Yellow("↑"), io.Bold(latest), current) + if checkOnly { + return nil + } + + executable, err := os.Executable() + if err != nil { + return fmt.Errorf("could not find this binary on disk: %w", err) + } + if cmd := update.DetectMethod(executable).UpgradeCommand(); cmd != "" { + io.Println() + io.Printf("This CLI was installed with a package manager. Upgrade it with:\n\n %s\n", io.Bold(cmd)) + return nil + } + + if !f.AssumeYes && io.IsStdinTTY() { + ok, cerr := io.Confirm(fmt.Sprintf("Replace %s with %s?", executable, latest), true) + if cerr != nil { + return cerr + } + if !ok { + return errCancelled + } + } + + if err := update.Replace(ctx, executable, func(step string) { + io.Errorf("%s %s\n", io.Gray("…"), step) + }); err != nil { + return err + } + + io.Printf("%s Upgraded to %s\n", io.Tick(), io.Bold(latest)) + return nil +} + +// nudgeAboutUpdates prints a one-line reminder after a command has finished, +// at most once a day, and only when someone is watching. +// +// It runs after the command's own output so it never delays a result, and +// every failure is swallowed: a version check is not worth an error message, +// and an air-gapped machine must not pay for one on every run. +func nudgeAboutUpdates(ctx context.Context, f *Factory) { + if !f.IO.IsStdoutTTY() || !f.IO.IsStdinTTY() { + return + } + if f.JSONOut || os.Getenv("WARMBLY_NO_UPDATE_CHECK") != "" || os.Getenv("CI") != "" { + return + } + + state := config.LoadState() + if latest := state.LatestVersion; latest != "" && update.IsNewer(version.String(), latest) { + f.IO.Errorf("\n%s %s is available. Run %s\n", + f.IO.Yellow("↑"), f.IO.Bold(latest), f.IO.Bold("warmbly upgrade")) + } + if time.Since(state.LastUpdateCheck) < update.CheckInterval { + return + } + + // Two seconds is the whole budget: this happens after the user already has + // what they asked for, and a slow network must not make the CLI feel slow. + latest, err := update.LatestVersion(ctx, 2*time.Second) + if err != nil { + return + } + state.LastUpdateCheck = time.Now().UTC() + state.LatestVersion = latest + _ = state.Save() +} diff --git a/cmd/cli/version.go b/cmd/cli/version.go new file mode 100644 index 00000000..cb4721f9 --- /dev/null +++ b/cmd/cli/version.go @@ -0,0 +1,45 @@ +package main + +import ( + "encoding/json" + "runtime" + + "github.com/spf13/cobra" + + "github.com/warmbly/warmbly/internal/version" +) + +func newVersionCmd(f *Factory) *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the CLI's version", + GroupID: groupSetup, + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + info := version.Current() + if f.JSONOut { + payload, err := json.MarshalIndent(map[string]string{ + "version": info.Version, + "commit": info.Commit, + "built": info.BuiltAt, + "go": runtime.Version(), + "os": runtime.GOOS, + "arch": runtime.GOARCH, + }, "", " ") + if err != nil { + return err + } + f.IO.Println(string(payload)) + return nil + } + f.IO.Printf("warmbly %s (%s/%s)\n", info.Version, runtime.GOOS, runtime.GOARCH) + if c := version.ShortCommit(); c != "" { + f.IO.Printf("commit %s\n", c) + } + if info.BuiltAt != "" { + f.IO.Printf("built %s\n", info.BuiltAt) + } + return nil + }, + } +} diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index d6b65e74..88d6398f 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -19,13 +19,16 @@ import ( "github.com/warmbly/warmbly/internal/app/advanced" "github.com/warmbly/warmbly/internal/app/cipher" jobs "github.com/warmbly/warmbly/internal/app/consumer" + "github.com/warmbly/warmbly/internal/app/contact" "github.com/warmbly/warmbly/internal/app/credits" "github.com/warmbly/warmbly/internal/app/creditwatch" "github.com/warmbly/warmbly/internal/app/feature" "github.com/warmbly/warmbly/internal/app/inboxagent" + "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/app/integration" "github.com/warmbly/warmbly/internal/app/nativeactions" "github.com/warmbly/warmbly/internal/app/notification" + "github.com/warmbly/warmbly/internal/app/opsnotify" "github.com/warmbly/warmbly/internal/app/replyclassify" warmupapp "github.com/warmbly/warmbly/internal/app/warmup" "github.com/warmbly/warmbly/internal/app/webhook" @@ -47,6 +50,7 @@ import ( "github.com/warmbly/warmbly/internal/observability" "github.com/warmbly/warmbly/internal/pkg/encrypt" "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/geo" "github.com/warmbly/warmbly/internal/repository" ) @@ -313,10 +317,18 @@ func main() { // wiring native actions here a reply-triggered automation's add_tag / // create_deal / label_email node would fail with "native actions are not // available". Mirrors the backend wiring. + // The lead-intake actions (create or update contact, add to campaign) write + // through a contact service so a reply-triggered flow in this process gets + // the same plan check, campaign wake and contact.created as the backend. + contactServiceC := contact.NewService(contactRepo, subscriptionRepoConsumer, planRepoConsumer, streamingPublisher) + if aware, ok := contactServiceC.(contact.WebhookAware); ok { + aware.WireWebhooks(webhookService) + } integrationServiceC.SetNativeActions(nativeactions.Adapter{ - Adv: advancedService, - Contacts: contactRepo, - Orgs: orgRepoConsumer, + Adv: advancedService, + Contacts: contactRepo, + Orgs: orgRepoConsumer, + ContactSvc: contactServiceC, }) // In-app notifications: the reply/bounce/complaint gate fires in THIS // process (inbox ingest + deliverability ingest run in the consumer), so the @@ -339,6 +351,22 @@ func main() { log.Printf("Warning: notification email disabled, EMAIL_NAME/EMAIL_ADDRESS not set: %v", ecErr) } notificationService.WireDelivery(notifEmail, integrationServiceC, repository.NewUserRepostory(primaryDB, kmsClient), orgRepoConsumer) + + // Operator alerts. The dead-worker detector runs in this process, and a + // stranded fleet is the operator's problem, not a tenant's. Reads the same + // channel list the admin panel writes; a mail transport is optional (the + // chat and webhook transports do not need one). + var opsMailer opsnotify.Mailer + if notifEmail != nil { + if m, ok := notifEmail.(opsnotify.Mailer); ok { + opsMailer = m + } + } + opsNotifierC := opsnotify.NewService( + instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool)), + opsMailer, + config.AppBaseURL(), + ) // Mobile push (APNs) fires from THIS process too: reply/bounce/complaint // notifications are created here. Redis backs the immediate-then-digest // window shared with the backend. The sender stays a nil interface (not a @@ -398,6 +426,7 @@ func main() { AdminRepo: repository.NewAdminRepository(primaryDB.Pool), AssignmentService: workerAssignmentSvc, Notifier: notificationService, + OpsNotifier: opsNotifierC, TaskRepo: taskRepo, CampaignRepo: campaignRepo, CampaignProgressRepo: campaignProgressRepo, @@ -468,6 +497,14 @@ func main() { // open/click action chains (advancedService), the open/click analog of the // reply path. Decodes with the same codec the Rust tracking service writes // (Avro on Kafka, JSON on NATS). + // GeoIP is optional here as on the backend: it only turns an open or + // click's source network into a country and city on the logs. + geoPath, _ := cfg.LoadGeoDBPath(ctx) + geoloc, gerr := geo.New(geoPath) + if gerr != nil { + log.Printf("GeoIP database not found at %s; engagement locations are disabled.", geoPath) + geoloc, _ = geo.New("") + } if trackingCfg, terr := cfg.LoadTrackingConsumerConfig(ctx); terr != nil { log.Println("tracking consumer config unavailable; opens/clicks not consumed:", terr) } else if trackingConsumer, terr := jobs.NewTrackingConsumer( @@ -481,11 +518,18 @@ func main() { contactRepo, streamingPublisher, repository.NewTrackingDedupeRepository(primaryDB.Pool), + repository.NewTrackedLinkRepository(primaryDB.Pool), + repository.NewLinkClickRepository(primaryDB.Pool), advancedService, verificationEvidence, + repository.NewEmailOpenRepository(primaryDB.Pool), + geoloc, ); terr != nil { log.Println("tracking consumer unavailable; opens/clicks not consumed:", terr) } else { + // The engagement prune reads its window from the instance settings on + // every pass, so shortening it in the admin panel needs no restart. + trackingConsumer.WireRetention(instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool))) defer trackingConsumer.Close() go func() { if err := trackingConsumer.Start(ctx); err != nil { diff --git a/cmd/updater/main.go b/cmd/updater/main.go new file mode 100644 index 00000000..48673763 --- /dev/null +++ b/cmd/updater/main.go @@ -0,0 +1,114 @@ +// The updater is the host-side agent behind "Update and restart" in the admin +// panel. It moves the install forward and restarts the stack, then reports +// progress to the backend. UPDATER_MODE picks how: compose pulls a git +// checkout and rebuilds it, image pins a release tag in the install's .env and +// pulls the published images (the clone-free install.sh install), command +// hands the rebuild to a script of your own. +// See docs/content/docs/development/updates.mdx. +package main + +import ( + "context" + "errors" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/warmbly/warmbly/internal/updater" + "github.com/warmbly/warmbly/internal/version" +) + +func main() { + log.SetFlags(log.LstdFlags | log.LUTC) + + repoDir := getenv("UPDATER_REPO_DIR", "") + if repoDir == "" { + if wd, err := os.Getwd(); err == nil { + repoDir = wd + } + } + cfg := updater.Config{ + Mode: updater.Mode(getenv("UPDATER_MODE", string(updater.ModeCompose))), + RepoDir: repoDir, + Remote: getenv("UPDATER_REMOTE", "origin"), + Command: os.Getenv("UPDATER_COMMAND"), + ComposeProject: getenv("UPDATER_COMPOSE_PROJECT", "warmbly"), + ComposeProfiles: getenv("UPDATER_COMPOSE_PROFILES", "updater"), + BackendHealthURL: getenv("UPDATER_BACKEND_HEALTH_URL", "http://backend:8080/health"), + StateDir: getenv("UPDATER_STATE_DIR", "/var/lib/warmbly-updater"), + FetchInterval: duration("UPDATER_FETCH_INTERVAL", 30*time.Minute), + Prune: boolean("UPDATER_PRUNE", true), + AllowDirty: boolean("UPDATER_ALLOW_DIRTY", false), + Version: version.String(), + } + switch cfg.Mode { + case updater.ModeCompose, updater.ModeCommand, updater.ModeImage: + default: + log.Fatalf("updater: UPDATER_MODE must be compose, image or command, got %q", cfg.Mode) + } + + runner, err := updater.NewRunner(cfg) + if err != nil { + log.Fatalf("updater: %v", err) + } + // The updater shares the backend's internal token unless given its own. + token := getenv("UPDATER_TOKEN", os.Getenv("INTERNAL_API_TOKEN")) + server, err := updater.NewServer(runner, token) + if err != nil { + log.Fatalf("updater: %v", err) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + runner.Start(ctx) + + addr := getenv("UPDATER_ADDR", ":8095") + srv := &http.Server{Addr: addr, Handler: server.Handler(), ReadHeaderTimeout: 10 * time.Second} + go func() { + <-ctx.Done() + runner.Stop() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + }() + log.Printf("updater: %s mode, install %s, listening on %s (version %s)", cfg.Mode, cfg.RepoDir, addr, cfg.Version) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("updater: %v", err) + } +} + +func getenv(key, def string) string { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + return def +} + +func duration(key string, def time.Duration) time.Duration { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil || d < time.Minute { + return def + } + return d +} + +func boolean(key string, def bool) bool { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return def + } + b, err := strconv.ParseBool(v) + if err != nil { + return def + } + return b +} diff --git a/cmd/warmblyctl/api_resources.go b/cmd/warmblyctl/api_resources.go index 644b1416..98edd48e 100644 --- a/cmd/warmblyctl/api_resources.go +++ b/cmd/warmblyctl/api_resources.go @@ -82,6 +82,11 @@ var apiSpecs = []apiSpec{ {name: "contact import-commit", summary: "Commit a previewed bulk import", method: "POST", path: "/contacts/import/commit", body: bodyRequired}, {name: "contact export", summary: "Export contacts", method: "POST", path: "/contacts/export", body: bodyOptional}, + // Suppression list. + {name: "suppression list", summary: "The workspace suppression list: addresses and domains that get no campaign mail", method: "GET", path: "/suppressions", query: []string{"limit", "cursor", "q"}}, + {name: "suppression add", summary: "Add addresses or domains to the suppression list", method: "POST", path: "/suppressions", body: bodyRequired}, + {name: "suppression remove", summary: "Lift a suppression entry so the address can be emailed again", method: "DELETE", path: "/suppressions/{id}"}, + // Mailboxes (email accounts). {name: "mailbox list", summary: "List connected mailboxes", method: "GET", path: "/emails", query: []string{"limit", "cursor", "q"}}, {name: "mailbox get", summary: "Get one mailbox", method: "GET", path: "/emails/{id}"}, @@ -169,24 +174,25 @@ var apiSpecs = []apiSpec{ // apiFamilyOrder keeps the top-level help stable; maps iterate randomly. var apiFamilyOrder = []string{ - "me", "campaign", "contact", "mailbox", "inbox", "analytics", + "me", "campaign", "contact", "suppression", "mailbox", "inbox", "analytics", "settings", "webhook", "apikey", "template", "crm", "tool", } // apiFamilies drives top-level dispatch and help for the typed commands. var apiFamilies = map[string]string{ - "me": "Who the API key is", - "campaign": "Campaigns and their sequences", - "contact": "Contacts, notes, and imports", - "mailbox": "Connected mailboxes and warmup", - "inbox": "The unified inbox", - "analytics": "Analytics and the audit trail", - "settings": "Organization outreach settings", - "webhook": "Webhook endpoints", - "apikey": "API keys", - "template": "Reply templates", - "crm": "Pipelines, deals, and CRM tasks", - "tool": "AI agent tools (list and call the registry)", + "me": "Who the API key is", + "campaign": "Campaigns and their sequences", + "contact": "Contacts, notes, and imports", + "suppression": "The suppression list: addresses and domains that get no campaign mail", + "mailbox": "Connected mailboxes and warmup", + "inbox": "The unified inbox", + "analytics": "Analytics and the audit trail", + "settings": "Organization outreach settings", + "webhook": "Webhook endpoints", + "apikey": "API keys", + "template": "Reply templates", + "crm": "Pipelines, deals, and CRM tasks", + "tool": "AI agent tools (list and call the registry)", } // runAPIResource runs one typed command: `warmblyctl campaign start --id ...`. diff --git a/cmd/warmblyctl/backup.go b/cmd/warmblyctl/backup.go new file mode 100644 index 00000000..e7e9f83d --- /dev/null +++ b/cmd/warmblyctl/backup.go @@ -0,0 +1,922 @@ +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/warmbly/warmbly/internal/version" +) + +// Instance-level backup and restore: one bundle holding everything a Warmbly +// install is, restorable onto a fresh host. +// +// This is deliberately not `warmblyctl org export`. That one moves ONE +// workspace between two running instances and re-seals its secrets for the +// destination's keys. This one moves THE INSTANCE: every workspace, every +// user, the platform admins, the API keys, and the keys those secrets are +// sealed with, onto a host that has nothing yet. +// +// The reason it is one command rather than a documented list of steps is that +// the parts are only a backup together. A pg_dump alone restores an instance +// whose every mailbox credential decrypts to nothing, because the ciphertext +// is in the database and the key that opens it is in .env; the blob root alone +// restores bodies nothing points at. The bundle carries all three and the +// restore refuses to run when the keys do not match, which is the failure this +// command exists to make impossible. + +// bundleVersion is the archive layout. The restore refuses a version it does +// not know rather than half-applying an archive it cannot read. +const bundleVersion = 1 + +// Paths inside the bundle. +const ( + manifestPath = "manifest.json" + databasePath = "database.sql" + keysPath = "keys.env" + blobsPrefix = "blobs/" +) + +// backupKeys are the environment values that belong in the bundle. The first +// two are unrecoverable: every sealed mailbox credential and every per-org DEK +// is opened with them, so a database without them is not a backup. The rest +// are here because restoring without them signs out every session and breaks +// every worker and websocket connection, which reads as a broken restore. +var backupKeys = []string{ + "CREDENTIALS_ENCRYPTION_KEY", + "KMS_LOCAL_MASTER_KEY", + "AUTH_SECRET", + "INTERNAL_API_TOKEN", + "SECRET_KEY_BASE", +} + +// unrecoverableKeys are the two the restore checks. A mismatch on either is +// refused, because it produces an instance that looks restored and whose +// mailboxes authenticate against nothing. +var unrecoverableKeys = []string{"CREDENTIALS_ENCRYPTION_KEY", "KMS_LOCAL_MASTER_KEY"} + +// manifest describes what a bundle holds, so the restore can say what it is +// about to do before it does it. +type manifest struct { + Version int `json:"version"` + CreatedAt time.Time `json:"created_at"` + AppVersion string `json:"app_version"` + AppURL string `json:"app_url,omitempty"` + Database string `json:"database"` + DatabaseSHA string `json:"database_sha256"` + // Counts are for the confirmation line, not for logic. + Organizations int `json:"organizations"` + Users int `json:"users"` + Mailboxes int `json:"mailboxes"` + // BlobProvider records why blobs are or are not in the bundle. An s3 + // install keeps its bodies in the bucket, and the bucket is not ours to + // copy, so the archive says so instead of pretending to be complete. + BlobProvider string `json:"blob_provider"` + BlobRoot string `json:"blob_root,omitempty"` + BlobFiles int `json:"blob_files"` + BlobBytes int64 `json:"blob_bytes"` + // Keys is the list of key names in keys.env, never their values. + Keys []string `json:"keys"` +} + +// ---------- backup ---------- + +func runBackup(ctx context.Context, args []string) error { + fs := newFlagSet("backup") + out := fs.String("out", "", "Path to write the bundle to. Defaults to ./warmbly-backup-.tar.gz") + noKeys := fs.Bool("no-keys", false, "Leave the encryption keys out. The bundle is then not restorable on its own.") + noBlobs := fs.Bool("no-blobs", false, "Leave message bodies, attachments and avatars out. Database only.") + force := fs.Bool("force", false, "Overwrite the output file if it already exists") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + + target := strings.TrimSpace(*out) + if target == "" { + target = fmt.Sprintf("warmbly-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405")) + } + if _, err := os.Stat(target); err == nil && !*force { + return fmt.Errorf("%s already exists. Pass --force to overwrite it, or --out with another path.", target) + } + + dsn, err := dbEndpoint(ctx) + if err != nil { + return err + } + if _, err := exec.LookPath("pg_dump"); err != nil { + return errors.New("pg_dump is not on PATH, so the database cannot be dumped.\nRun this inside the backend container, where it is installed:\n " + composeExec + "backup --out /data/blobs/warmbly-backup.tar.gz") + } + + m := manifest{ + Version: bundleVersion, + CreatedAt: time.Now().UTC(), + AppVersion: version.String(), + AppURL: strings.TrimSpace(os.Getenv("APP_URL")), + Database: databaseName(dsn), + } + + // Counts first: they are the line that tells an operator this bundle is of + // the instance they meant, and a database that cannot be counted cannot be + // dumped either, so failing here fails before anything is written. + c, err := connect(ctx) + if err != nil { + return err + } + m.Organizations, m.Users, m.Mailboxes = instanceCounts(ctx, c) + c.close() + + fmt.Printf("Backing up %s\n", redact(dsn)) + + dumpFile, dumpSize, dumpSHA, err := dumpDatabase(ctx, dsn) + if err != nil { + return err + } + defer func() { + _ = os.Remove(dumpFile) + }() + m.DatabaseSHA = dumpSHA + fmt.Printf(" database %s\n", humanBytes(dumpSize)) + + var blobFiles []blobEntry + m.BlobProvider = strings.ToLower(strings.TrimSpace(os.Getenv("BLOB_PROVIDER"))) + if m.BlobProvider == "" { + m.BlobProvider = "filesystem" + } + switch { + case *noBlobs: + fmt.Println(" blobs skipped (--no-blobs)") + case m.BlobProvider != "filesystem": + fmt.Printf(" blobs not included: this instance stores them in %s. Back that store up separately.\n", m.BlobProvider) + default: + root := blobRoot() + m.BlobRoot = root + // The documented hand-off writes the bundle into the blob root, because + // that is the one path the container and the host both see. Archiving + // it into itself would grow a bundle by the size of the last one on + // every run, so the output is excluded by path. + blobFiles, err = collectBlobs(root, target) + if err != nil { + return err + } + for _, b := range blobFiles { + m.BlobBytes += b.size + } + m.BlobFiles = len(blobFiles) + fmt.Printf(" blobs %d files, %s from %s\n", m.BlobFiles, humanBytes(m.BlobBytes), root) + } + + keys := map[string]string{} + if !*noKeys { + for _, k := range backupKeys { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + keys[k] = v + m.Keys = append(m.Keys, k) + } + } + sort.Strings(m.Keys) + missing := missingUnrecoverable(keys) + if len(missing) > 0 { + warn("%s not set in this environment, so %s not in the bundle. Restoring it elsewhere will not open sealed mailbox credentials.", + strings.Join(missing, " and "), plural(len(missing), "it is", "they are")) + } + fmt.Printf(" keys %d (%s)\n", len(m.Keys), strings.Join(m.Keys, ", ")) + } else { + fmt.Println(" keys skipped (--no-keys); this bundle cannot restore a working instance on its own") + } + + if err := writeBundle(target, m, dumpFile, blobFiles, keys); err != nil { + return err + } + + st, _ := os.Stat(target) + size := int64(0) + if st != nil { + size = st.Size() + } + fmt.Printf("\nWrote %s (%s)\n", target, humanBytes(size)) + steps := []string{ + "Copy it off this host. It holds every mailbox credential and the keys that open them,", + "so treat the file as you would the instance itself: 0600, encrypted at rest, off-site.", + "", + "Restore it on another host with:", + " " + composeExec + "restore --file /path/to/" + filepath.Base(target), + } + if len(m.Keys) == 0 { + steps = append(steps, "", "This bundle carries no keys. Copy CREDENTIALS_ENCRYPTION_KEY and KMS_LOCAL_MASTER_KEY", "from this instance's .env by hand, or the restored mailboxes will not connect.") + } + printSteps("Next:", steps) + return nil +} + +// ---------- restore ---------- + +func runRestore(ctx context.Context, args []string) error { + fs := newFlagSet("restore") + file := fs.String("file", "", "Path to a bundle written by warmblyctl backup (required)") + yes := fs.Bool("yes", false, "Skip the typed confirmation. For scripts.") + noBlobs := fs.Bool("no-blobs", false, "Restore the database only, leaving the blob root alone") + force := fs.Bool("force", false, "Restore even though this host's encryption keys differ from the bundle's") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + if strings.TrimSpace(*file) == "" { + return errors.New("--file is required. Point it at a bundle written by `warmblyctl backup`.") + } + if _, err := exec.LookPath("psql"); err != nil { + return errors.New("psql is not on PATH, so the database cannot be restored.\nRun this inside the backend container, where it is installed.") + } + + m, err := readManifest(*file) + if err != nil { + return err + } + if m.Version != bundleVersion { + return fmt.Errorf("this bundle is layout version %d and this warmblyctl reads version %d. Restore it with the Warmbly it was written by (%s).", m.Version, bundleVersion, m.AppVersion) + } + + dsn, err := dbEndpoint(ctx) + if err != nil { + return err + } + + fmt.Printf("Bundle %s\n", *file) + fmt.Printf(" written %s by Warmbly %s\n", m.CreatedAt.Local().Format(time.RFC1123), m.AppVersion) + if m.AppURL != "" { + fmt.Printf(" from %s\n", m.AppURL) + } + fmt.Printf(" holds %d organizations, %d users, %d mailboxes\n", m.Organizations, m.Users, m.Mailboxes) + if m.BlobFiles > 0 { + fmt.Printf(" %d blob files, %s\n", m.BlobFiles, humanBytes(m.BlobBytes)) + } + fmt.Printf("Target %s\n\n", redact(dsn)) + + // The key check is the point of this command. A restore onto a host whose + // keys differ produces an instance that looks fine and whose every mailbox + // fails to authenticate, days later, with no error that names the cause. + keysMatch, err := checkRestoreKeys(*file, m, *force) + if err != nil { + return err + } + + if !*yes { + fmt.Printf("This REPLACES everything in %s. Every organization, user, campaign and\nmailbox currently on this instance is dropped and replaced by the bundle's.\n\n", databaseName(dsn)) + ok, cerr := confirmPhrase("Type 'restore' to continue: ", "restore") + if cerr != nil { + return cerr + } + if !ok { + return errors.New("nothing was changed") + } + fmt.Println() + } + + // Checked before anything is dropped: the restore empties the schema first, + // so finding out afterwards that the bundle is truncated leaves the target + // with neither its own data nor the bundle's. + if m.DatabaseSHA != "" { + if err := verifyDump(*file, m.DatabaseSHA); err != nil { + return err + } + fmt.Println(" bundle intact") + } + + fmt.Println("Restoring the database...") + if err := restoreDatabase(ctx, dsn, *file); err != nil { + return err + } + fmt.Println(" database restored") + + if !*noBlobs && m.BlobFiles > 0 { + root := blobRoot() + n, bytes, rerr := restoreBlobs(*file, root) + if rerr != nil { + return rerr + } + fmt.Printf(" blobs %d files, %s into %s\n", n, humanBytes(bytes), root) + } + + steps := []string{} + switch { + case len(m.Keys) == 0: + case keysMatch: + steps = append(steps, + "The bundle's keys match this host, so sealed mailbox credentials open as they did.", + ) + default: + steps = append(steps, + "The keys did NOT match and --force was given, so every restored mailbox", + "credential is unreadable. Reconnect each mailbox, or put the bundle's keys", + "in .env and restore again.", + ) + } + steps = append(steps, + "Restart the stack so every service picks the restored database up:", + " docker compose -p warmbly restart", + "", + "Then check it:", + " "+composeExec+"status", + ) + printSteps("Next:", steps) + return nil +} + +// checkRestoreKeys refuses a restore whose ciphertext this host cannot open. +// +// The bundle's keys are compared against the environment rather than written +// anywhere: warmblyctl runs inside a container and the .env that would have to +// change is on the host, so the honest thing is to print the two lines and +// stop, not to half-apply and report success. +// checkRestoreKeys reports whether this host can open the bundle's ciphertext, +// and refuses the restore when it cannot unless force overrides it. +func checkRestoreKeys(file string, m manifest, force bool) (bool, error) { + if len(m.Keys) == 0 { + warn("This bundle carries no encryption keys. If this host's CREDENTIALS_ENCRYPTION_KEY and\nKMS_LOCAL_MASTER_KEY are not the ones the bundle was written with, every restored mailbox\ncredential will fail to decrypt.") + return false, nil + } + bundled, err := readKeys(file) + if err != nil { + return false, err + } + var mismatched, missing []string + for _, k := range unrecoverableKeys { + want, inBundle := bundled[k] + if !inBundle { + continue + } + got := strings.TrimSpace(os.Getenv(k)) + switch { + case got == "": + missing = append(missing, k+"="+want) + case got != want: + mismatched = append(mismatched, k+"="+want) + } + } + if len(mismatched) == 0 && len(missing) == 0 { + return true, nil + } + + lines := append(append([]string{}, missing...), mismatched...) + msg := strings.Builder{} + msg.WriteString("This host's encryption keys are not the ones the bundle was sealed with.\n") + msg.WriteString("Restoring anyway gives you an instance whose mailbox credentials cannot be\ndecrypted, and there is no way to recover them afterwards.\n\n") + msg.WriteString("Put these in the install's .env on the host, recreate the containers, then run\nthe restore again:\n\n") + for _, l := range lines { + msg.WriteString(" " + l + "\n") + } + if force { + warn("%s", msg.String()) + warn("--force was given, so the restore continues. Mailboxes will need reconnecting.") + return false, nil + } + msg.WriteString("\nPass --force only if you accept losing every stored mailbox credential.") + return false, errors.New(msg.String()) +} + +// ---------- database ---------- + +// dumpDatabase writes a plain-SQL dump to a temp file and returns its path, +// size and digest. Plain rather than custom format so the bundle can be +// inspected, and so a restore needs psql alone. +func dumpDatabase(ctx context.Context, dsn string) (string, int64, string, error) { + tmp, err := os.CreateTemp("", "warmbly-dump-*.sql") + if err != nil { + return "", 0, "", err + } + path := tmp.Name() + + sum := sha256.New() + w := bufio.NewWriterSize(io.MultiWriter(tmp, sum), 1<<20) + + // No owner or ACL statements: the destination's database user is whatever + // its own compose file created, and it is never guaranteed to carry the + // same name as this one's. + cmd := pgCommand(ctx, "pg_dump", dsn, "--no-owner", "--no-privileges", "--format=plain") + cmd.Stdout = w + var stderr strings.Builder + cmd.Stderr = &stderr + if rerr := cmd.Run(); rerr != nil { + _ = tmp.Close() + _ = os.Remove(path) + return "", 0, "", fmt.Errorf("pg_dump failed: %s", strings.TrimSpace(stderr.String())) + } + if ferr := w.Flush(); ferr != nil { + _ = tmp.Close() + _ = os.Remove(path) + return "", 0, "", ferr + } + size, _ := tmp.Seek(0, io.SeekCurrent) + if cerr := tmp.Close(); cerr != nil { + _ = os.Remove(path) + return "", 0, "", cerr + } + return path, size, hex.EncodeToString(sum.Sum(nil)), nil +} + +// verifyDump reads the bundle's dump once and compares its digest with the one +// the manifest recorded, so a bundle truncated by a failed copy is refused +// rather than half-applied. +func verifyDump(bundle, want string) error { + sql, closeFn, err := openBundleEntry(bundle, databasePath) + if err != nil { + return err + } + defer closeFn() + + sum := sha256.New() + if _, cerr := io.Copy(sum, sql); cerr != nil { + return fmt.Errorf("reading the bundle's database dump: %w", cerr) + } + got := hex.EncodeToString(sum.Sum(nil)) + if got != want { + return fmt.Errorf("this bundle is damaged: its database dump hashes to %s and the manifest says %s.\nNothing was changed. Copy the bundle again from wherever it came from.", short(got), short(want)) + } + return nil +} + +// short trims a digest to something a person can compare by eye. +func short(sum string) string { + if len(sum) > 12 { + return sum[:12] + } + return sum +} + +// restoreDatabase empties the schema and replays the dump into it. The schema +// is dropped rather than the dump carrying DROP statements, because the +// destination has already had migrations applied at boot and objects the +// bundle does not know about would otherwise survive into the restored +// instance. +func restoreDatabase(ctx context.Context, dsn, bundle string) error { + reset := pgCommand(ctx, "psql", dsn, "-v", "ON_ERROR_STOP=1", "-q", + "-c", "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;") + var resetErr strings.Builder + reset.Stderr = &resetErr + if err := reset.Run(); err != nil { + return fmt.Errorf("could not empty the target schema: %s", strings.TrimSpace(resetErr.String())) + } + + sql, closeFn, err := openBundleEntry(bundle, databasePath) + if err != nil { + return err + } + defer closeFn() + + load := pgCommand(ctx, "psql", dsn, "-v", "ON_ERROR_STOP=1", "-q") + load.Stdin = sql + var loadErr strings.Builder + load.Stderr = &loadErr + if err := load.Run(); err != nil { + return fmt.Errorf("the dump did not load cleanly, so the database is now empty and unusable.\nFix the cause and run the restore again:\n%s", strings.TrimSpace(loadErr.String())) + } + return nil +} + +// pgCommand builds a libpq invocation with the password moved out of argv. +// +// A connection string on the command line is readable by anyone on the host who +// can list processes, and this one opens the database that holds every sealed +// mailbox credential. libpq reads PGPASSWORD from the environment, which is not +// world-readable on Linux, so the DSN that reaches argv carries no password. +func pgCommand(ctx context.Context, name, dsn string, args ...string) *exec.Cmd { + clean, password := splitDSNPassword(dsn) + cmd := exec.CommandContext(ctx, name, append([]string{"--dbname=" + clean}, args...)...) //nolint:gosec // name is a literal at every call site + cmd.Env = os.Environ() + if password != "" { + cmd.Env = append(cmd.Env, "PGPASSWORD="+password) + } + return cmd +} + +// splitDSNPassword returns the connection string with its password removed and +// the password separately. A DSN this cannot parse (libpq also accepts +// keyword/value form) is returned unchanged, because a connection that fails is +// worse than a password in argv. +func splitDSNPassword(dsn string) (string, string) { + parsed, err := url.Parse(dsn) + if err != nil || parsed.User == nil { + return dsn, "" + } + password, ok := parsed.User.Password() + if !ok || password == "" { + return dsn, "" + } + parsed.User = url.User(parsed.User.Username()) + return parsed.String(), password +} + +func instanceCounts(ctx context.Context, c *conn) (orgs, users, mailboxes int) { + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM organizations`).Scan(&orgs) + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&users) + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM email_accounts`).Scan(&mailboxes) + return +} + +// databaseName is the database a connection string names, for the line that +// tells an operator what is about to be replaced. +func databaseName(dsn string) string { + if i := strings.LastIndex(dsn, "/"); i >= 0 { + name := dsn[i+1:] + if j := strings.IndexAny(name, "?"); j >= 0 { + name = name[:j] + } + if name != "" { + return name + } + } + return "the database" +} + +// ---------- blobs ---------- + +type blobEntry struct { + abs string + rel string + size int64 + mode os.FileMode +} + +func blobRoot() string { + if v := strings.TrimSpace(os.Getenv("BLOB_FS_ROOT")); v != "" { + return v + } + return "/data/blobs" +} + +// collectBlobs walks the blob root, skipping the bundle being written and +// warning about any earlier one still sitting there. +func collectBlobs(root, exclude string) ([]blobEntry, error) { + if abs, aerr := filepath.Abs(exclude); aerr == nil { + exclude = abs + } + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + warn("the blob root %s does not exist on this host, so no bodies or attachments are in the bundle.", root) + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + var out []blobEntry + err = filepath.WalkDir(root, func(path string, d os.DirEntry, werr error) error { + if werr != nil { + return werr + } + // Symlinks are skipped rather than followed: the blob root is written + // by the app, and a link out of it would put arbitrary host files in a + // bundle the operator believes holds message bodies. + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + if abs, aerr := filepath.Abs(path); aerr == nil && abs == exclude { + return nil + } + rel, rerr := filepath.Rel(root, path) + if rerr != nil { + return rerr + } + // An earlier bundle left in the blob root is not blob data, and putting + // it inside this one doubles the archive for nothing. + if !strings.Contains(rel, string(os.PathSeparator)) && + strings.HasPrefix(rel, "warmbly-") && strings.HasSuffix(rel, ".tar.gz") { + warn("%s looks like an earlier backup and was left out of this one. Delete it: it is sitting in the blob root, where it is backed up over and over.", filepath.Join(root, rel)) + return nil + } + st, serr := d.Info() + if serr != nil { + // Vanished between the walk and the stat. A live instance writes + // bodies constantly, and one disappearing is not a failed backup. + if os.IsNotExist(serr) { + return nil + } + return serr + } + out = append(out, blobEntry{abs: path, rel: filepath.ToSlash(rel), size: st.Size(), mode: st.Mode().Perm()}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].rel < out[j].rel }) + return out, nil +} + +// restoreBlobs unpacks the bundle's blob tree under root. Existing files are +// overwritten; files the bundle does not know about are left alone, so a +// restore onto a host that already holds bodies is additive rather than a +// silent deletion. +func restoreBlobs(bundle, root string) (int, int64, error) { + f, gz, tr, err := openBundle(bundle) + if err != nil { + return 0, 0, err + } + defer f.Close() + defer gz.Close() + + var n int + var total int64 + for { + hdr, nerr := tr.Next() + if nerr == io.EOF { + break + } + if nerr != nil { + return n, total, nerr + } + if hdr.Typeflag != tar.TypeReg || !strings.HasPrefix(hdr.Name, blobsPrefix) { + continue + } + rel := strings.TrimPrefix(hdr.Name, blobsPrefix) + dest, derr := safeJoin(root, rel) + if derr != nil { + return n, total, derr + } + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return n, total, err + } + out, oerr := os.OpenFile(dest, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode).Perm()) //nolint:gosec // path is checked by safeJoin + if oerr != nil { + return n, total, oerr + } + written, cerr := io.Copy(out, tr) //nolint:gosec // sizes come from a bundle the operator wrote + _ = out.Close() + if cerr != nil { + return n, total, cerr + } + n++ + total += written + } + return n, total, nil +} + +// safeJoin refuses a bundle entry that would land outside the destination. +// A backup is an operator's own file, but it is also the kind of file that +// gets emailed around, and a path traversal in one is a host compromise. +func safeJoin(root, rel string) (string, error) { + clean := filepath.Clean("/" + filepath.FromSlash(rel)) + joined := filepath.Join(root, clean) + if !strings.HasPrefix(joined, filepath.Clean(root)+string(os.PathSeparator)) { + return "", fmt.Errorf("the bundle holds an entry that would be written outside %s: %s", root, rel) + } + return joined, nil +} + +// ---------- bundle io ---------- + +func writeBundle(target string, m manifest, dumpFile string, blobs []blobEntry, keys map[string]string) error { + // 0600 from creation, not chmod after: the file holds every secret the + // instance has, and a window in which it is world-readable is a window. + f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) //nolint:gosec // operator-supplied output path + if err != nil { + return err + } + defer f.Close() + + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + + if err := tarBytes(tw, manifestPath, mustJSON(m), 0o600, m.CreatedAt); err != nil { + return err + } + if len(keys) > 0 { + if err := tarBytes(tw, keysPath, []byte(renderKeys(keys)), 0o600, m.CreatedAt); err != nil { + return err + } + } + if err := tarFile(tw, databasePath, dumpFile, 0o600, m.CreatedAt); err != nil { + return err + } + for _, b := range blobs { + if err := tarFile(tw, blobsPrefix+b.rel, b.abs, b.mode, m.CreatedAt); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + return f.Close() +} + +func tarBytes(tw *tar.Writer, name string, body []byte, mode os.FileMode, when time.Time) error { + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: int64(mode), Size: int64(len(body)), ModTime: when, Typeflag: tar.TypeReg, + }); err != nil { + return err + } + _, err := tw.Write(body) + return err +} + +// tarFile writes one file, tolerating the instance writing to it underneath. +// +// A tar entry's size is fixed by its header, so a file that grows between the +// stat and the copy has to be truncated and one that shrinks has to be padded; +// getting either wrong corrupts the whole archive. A file that disappears is +// skipped, because a live instance is writing message bodies the entire time a +// backup runs and one of them vanishing is not a failed backup. +func tarFile(tw *tar.Writer, name, path string, mode os.FileMode, when time.Time) error { + st, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + f, err := os.Open(path) //nolint:gosec // paths come from the instance's own blob root + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + + size := st.Size() + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: int64(mode), Size: size, ModTime: when, Typeflag: tar.TypeReg, + }); err != nil { + return err + } + written, err := io.CopyN(tw, f, size) + if err != nil && err != io.EOF { + return err + } + if written < size { + // It shrank. The header is already committed, so the entry is padded + // to the length it promised. + if _, perr := io.CopyN(tw, zeroReader{}, size-written); perr != nil { + return perr + } + } + return nil +} + +// zeroReader pads a tar entry whose file shrank while it was being read. +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = 0 + } + return len(p), nil +} + +func openBundle(path string) (*os.File, *gzip.Reader, *tar.Reader, error) { + f, err := os.Open(path) //nolint:gosec // operator-supplied bundle path + if err != nil { + return nil, nil, nil, fmt.Errorf("could not read %s: %w", path, err) + } + gz, gerr := gzip.NewReader(bufio.NewReaderSize(f, 1<<20)) + if gerr != nil { + _ = f.Close() + return nil, nil, nil, fmt.Errorf("%s is not a Warmbly bundle (it is not gzip): %w", path, gerr) + } + return f, gz, tar.NewReader(gz), nil +} + +// openBundleEntry streams one entry out of the bundle. The caller closes. +func openBundleEntry(path, want string) (io.Reader, func(), error) { + f, gz, tr, err := openBundle(path) + if err != nil { + return nil, func() {}, err + } + closeFn := func() { + _ = gz.Close() + _ = f.Close() + } + for { + hdr, nerr := tr.Next() + if nerr == io.EOF { + closeFn() + return nil, func() {}, fmt.Errorf("%s holds no %s; it is not a Warmbly bundle", path, want) + } + if nerr != nil { + closeFn() + return nil, func() {}, nerr + } + if hdr.Name == want { + return tr, closeFn, nil + } + } +} + +func readBundleEntry(path, want string, limit int64) ([]byte, error) { + r, closeFn, err := openBundleEntry(path, want) + if err != nil { + return nil, err + } + defer closeFn() + return io.ReadAll(io.LimitReader(r, limit)) +} + +func readManifest(path string) (manifest, error) { + var m manifest + raw, err := readBundleEntry(path, manifestPath, 1<<20) + if err != nil { + return m, err + } + if err := json.Unmarshal(raw, &m); err != nil { + return m, fmt.Errorf("%s holds an unreadable manifest: %w", path, err) + } + return m, nil +} + +func readKeys(path string) (map[string]string, error) { + raw, err := readBundleEntry(path, keysPath, 1<<20) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + out[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + return out, nil +} + +func renderKeys(keys map[string]string) string { + names := make([]string, 0, len(keys)) + for k := range keys { + names = append(names, k) + } + sort.Strings(names) + b := strings.Builder{} + b.WriteString("# Warmbly instance keys. These belong in the destination's .env BEFORE the\n") + b.WriteString("# restore runs; warmblyctl restore refuses to run without them and says so.\n") + for _, k := range names { + b.WriteString(k + "=" + keys[k] + "\n") + } + return b.String() +} + +// ---------- small helpers ---------- + +func missingUnrecoverable(keys map[string]string) []string { + var out []string + for _, k := range unrecoverableKeys { + if keys[k] == "" { + out = append(out, k) + } + } + return out +} + +func mustJSON(v any) []byte { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return []byte("{}") + } + return append(b, '\n') +} + +// confirmPhrase reads one line and reports whether it is the phrase. It needs +// a TTY; piped into a container without one, --yes is the documented path. +func confirmPhrase(prompt, phrase string) (bool, error) { + fmt.Print(prompt) + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil && line == "" { + return false, errors.New("could not read the confirmation. Pass --yes to skip it, or run this without `exec -T`.") + } + return strings.TrimSpace(line) == phrase, nil +} + +func plural(n int, one, many string) string { + if n == 1 { + return one + } + return many +} diff --git a/cmd/warmblyctl/main.go b/cmd/warmblyctl/main.go index 98e5d580..5a06279f 100644 --- a/cmd/warmblyctl/main.go +++ b/cmd/warmblyctl/main.go @@ -69,6 +69,10 @@ func dispatch(ctx context.Context, args []string) error { return runUser(ctx, args[1:]) case "org": return runOrg(ctx, args[1:]) + case "backup": + return runBackup(ctx, args[1:]) + case "restore": + return runRestore(ctx, args[1:]) case "api": return runAPI(ctx, args[1:]) } @@ -99,6 +103,8 @@ var commands = []command{ {"user revoke-admin", "Take platform admin away from an account", composeExec + "user revoke-admin --email old@example.com"}, {"user disable-2fa", "Clear an account's TOTP enrollment after a lost authenticator", composeExec + "user disable-2fa --email you@example.com"}, {"hash-password", "Print an argon2 hash for WARMBLY_BOOTSTRAP_PASSWORD_HASH", "printf '%s' 'your-password' | docker compose -p warmbly exec -T backend warmblyctl hash-password"}, + {"backup", "Write the whole instance (database, blobs, keys) to one restorable bundle", composeExec + "backup --out /data/blobs/warmbly-backup.tar.gz"}, + {"restore", "Restore a bundle onto this instance, replacing everything on it", composeExec + "restore --file /data/blobs/warmbly-backup.tar.gz"}, {"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"}, {"org export", "Write a whole workspace to a portable archive file", composeExec + "org export --org you@example.com --out /tmp/workspace.warmbly.zip"}, {"org import", "Apply an archive to a workspace on this instance", composeExec + "org import --org you@example.com --file /tmp/workspace.warmbly.zip"}, @@ -146,6 +152,10 @@ Environment: WARMBLY_API_URL API base URL for the API commands. Defaults to this instance's API_PUBLIC_URL, then the hosted service. + backup and restore read the same crypto settings as org export/import, plus + BLOB_FS_ROOT for the blob tree. Run them inside the backend container, which + is also where pg_dump and psql live. + org export and org import additionally read the instance's crypto settings, because a workspace's sealed values have to be opened on the way out and re-sealed on the way in: KMS_PROVIDER (with KMS_LOCAL_MASTER_KEY or the AWS diff --git a/cmd/warmblyctl/status.go b/cmd/warmblyctl/status.go index fb69ea4c..eeadd3cc 100644 --- a/cmd/warmblyctl/status.go +++ b/cmd/warmblyctl/status.go @@ -11,6 +11,7 @@ import ( "github.com/warmbly/warmbly/internal/app/instancecheck" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/version" ) // errChecksFailed is how `make doctor` fails a script. The findings are already @@ -41,6 +42,9 @@ type instanceStatus struct { Checks []instancecheck.Finding `json:"checks"` Summary instancecheck.Summary `json:"summary"` + + Version string `json:"version"` + Commit string `json:"commit,omitempty"` } func runStatus(ctx context.Context, args []string) error { @@ -120,6 +124,9 @@ func collectStatus(ctx context.Context, c *conn) (*instanceStatus, error) { AppURL: config.AppBaseURL(), AppURLSource: appURLSource(), + + Version: version.String(), + Commit: version.ShortCommit(), } state.NextSteps = nextSteps(state) state.Checks, state.Summary = runChecks(ctx, c) @@ -133,7 +140,13 @@ func printStatus(s *instanceStatus) { } mail := fmt.Sprintf("%s (%s; %s)", s.MailTransport, mailEffect(s.MailTransport, s.MailDelivers), s.MailTransportSource) + build := s.Version + if s.Commit != "" { + build += " (" + s.Commit + ")" + } + fmt.Println("Instance") + fmt.Printf(" Version %s\n", build) fmt.Printf(" Accounts %d\n", s.Accounts) fmt.Printf(" Claimed %s\n", claimed) fmt.Printf(" Platform admins %d\n", s.AdminCount) diff --git a/deploy/config/env.example b/deploy/config/env.example index 8160327a..8093454f 100644 --- a/deploy/config/env.example +++ b/deploy/config/env.example @@ -31,6 +31,32 @@ INTERNAL_API_TOKEN=change-me-internal-token PRIMARY_DB=postgres://warmbly:warmbly@localhost:5432/warmbly_dev?sslmode=disable REDIS=redis://localhost:6379 +# === Release and storage layout (docker-compose.yml only) === +# Both keys exist on every app service, so this one compose file either pulls +# the published images or builds this checkout. Pin a release and every later +# `docker compose up` is reproducible; `install.sh` writes these for you. +# WARMBLY_TAG=v1.4.2 # default: prod (a moving tag) +# WARMBLY_IMAGE_PREFIX=ghcr.io/warmbly/warmbly +# +# One variable per store. Compose reads a source starting with / as a bind +# mount and anything else as a named volume, so a path here is the difference +# between "rsync this directory" and "ask docker volume inspect where it went". +# WARMBLY_PG_DATA=/mnt/data/warmbly/postgres +# WARMBLY_BLOBS=/mnt/data/warmbly/blobs +# WARMBLY_NATS_DATA=/mnt/data/warmbly/nats +# WARMBLY_REDIS_DATA=/mnt/data/warmbly/redis +# WARMBLY_WORKER_STATE=/mnt/data/warmbly/worker +# WARMBLY_UPDATER_STATE=/mnt/data/warmbly/updater + +# === Database-backed settings, seeded on first boot === +# The sync budgets and retention windows live in the database and are edited in +# the admin panel under Instance > Instance settings. This seeds them on an +# instance that has never saved that document, so an unattended install can +# ship its data-control answers with the rest of the environment. It is a no-op +# from the first save onwards, so leaving it here never undoes an edit made +# there. https://docs.warmbly.com/development/data-control/ +# WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"engagement_event_days":30,"form_event_days":30,"audit_log_days":30}} + # === Provider switches (no-cloud defaults) === AWS_CONFIG_ENABLED=false # true => read secrets from AWS SSM/Secrets Manager @@ -99,7 +125,9 @@ GIN_MODE=release # debug | release # PUBLIC_HOST=192.168.1.50 # API_PUBLIC_URL is the compose input for the API base the frontends and blob # URLs use (falls back to http://:8080); set it explicitly behind -# a reverse proxy. ENV_LABEL labels the admin panel (compose: WARMBLY_ENV_LABEL). +# a reverse proxy. Recipient unsubscribe links (the List-Unsubscribe header +# and the link in campaign emails) are served on this origin too, so it must +# be reachable from the public internet. ENV_LABEL labels the admin panel (compose: WARMBLY_ENV_LABEL). # API_PUBLIC_URL=https://api.example.com # ENV_LABEL=production APP_URL=http://localhost:5173 @@ -244,6 +272,10 @@ TRACKING_PAGEHIT_RATE_LIMIT_PER_MIN=60 # The header that proxy sets with the client address (x-forwarded-for, or # cf-connecting-ip behind Cloudflare). Nothing else is read. # TRACKING_CLIENT_IP_HEADER=x-forwarded-for +# Secret the source-address token in tracking events is keyed with; defaults +# to INTERNAL_API_TOKEN. Set it apart to rotate the token without rotating +# the internal API token. +# TRACKING_IP_HASH_KEY= # === Forms service (hosted form pages, embeds, submissions) === # Its own process (cmd/forms) on FORMS_PORT, so form traffic never touches the diff --git a/deploy/docker/backend.Dockerfile b/deploy/docker/backend.Dockerfile index 27515885..cec8561c 100644 --- a/deploy/docker/backend.Dockerfile +++ b/deploy/docker/backend.Dockerfile @@ -15,6 +15,8 @@ FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder ARG GO_TAGS="" ARG TARGETOS TARGETARCH +# Build identity shown in the admin panel; see internal/version. +ARG VERSION="" COMMIT="" BUILT_AT="" RUN apk add --no-cache git ca-certificates && \ if echo "$GO_TAGS" | grep -qw kafka; then apk add --no-cache gcc musl-dev librdkafka-dev; fi @@ -27,16 +29,20 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ set -eux; \ if echo "$GO_TAGS" | grep -qw kafka; then CGO=1; TAGS="musl kafka"; else CGO=0; TAGS=""; fi; \ - CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w" -o /out/backend ./cmd/backend; \ - CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w" -o /out/seed ./cmd/seed; \ - CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w" -o /out/migrate ./cmd/migrate; \ - CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w" -o /out/warmblyctl ./cmd/warmblyctl + CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/backend ./cmd/backend; \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/seed ./cmd/seed; \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/migrate ./cmd/migrate; \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/warmblyctl ./cmd/warmblyctl; \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/warmbly ./cmd/cli # Runtime stage FROM alpine:3.23 ARG GO_TAGS="" -RUN apk add --no-cache ca-certificates tzdata && \ +# postgresql-client is here for `warmblyctl backup` and `warmblyctl restore`: +# the instance bundle is a pg_dump and the restore replays it with psql, and the +# backend container is where the CLI already has PRIMARY_DB and the blob root. +RUN apk add --no-cache ca-certificates tzdata postgresql-client && \ if echo "$GO_TAGS" | grep -qw kafka; then apk add --no-cache librdkafka; fi && \ adduser -D -u 1000 warmbly @@ -55,6 +61,10 @@ COPY --from=builder /out/migrate /app/migrate # `docker compose exec backend warmblyctl status` and not a path. COPY --from=builder /out/warmblyctl /usr/local/bin/warmblyctl +# The customer CLI ships alongside it, so an operator who has exec on the box +# can drive the product as well as recover it without installing anything. +COPY --from=builder /out/warmbly /usr/local/bin/warmbly + # Installer script the worker orchestrator uploads + runs over SSH, and serves # at GET /worker-install.sh. The mode is explicit because COPY otherwise keeps # the checkout's: on a filesystem without POSIX permissions that is 0700, and diff --git a/deploy/docker/cli.Dockerfile b/deploy/docker/cli.Dockerfile new file mode 100644 index 00000000..c2b9c3a7 --- /dev/null +++ b/deploy/docker/cli.Dockerfile @@ -0,0 +1,46 @@ +# The `warmbly` CLI as an image, for CI jobs and anywhere installing a binary +# is more trouble than pulling one: +# +# docker run --rm -e WARMBLY_TOKEN ghcr.io/warmbly/warmbly/cli campaign list +# +# Distroless-style: the CLI is a static binary that talks to one HTTPS API, so +# the runtime needs certificates, timezone data and nothing else. +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION="" +ARG COMMIT="" +ARG BUILT_AT="" + +WORKDIR /app +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build \ + -ldflags="-s -w \ + -X github.com/warmbly/warmbly/internal/version.Version=$VERSION \ + -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT \ + -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" \ + -o /out/warmbly ./cmd/cli + +FROM alpine:3.23 + +RUN apk add --no-cache ca-certificates tzdata && adduser -D -u 1000 warmbly + +COPY --from=builder /out/warmbly /usr/local/bin/warmbly + +# A container has no browser, so `warmbly auth login` cannot finish here. +# WARMBLY_TOKEN is the documented way in, and the config directory is a volume +# mount point for anyone who would rather bring their hosts.yml. +ENV WARMBLY_CONFIG_DIR=/home/warmbly/.config/warmbly \ + WARMBLY_NO_UPDATE_CHECK=1 + +USER warmbly +WORKDIR /home/warmbly + +ENTRYPOINT ["warmbly"] +CMD ["--help"] diff --git a/deploy/docker/consumer.Dockerfile b/deploy/docker/consumer.Dockerfile index 912d1cbc..d1e1e3a7 100644 --- a/deploy/docker/consumer.Dockerfile +++ b/deploy/docker/consumer.Dockerfile @@ -7,6 +7,8 @@ FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder ARG GO_TAGS="" ARG TARGETOS TARGETARCH +# Build identity shown in the admin panel; see internal/version. +ARG VERSION="" COMMIT="" BUILT_AT="" RUN apk add --no-cache git ca-certificates && \ if echo "$GO_TAGS" | grep -qw kafka; then apk add --no-cache gcc musl-dev librdkafka-dev; fi @@ -19,7 +21,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ set -eux; \ if echo "$GO_TAGS" | grep -qw kafka; then CGO=1; TAGS="musl kafka"; else CGO=0; TAGS=""; fi; \ - CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w" -o /out/consumer ./cmd/consumer + CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/consumer ./cmd/consumer # Runtime stage FROM alpine:3.23 diff --git a/deploy/docker/forms.Dockerfile b/deploy/docker/forms.Dockerfile index 2e8d0fc9..59dcf10e 100644 --- a/deploy/docker/forms.Dockerfile +++ b/deploy/docker/forms.Dockerfile @@ -9,6 +9,8 @@ # $TARGETARCH (no QEMU). FROM --platform=$BUILDPLATFORM node:22-alpine AS appbuilder WORKDIR /app +# No TTY in a build: CI=true makes pnpm reinstall instead of prompting. +ENV CI=true RUN corepack enable && corepack prepare pnpm@11.9.0 --activate COPY forms/package.json forms/pnpm-lock.yaml forms/pnpm-workspace.yaml ./ RUN pnpm install --frozen-lockfile @@ -18,6 +20,8 @@ RUN pnpm build FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder ARG TARGETOS TARGETARCH +# Build identity shown in the admin panel; see internal/version. +ARG VERSION="" COMMIT="" BUILT_AT="" RUN apk add --no-cache git ca-certificates WORKDIR /app @@ -27,7 +31,7 @@ RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY . . RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w" -o /out/forms ./cmd/forms + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/forms ./cmd/forms # Runtime stage FROM alpine:3.23 diff --git a/deploy/docker/updater.Dockerfile b/deploy/docker/updater.Dockerfile new file mode 100644 index 00000000..dd278997 --- /dev/null +++ b/deploy/docker/updater.Dockerfile @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1.7 +# +# The updater is the host-side agent behind "Update and restart" in the admin +# panel: it pulls the checkout, rebuilds the images and recreates the +# containers. It needs git and the docker CLI with the compose plugin, and the +# docker socket mounted at runtime (see the updater service in +# docker-compose.yml). Builder runs on $BUILDPLATFORM and cross-compiles. +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder + +ARG TARGETOS TARGETARCH +ARG VERSION="" COMMIT="" BUILT_AT="" +RUN apk add --no-cache git ca-certificates + +WORKDIR /app +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/updater ./cmd/updater + +# Runtime: the official docker CLI image already carries the compose plugin. +FROM docker:27-cli + +RUN apk add --no-cache git ca-certificates tzdata && mkdir -p /var/lib/warmbly-updater + +COPY --from=builder /out/updater /app/updater + +# Runs as root on purpose: it holds the docker socket, and chowns what git +# wrote back to the checkout's owner after every update. +EXPOSE 8095 + +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \ + CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:8095/health || exit 1 + +ENTRYPOINT ["/app/updater"] diff --git a/deploy/docker/worker.Dockerfile b/deploy/docker/worker.Dockerfile index bf3135ff..5641f9c7 100644 --- a/deploy/docker/worker.Dockerfile +++ b/deploy/docker/worker.Dockerfile @@ -7,6 +7,8 @@ FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder ARG GO_TAGS="" ARG TARGETOS TARGETARCH +# Build identity shown in the admin panel; see internal/version. +ARG VERSION="" COMMIT="" BUILT_AT="" RUN apk add --no-cache git ca-certificates && \ if echo "$GO_TAGS" | grep -qw kafka; then apk add --no-cache gcc musl-dev librdkafka-dev; fi @@ -19,7 +21,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ set -eux; \ if echo "$GO_TAGS" | grep -qw kafka; then CGO=1; TAGS="musl kafka"; else CGO=0; TAGS=""; fi; \ - CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w" -o /out/worker ./cmd/worker + CGO_ENABLED=$CGO GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags "$TAGS" -ldflags="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" -o /out/worker ./cmd/worker # Runtime stage FROM alpine:3.23 diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md index fcfc0126..54df5e45 100644 --- a/deploy/systemd/README.md +++ b/deploy/systemd/README.md @@ -12,6 +12,7 @@ them is [Deploying without Docker](https://docs.warmbly.com/development/bare-met | `warmbly-tracking.service` | `/opt/warmbly/bin/tracking` | `/etc/warmbly/warmbly.env` | | `warmbly-realtime.service` | `/opt/warmbly/realtime/bin/realtime start` | `/etc/warmbly/warmbly.env` | | `warmbly-worker.service` | `/opt/warmbly/bin/worker` | `/etc/warmbly/worker.env` | +| `warmbly-updater.service` | `/opt/warmbly/bin/updater` | `/etc/warmbly/updater.env` | Every unit runs as the unprivileged `warmbly` user, may write only under `/var/lib/warmbly` (blob storage), and restarts on failure. Copy them to @@ -21,3 +22,13 @@ Every unit runs as the unprivileged `warmbly` user, may write only under `deploy/config/env.example` is the template for `warmbly.env`; the worker env file is what `POST /api/v1/workers/enroll` returns for an enrollment token. + +`warmbly-updater.service` is the exception to the unprivileged rule: it runs as +the user who owns the checkout (`deploy` in the unit; change it) and drives +`scripts/upgrade-bare-metal.sh`, which builds unprivileged and then runs +`warmbly-install-release.sh` (installed root-owned at +`/usr/local/sbin/warmbly-install-release`) through sudo. That installer takes no +arguments, touches only fixed paths and refuses symlinks, and is the single +command to allow in sudoers. `updater.env` holds only `UPDATER_TOKEN` (the +backend's `INTERNAL_API_TOKEN`). See +[Updates](https://docs.warmbly.com/development/updates/). diff --git a/deploy/systemd/warmbly-install-release.sh b/deploy/systemd/warmbly-install-release.sh new file mode 100755 index 00000000..9d195e59 --- /dev/null +++ b/deploy/systemd/warmbly-install-release.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# +# The privileged half of a bare-metal upgrade: install the artifacts that +# scripts/upgrade-bare-metal.sh built under /opt/warmbly/src/out and friends, +# then restart the units, backend first. It takes no arguments and touches only +# fixed paths, so it is the one command the checkout's owner may run through +# sudo without a password. Install it root-owned and not writable by that user: +# +# sudo install -o root -g root -m 0755 deploy/systemd/warmbly-install-release.sh /usr/local/sbin/warmbly-install-release +# deploy ALL=(root) NOPASSWD: /usr/local/sbin/warmbly-install-release # visudo +# +# A symlink under the build output is refused: a caller who owns the checkout +# must not be able to make root read or install a file from somewhere else. +set -euo pipefail + +SRC=/opt/warmbly/src +PREFIX=/opt/warmbly +HEALTH=http://127.0.0.1:8080/health + +[[ "$(id -u)" -eq 0 ]] || { echo "run through sudo" >&2; exit 1; } +[[ $# -eq 0 ]] || { echo "takes no arguments" >&2; exit 2; } + +log() { printf '==> %s\n' "$*"; } +regular() { [[ -f "$1" && ! -L "$1" ]]; } +# A directory qualifies only when nothing inside it is a symlink either: the +# checkout's owner must not be able to point root at a file elsewhere. +plain_dir() { + [[ -d "$1" && ! -L "$1" ]] || return 1 + if find "$1" -type l -print -quit | grep -q .; then + log "refusing $1: it contains a symlink" + return 1 + fi +} +has_unit() { systemctl list-unit-files "warmbly-$1.service" --no-legend 2>/dev/null | grep -q .; } + +log "installing binaries" +for bin in backend forms consumer worker migrate warmblyctl updater; do + f="$SRC/out/$bin" + if regular "$f"; then + install -o root -g root -m 0755 "$f" "$PREFIX/bin/$bin" + fi +done +ln -sf "$PREFIX/bin/warmblyctl" /usr/local/bin/warmblyctl + +if has_unit tracking && regular "$SRC/tracking/target/release/tracking"; then + install -o root -g root -m 0755 "$SRC/tracking/target/release/tracking" "$PREFIX/bin/tracking" +fi + +if has_unit realtime && plain_dir "$SRC/realtime/_build/prod/rel/realtime"; then + log "installing realtime" + rm -rf "$PREFIX/realtime" + cp -r --no-dereference "$SRC/realtime/_build/prod/rel/realtime" "$PREFIX/realtime" + chown -R warmbly:warmbly "$PREFIX/realtime" +fi + +# The runtime config.js is written by hand on a bare-metal install and a +# rebuilt dist/ would drop it, so it is kept across the copy. +for app in web admin; do + if plain_dir "$SRC/$app/dist" && plain_dir "$PREFIX/$app"; then + log "installing $app" + cfg="$(mktemp)" + [[ -f "$PREFIX/$app/config.js" ]] && cp "$PREFIX/$app/config.js" "$cfg" + rm -rf "$PREFIX/$app" + cp -r --no-dereference "$SRC/$app/dist" "$PREFIX/$app" + [[ -s "$cfg" ]] && cp --remove-destination "$cfg" "$PREFIX/$app/config.js" + rm -f "$cfg" + chown -R root:root "$PREFIX/$app" + chmod -R a+rX "$PREFIX/$app" + fi +done +if plain_dir "$SRC/forms/dist" && plain_dir "$PREFIX/forms"; then + log "installing forms" + rm -rf "$PREFIX/forms/dist" + cp -r --no-dereference "$SRC/forms/dist" "$PREFIX/forms/dist" + chown -R root:root "$PREFIX/forms/dist" + chmod -R a+rX "$PREFIX/forms/dist" +fi + +log "restarting backend" +systemctl restart warmbly-backend +healthy=0 +deadline=$((SECONDS + 120)) +while [[ $SECONDS -lt $deadline ]]; do + if curl -fsS --connect-timeout 2 --max-time 5 "$HEALTH" >/dev/null 2>&1; then healthy=1; break; fi + sleep 2 +done +if [[ "$healthy" -ne 1 ]]; then + log "the backend did not answer at $HEALTH within two minutes; the other services were not restarted" + exit 1 +fi + +rest=() +for svc in forms consumer tracking realtime worker; do + has_unit "$svc" && rest+=("warmbly-$svc") +done +if [[ ${#rest[@]} -gt 0 ]]; then + log "restarting ${rest[*]}" + systemctl restart "${rest[@]}" +fi +log "installed" diff --git a/deploy/systemd/warmbly-updater.service b/deploy/systemd/warmbly-updater.service new file mode 100644 index 00000000..766b6883 --- /dev/null +++ b/deploy/systemd/warmbly-updater.service @@ -0,0 +1,28 @@ +[Unit] +Description=Warmbly updater (one-click update from the admin panel) +Documentation=https://docs.warmbly.com/development/updates/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# Runs as the user who owns the checkout, so git writes stay theirs. The +# upgrade script builds unprivileged and then runs the fixed-path installer +# (/usr/local/sbin/warmbly-install-release, root-owned) through sudo; that +# installer is the only command to allow in sudoers. See the docs. +User=deploy +Group=deploy +EnvironmentFile=/etc/warmbly/updater.env +WorkingDirectory=/opt/warmbly/src +Environment=UPDATER_MODE=command +Environment=UPDATER_REPO_DIR=/opt/warmbly/src +Environment=UPDATER_COMMAND=/opt/warmbly/src/scripts/upgrade-bare-metal.sh +Environment=UPDATER_BACKEND_HEALTH_URL=http://127.0.0.1:8080/health +Environment=UPDATER_STATE_DIR=/var/lib/warmbly-updater +Environment=UPDATER_ADDR=127.0.0.1:8095 +ExecStart=/opt/warmbly/bin/updater +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/docker-compose.yml b/docker-compose.yml index 12f9ad9f..f678ae1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,19 @@ # # To run on Kafka + S3 instead, set the provider vars above and rebuild the Go # images with --build-arg GO_TAGS=kafka (tracking: --build-arg CARGO_FEATURES=kafka). +# +# EVERY APP SERVICE CARRIES BOTH image: AND build:, so this one file serves both +# installs. `docker compose pull && docker compose up -d` runs the published +# release images and compiles nothing; `docker compose up --build` still builds +# from this checkout. Pin a release with WARMBLY_TAG=v1.4.2 in .env, or point +# WARMBLY_IMAGE_PREFIX at your own registry. +# +# EVERY STORE'S VOLUME SOURCE IS A VARIABLE. Compose reads a source starting +# with / or . as a bind mount and anything else as a named volume, so +# WARMBLY_PG_DATA=/mnt/ssd/warmbly/postgres puts the database on a path you can +# rsync and snapshot, while the default keeps the named volume. The full set is +# WARMBLY_PG_DATA, WARMBLY_REDIS_DATA, WARMBLY_NATS_DATA, WARMBLY_BLOBS, +# WARMBLY_WORKER_STATE and WARMBLY_UPDATER_STATE. # Everything below is overridable from a single .env next to this file. The # defaults are the no-cloud stack, so an empty .env still boots; set only what @@ -216,7 +229,7 @@ services: POSTGRES_DB: warmbly_dev ports: ["15432:5432"] volumes: - - postgres_data:/var/lib/postgresql/data + - ${WARMBLY_PG_DATA:-postgres_data}:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U warmbly"] interval: 5s @@ -228,7 +241,7 @@ services: image: redis:7-alpine ports: ["16379:6379"] volumes: - - redis_data:/data + - ${WARMBLY_REDIS_DATA:-redis_data}:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -242,7 +255,7 @@ services: command: ["-js", "-sd", "/data", "-m", "8222"] ports: ["4222:4222", "8222:8222"] volumes: - - nats_data:/data + - ${WARMBLY_NATS_DATA:-nats_data}:/data healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8222/healthz"] interval: 5s @@ -295,9 +308,14 @@ services: backend: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/backend:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/backend.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} ports: ["8080:8080"] environment: <<: *selfhost-env @@ -340,8 +358,21 @@ services: WEBAUTHN_RP_ORIGINS: ${WEBAUTHN_RP_ORIGINS:-} # Worker image the orchestrator installs on remote machines. WORKER_IMAGE: ${WORKER_IMAGE:-} + # Update indicator: polls GitHub Releases and shows a newer version in the + # admin panel's top bar. Applying it goes through the updater service + # below. The address is always set because a profile cannot be tested + # here; with the profile off the host does not resolve and the backend + # reports the updater as not running (report-only), never as broken. + # UPDATER_URL=none in .env turns the button off explicitly. + UPDATE_CHECK_ENABLED: ${UPDATE_CHECK_ENABLED:-true} + UPDATE_CHECK_INTERVAL: ${UPDATE_CHECK_INTERVAL:-30m} + UPDATE_CHANNEL: ${UPDATE_CHANNEL:-stable} + RELEASES_GITHUB_REPO: ${RELEASES_GITHUB_REPO:-warmbly/warmbly} + RELEASES_GITHUB_TOKEN: ${RELEASES_GITHUB_TOKEN:-} + UPDATER_URL: ${UPDATER_URL:-http://updater:8095} + UPDATER_TOKEN: ${UPDATER_TOKEN:-${INTERNAL_API_TOKEN:-local-dev-internal-token}} volumes: - - blobs:/data/blobs + - ${WARMBLY_BLOBS:-blobs}:/data/blobs depends_on: postgres: { condition: service_healthy } redis: { condition: service_healthy } @@ -357,9 +388,14 @@ services: # database; the backend's internal API is its only dependency. forms: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/forms:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/forms.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} ports: ["${FORMS_PORT:-8090}:8090"] environment: GIN_MODE: release @@ -383,14 +419,22 @@ services: consumer: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/consumer:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/consumer.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} environment: <<: *selfhost-env ENCRYPTED_KEYS_PROVIDER: postgres + # Optional, as on the backend: turns an open or click's network into a + # country and city on the contact's timeline. + GEODB_PATH: ${GEODB_PATH:-/app/data/GeoLite2-City.mmdb} volumes: - - blobs:/data/blobs + - ${WARMBLY_BLOBS:-blobs}:/data/blobs depends_on: backend: { condition: service_healthy } nats: { condition: service_healthy } @@ -400,9 +444,14 @@ services: # belong to the mail provider, so more workers = more parallelism, not more IPs. worker: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/worker:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/worker.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} environment: <<: *selfhost-env # Dovecot in the sandbox profile uses a self-signed cert. Set @@ -423,14 +472,15 @@ services: BOX_OUTLOOK_CLIENT_ID: ${BOX_OUTLOOK_CLIENT_ID:-} BOX_OUTLOOK_CLIENT_SECRET: ${BOX_OUTLOOK_CLIENT_SECRET:-} volumes: - - blobs:/data/blobs - - worker_state:/data/state + - ${WARMBLY_BLOBS:-blobs}:/data/blobs + - ${WARMBLY_WORKER_STATE:-worker_state}:/data/state depends_on: backend: { condition: service_healthy } nats: { condition: service_healthy } tracking: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/tracking:${WARMBLY_TAG:-prod} build: context: ./tracking dockerfile: Dockerfile @@ -451,6 +501,7 @@ services: TRACKING_PAGEHIT_RATE_LIMIT_PER_MIN: ${TRACKING_PAGEHIT_RATE_LIMIT_PER_MIN:-} TRACKING_TRUSTED_PROXIES: ${TRACKING_TRUSTED_PROXIES:-} TRACKING_CLIENT_IP_HEADER: ${TRACKING_CLIENT_IP_HEADER:-} + TRACKING_IP_HASH_KEY: ${TRACKING_IP_HASH_KEY:-} SENTRY_DSN: ${SENTRY_DSN:-} # Overridable so `make dev` (native backend) can point at # host.docker.internal:8080 while `docker compose up` uses the container. @@ -463,6 +514,7 @@ services: realtime: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/realtime:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/realtime.Dockerfile @@ -497,6 +549,7 @@ services: # use `make app` / docker-compose.dev.yml instead. web: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/web:${WARMBLY_TAG:-prod} build: context: ./web dockerfile: Dockerfile @@ -513,6 +566,7 @@ services: admin: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/admin:${WARMBLY_TAG:-prod} build: context: ./admin dockerfile: Dockerfile @@ -525,13 +579,56 @@ services: depends_on: backend: { condition: service_healthy } + # One-click updates from the admin panel (the version pill in the top bar). + # Pulls this checkout, rebuilds the images and recreates the containers when + # the backend asks, then waits for the backend to answer again. It holds the + # docker socket, which is root on this host, so it only runs under the + # "updater" profile: `make up` enables it, and COMPOSE_PROFILES=updater in + # .env does the same for a plain `docker compose up`. Leave the profile off + # to update by hand with `git pull && make up`. + updater: + restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/updater:${WARMBLY_TAG:-prod} + build: + context: . + dockerfile: deploy/docker/updater.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} + environment: + UPDATER_TOKEN: ${UPDATER_TOKEN:-${INTERNAL_API_TOKEN:-local-dev-internal-token}} + # The checkout is mounted at its host path so the compose file's relative + # paths resolve identically inside and outside the container. + UPDATER_REPO_DIR: ${WARMBLY_REPO_DIR:-${PWD}} + UPDATER_COMPOSE_PROJECT: warmbly + UPDATER_BACKEND_HEALTH_URL: http://backend:8080/health + UPDATER_FETCH_INTERVAL: ${UPDATER_FETCH_INTERVAL:-30m} + UPDATER_PRUNE: ${UPDATER_PRUNE:-true} + UPDATER_ALLOW_DIRTY: ${UPDATER_ALLOW_DIRTY:-false} + working_dir: ${WARMBLY_REPO_DIR:-${PWD}} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${WARMBLY_REPO_DIR:-${PWD}}:${WARMBLY_REPO_DIR:-${PWD}} + - ${WARMBLY_UPDATER_STATE:-updater_state}:/var/lib/warmbly-updater + profiles: ["updater"] + # ─── one-shots ──────────────────────────────────────────────────────── # Optional demo seed. Run explicitly: docker compose --profile seed run --rm seed + # Build-only on purpose, unlike every service above. The seed is not a + # published image (it plants a known super-admin credential, so it must never + # be one `docker compose pull` away on a real host), and sharing the backend's + # image tag would mean `make seed-demo` retagged the image the running backend + # was created from. seed: build: context: . dockerfile: deploy/docker/backend.Dockerfile + args: + VERSION: ${WARMBLY_BUILD_VERSION:-} + COMMIT: ${WARMBLY_BUILD_COMMIT:-} + BUILT_AT: ${WARMBLY_BUILD_TIME:-} entrypoint: ["/app/seed"] environment: <<: *selfhost-env @@ -543,9 +640,13 @@ services: backend: { condition: service_healthy } profiles: ["seed"] +# Declared for the defaults. A store pointed at a host path by its +# WARMBLY_*_DATA variable simply stops using the named volume below; an unused +# declaration costs nothing and keeps switching back a one-line .env edit. volumes: postgres_data: redis_data: nats_data: blobs: worker_state: + updater_state: diff --git a/docs/content/docs/api/authentication.mdx b/docs/content/docs/api/authentication.mdx index 72c07086..def0a1fb 100644 --- a/docs/content/docs/api/authentication.mdx +++ b/docs/content/docs/api/authentication.mdx @@ -17,6 +17,28 @@ The `wmbly_` prefix identifies the key as a Warmbly key. The remaining 43 charac Keys are stored as a SHA-256 hash; the plaintext is shown exactly once on creation. To help you spot a key in the dashboard without exposing the secret, we store the first 8 characters (`key_prefix`, e.g. `wmbly_ab`) and the last 4 characters (`key_suffix`, e.g. `wxyz`). Render them as `wmbly_ab…wxyz`. +## Three ways to get a key + +1. **The dashboard.** Settings > API keys, pick the scopes, copy the secret. This is the right path for a key a server will use. +2. **The CLI.** [`warmbly auth login`](/api/cli/) opens a browser approval and mints a key named for the machine that asked, then stores it at 0600. This is the right path for a key you will use yourself, and it is the only one that does not involve pasting a secret into a shell. +3. **The API.** `POST /v1/api-keys` with a key that carries `API_KEYS`. The secret is in that response and nowhere else. + +All three produce the same thing: a `wmbly_` key with a scope bitmask, listed under Settings > API keys, revocable there. + +### The CLI device flow + +`warmbly auth login` uses a device-code handshake, so the terminal never handles your password and the browser never handles the key: + +1. The CLI calls `POST /v1/auth/cli/code` with the scopes it wants and the machine's hostname. It gets back a `device_code` it keeps, a `user_code` it prints, and a `verification_uri_complete` it opens. +2. You approve at `app.warmbly.com/cli`, choosing which workspace the key belongs to. The approval is what mints the key, so it requires the `MANAGE_API_KEYS` organization permission. +3. The CLI polls `POST /v1/auth/cli/poll` and receives the key exactly once, on the first poll after approval. + +Codes expire after ten minutes, both halves are per-IP rate limited, and the `device_code` is stored hashed. Anything else can drive the same flow: it is two public endpoints and a browser. + +### Ending a key + +`DELETE /v1/api-keys/:id` revokes any key in the workspace and needs the `API_KEYS` scope. `DELETE /v1/api-keys/self` revokes the key the call was made with and needs no scope at all, so a narrowly scoped credential can always end itself. This is what `warmbly auth logout` uses. + ## Using your API key Include your API key in the `Authorization` header of every request: diff --git a/docs/content/docs/api/cli.mdx b/docs/content/docs/api/cli.mdx new file mode 100644 index 00000000..0df9bacc --- /dev/null +++ b/docs/content/docs/api/cli.mdx @@ -0,0 +1,382 @@ +--- +title: CLI +description: The warmbly command line interface. Sign in once, then drive campaigns, contacts, mailboxes and the inbox from your terminal, from CI, or from an agent. +--- + +`warmbly` is the command line interface to Warmbly. It signs in as you, holds one credential per host, and speaks only the public REST API, so it works against the hosted service and against any self-hosted instance you can reach. + +```bash +warmbly auth login +warmbly campaign list +warmbly inbox list --unseen +warmbly api "/campaigns?limit=10" +``` + + +There are two CLIs and they answer different questions. `warmbly` is the one you install on your machine to use the product. [`warmblyctl`](/development/warmblyctl/) is the operator's tool: it talks to Postgres directly, runs inside the backend container, and exists for recovery, accounts, health and backups. If you are asking "what is wrong with this install", that is the one you want. + + +## Install + +Pick one. All of them produce the same single binary. + +**macOS and Linux** + +```bash +curl -fsSL https://warmbly.com/cli.sh | sh +``` + +Installs to `~/.local/bin`, so it needs no root and no toolchain. The script verifies what it downloaded against the published checksum and installs nothing if they disagree, writes shell completions, and tells you the one line to add to your profile if that directory is not already on your PATH. + +**Windows** + +```powershell +irm https://warmbly.com/cli.ps1 | iex +``` + +Installs to `%LOCALAPPDATA%\Warmbly\bin` and adds it to your user PATH. No admin rights. + +**Homebrew** (macOS and Linux) + +```bash +brew install warmbly/tap/warmbly +``` + +**Scoop** (Windows) + +```powershell +scoop bucket add warmbly https://github.com/warmbly/homebrew-tap +scoop install warmbly +``` + +**Docker**, for CI or anywhere installing a binary is more trouble than pulling one: + +```bash +docker run --rm -e WARMBLY_TOKEN ghcr.io/warmbly/warmbly/cli campaign list +``` + +A container has no browser, so sign in with `WARMBLY_TOKEN` rather than `auth login`. + +**From source**, if you have Go: + +```bash +go install github.com/warmbly/warmbly/cmd/cli@latest +mv "$(go env GOPATH)/bin/cli" "$(go env GOPATH)/bin/warmbly" +``` + +The package directory is `cmd/cli`, so `go install` names the binary `cli`. Rename it, or use one of the channels above. + +**Already have a Warmbly instance?** The binary ships inside the backend image: + +```bash +docker compose -p warmbly exec backend warmbly --help +``` + +### Keeping it current + +```bash +warmbly version +warmbly upgrade +``` + +`upgrade` replaces the binary in place, verifying the download against the release checksums first. When the CLI came from Homebrew or Scoop it says which command to run instead, rather than overwriting a file a package manager owns. The CLI also checks for a new release once a day and mentions it in one line; `WARMBLY_NO_UPDATE_CHECK=1` turns that off, and it never runs in CI or when output is piped. + +### Installer flags + +The install script takes flags after `--`, and every one of them is also an environment variable, so the same install runs from Ansible, cloud-init or a Dockerfile: + +```bash +curl -fsSL https://warmbly.com/cli.sh | sh -s -- --dir /usr/local/bin +curl -fsSL https://warmbly.com/cli.sh | sh -s -- --version v1.4.0 +curl -fsSL https://warmbly.com/cli.sh | sh -s -- --no-modify-path --no-completions +curl -fsSL https://warmbly.com/cli.sh | sh -s -- --dry-run +curl -fsSL https://warmbly.com/cli.sh | sh -s -- --uninstall +``` + +| Flag | Variable | What it does | +|---|---|---| +| `--dir PATH` | `WARMBLY_INSTALL_DIR` | Where the binary goes. Default `~/.local/bin` | +| `--version TAG` | `WARMBLY_CLI_VERSION` | Pin a release instead of taking the newest | +| `--base-url URL` | `WARMBLY_CLI_BASE_URL` | Download from an internal mirror of the release assets | +| `--no-modify-path` | `WARMBLY_NO_MODIFY_PATH` | Never touch a shell profile | +| `--no-completions` | `WARMBLY_NO_COMPLETIONS` | Skip the completion files | +| `--dry-run` | | Print what would happen, change nothing | +| `--uninstall` | | Remove the binary and completions, keep your sign-ins | + +`--help` lists them in the terminal. `sh -s -- --help` works through the pipe. + +### Reading it before you run it + +Piping a script into a shell is worth being careful about, which is why the checksum is published next to it: + +```bash +curl -fsSLO https://warmbly.com/cli.sh +curl -fsSLO https://warmbly.com/cli.sh.sha256 +sha256sum -c cli.sh.sha256 +less cli.sh && sh cli.sh +``` + +Every release archive is covered by `checksums.txt` on the release, and the installer verifies the archive against it before unpacking. If the two disagree it installs nothing and says so. + +### Completions + +The installer writes them for your shell. To do it by hand: + +```bash +warmbly completion bash > /etc/bash_completion.d/warmbly +warmbly completion zsh > "${fpath[1]}/_warmbly" +warmbly completion fish > ~/.config/fish/completions/warmbly.fish +``` + +## Signing in + +```bash +warmbly auth login +``` + +It asks two questions: which instance (the hosted service, or a hostname of your own), and how (a browser approval, or pasting a key you already have). + +The browser path shows an eight character code, opens `app.warmbly.com/cli`, and waits. You approve there, choosing which workspace the CLI is signing in to. The approval creates **one API key named for your machine**, which appears under Settings > API keys and is revocable there or with `warmbly auth logout`. The terminal never handles your password, and the browser never handles the key. + +``` + Your code: K4TM-9RQD + Approve at: https://app.warmbly.com/cli?code=K4TM-9RQD + +… Waiting for approval (the code expires in 10 minutes) +✓ Signed in to warmbly.com as jane@example.com + Workspace Acme +``` + +Non-interactive forms, for a script or an agent: + +```bash +warmbly auth login --hostname warmbly.acme.com --web +echo "$WARMBLY_KEY" | warmbly auth login --with-token +warmbly auth login --scopes read-only +``` + +### Scopes + +The CLI asks for full access by default, and the approval screen lists exactly what that means before you agree. Narrow it with `--scopes`: + +```bash +warmbly auth login --scopes read-only +warmbly auth login --scopes read_campaigns,read_contacts,send_campaigns +``` + +A key's scopes are fixed once created, so widening them means a new key. `warmbly auth refresh --scopes full` runs the sign-in again and revokes the key it replaces, which is why refreshing does not leave a trail of keys behind. + +Scope names are the ones in the [permissions reference](/api/permissions/), in either case, with `full` and `read-only` as shorthands. + +### Several instances + +The CLI holds one credential per host. The `*` in `auth status` is the one commands use. + +```bash +warmbly auth login --hostname warmbly.acme.com +warmbly auth status +warmbly auth switch warmbly.acme.com +warmbly campaign list --host warmbly.com # one command, other host +``` + +`warmbly auth status` also tells you where the token came from, which is the answer nine times out of ten when a command fails unexpectedly: + +``` +* warmbly.com + ✓ signed in as jane@example.com + - workspace: Acme + - scopes: all 24 + - api: https://api.warmbly.com + - token from: /home/jane/.config/warmbly/hosts.yml + - token: wmbly_ab********wxyz +``` + +### In CI + +Set `WARMBLY_TOKEN` and skip the login entirely. It overrides the file and is never written to it. + +```yaml +env: + WARMBLY_TOKEN: ${{ secrets.WARMBLY_TOKEN }} + WARMBLY_HOST: warmbly.com # omit for the hosted service +run: warmbly campaign list --json +``` + +`warmbly auth token` prints the active token and nothing else, for handing to another tool. + +## Output + +Commands print a table on a terminal and JSON everywhere else, so the same command is readable by a person and parseable by a pipe. + +```bash +warmbly campaign list # a table +warmbly campaign list > campaigns.json # JSON, no flag needed +warmbly campaign list --json | jq '.data[].name' +warmbly campaign list --fields name,status +warmbly campaign list --template '{{range .data}}{{.name}}{{"\n"}}{{end}}' +``` + +`--all` walks the cursor on any list command and merges every page into one response. + +## Sending real mail + +Anything that puts mail on the wire asks first, and refuses rather than sending when there is no terminal to ask on: + +``` +$ warmbly campaign start 6f1c… +! `warmbly campaign start` sends real mail. Continue? [y/N] +``` + +`--yes` is the only way past it, which makes it the flag to grep for in a script review. The commands that behave this way are `campaign start`, `campaign test`, `mailbox send`, `inbox reply`, `inbox compose` and `inbox approve-draft`. + +## Commands + +Run `warmbly --help` for the flags, and `warmbly --help` for one command's arguments. + +| Command | What it covers | +|---|---| +| `auth` | login, logout, status, token, switch, refresh | +| `status` | one screen: mailboxes needing attention, what is sending, what is unread | +| `browse` | open the dashboard, or one record, in a browser | +| `campaign` | list, view, create, edit, steps, senders, segments, preflight, test, start, stop, logs | +| `contact` | list, view, create, edit, delete, lookup, timeline, notes, import, export, verify | +| `mailbox` | list, view, edit, health checks, sync state, sending behaviour, warmup, hold, send | +| `inbox` | list, view, threads, read, reply, compose, drafts, scheduled sends, snoozes | +| `suppression` | the addresses and domains that get no campaign mail | +| `segment` | live audiences and their conditions | +| `template` | reply templates | +| `automation` | automations, their runs and test firing | +| `form` | lead capture forms, submissions and stats | +| `deal`, `pipeline`, `task` | the CRM | +| `analytics` | dashboard, deliverability, warmup, per-mailbox and per-campaign numbers | +| `audit` | the workspace's audit trail | +| `advisor` | recommendations, and applying or dismissing them | +| `webhook` | endpoints, deliveries, redelivery, event types | +| `key` | API keys, their scopes and their usage | +| `oauth-app` | OAuth applications you publish | +| `integration` | third-party connections | +| `org` | which workspace this credential belongs to | +| `team` | named groups of members, for CRM ownership and routing | +| `settings` | workspace-wide outreach and suppression settings | +| `warmup-routing` | which mailboxes warm with which | +| `tool` | the AI tool registry, listed and called | +| `events` | the live event stream | +| `api` | any endpoint at all | +| `upgrade` | replace this binary with the newest release | +| `config`, `alias`, `completion`, `version` | the CLI itself | + +### Examples + +```bash +# Create a campaign, add a step, check it, start it +warmbly campaign create --name "Q3 outbound" --daily-limit 40 +warmbly campaign add-step CAMPAIGN_ID --subject "Quick question" --wait-after 0 +warmbly campaign preflight CAMPAIGN_ID +warmbly campaign start CAMPAIGN_ID + +# Mailbox health across the workspace +warmbly mailbox list +warmbly mailbox check MAILBOX_ID +warmbly mailbox edit MAILBOX_ID --daily-limit 40 + +# The inbox +warmbly inbox list --unseen --limit 20 +warmbly inbox thread --email-id EMAIL_ID + +# Contacts in and out +warmbly contact create --email jane@example.com --first-name Jane --company Acme +warmbly contact list --all --json > contacts.json +``` + +## What needs the dashboard + +Three things the CLI deliberately does not do, because the API does not let a key do them: + +- **Connecting a mailbox.** It needs OAuth consent or a credential form in a browser. `warmbly browse mailboxes` opens the right page. +- **Workspace administration.** Members, roles, invitations, workspace exports and the danger zone are session-only on the API: they depend on a human-bound session and refuse an API key. `warmbly browse members` and `warmbly browse settings` open them. +- **Billing.** Plans, checkout and credits are session-only for the same reason. `warmbly browse billing`. + +`warmbly org view` still shows which workspace you are in, and `warmbly team`, `warmbly settings` and `warmbly audit` cover the workspace surface a key can reach. + +## Watching events + +`warmbly events tail` streams the developer WebSocket into your terminal: the same events the dashboard runs on, printed as they happen. It is the fastest way to see whether an integration is receiving what you think it is, without standing up a public endpoint first. + +```bash +warmbly events tail +warmbly events tail --intent EMAIL --intent CAMPAIGN +warmbly events tail --json | jq 'select(.event_type == "EMAIL_REPLIED")' +``` + +It needs a key with `REALTIME_SUBSCRIBE`; `warmbly auth refresh --scopes full` gets one. The stream, its intents and its event types are documented under [Realtime](/api/realtime/). + +## Calling the API directly + +`warmbly api` reaches every endpoint, including the ones with no command of their own. Paths are relative to `/v1`. + +```bash +warmbly api /me +warmbly api "/campaigns?limit=10" --paginate +warmbly api /contacts -f email=jane@example.com -f first_name=Jane +warmbly api /campaigns/CAMPAIGN_ID -X PATCH -F daily_limit=40 +warmbly api /contacts/search -X POST --input filter.json +warmbly api /webhooks/WEBHOOK_ID -X DELETE +``` + +`-f` keeps a value a string. `-F` guesses the type, so `true`, `false`, `null` and numbers arrive as themselves, `@file` reads a value from a file, `key[sub]=v` nests and repeated `key[]=v` builds an array. `--paginate` follows the cursor, `-i` includes the status and headers, `--idempotency-key` rides the [documented header](/api/authentication/). + +## Configuration + +Two files under `~/.config/warmbly` (or `XDG_CONFIG_HOME`, or `WARMBLY_CONFIG_DIR`): + +- `hosts.yml`, one credential per host, written 0600 +- `config.yml`, preferences and aliases + +Signing in records the instance's own API and dashboard URLs alongside the credential, taken from what the instance reports, so `browse` and `events tail` work on a self-hosted layout without anyone configuring a second address. + +```bash +warmbly config list +warmbly config set output json # default to JSON even on a terminal +warmbly config set confirm always # confirm every write, not only sends +warmbly config set browser firefox +``` + +### Aliases + +```bash +warmbly alias set hot "campaign list --status active" +warmbly hot --json +``` + +Anything you type after the alias is appended, so an alias is a starting point rather than a fixed command. + +### Environment + +| Variable | What it does | +|---|---| +| `WARMBLY_TOKEN` | The API key to use. Overrides `hosts.yml` and is never written to it | +| `WARMBLY_API_KEY` | The same thing under the name `warmblyctl` uses | +| `WARMBLY_HOST` | Which signed-in host to use | +| `WARMBLY_API_URL` | The API base URL, when it is not derivable from the host | +| `WARMBLY_CONFIG_DIR` | Where the two files live | +| `WARMBLY_NO_UPDATE_CHECK` | Never check for a newer release | +| `NO_COLOR` | Turns colour off, as everywhere else | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | It worked | +| `1` | The command failed, or you declined a prompt | +| `2` | The command line was wrong, or an answer was needed with no terminal to ask on | +| `4` | Not signed in, or the credential was rejected or lacks a scope | + +Every API failure prints the response's machine-readable `code` and `request_id` to stderr, so a script can branch without reading prose. + +## See also + +- [Authentication](/api/authentication/) for how the device flow mints a key +- [Permissions](/api/permissions/) for what each scope allows +- [Endpoint scope map](/api/endpoints/) for what `warmbly api` can reach +- [Realtime](/api/realtime/) for the stream behind `warmbly events tail` +- [warmblyctl](/development/warmblyctl/) for the operator's CLI diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index 79a96663..f7108dab 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -23,6 +23,7 @@ All paths below are relative to the versioned base URL `https://api.warmbly.com/ | GET | `/emails/:id` | `READ_EMAILS` | | PATCH | `/emails/:id` | `WRITE_EMAILS` | | PATCH | `/emails/tags` | `WRITE_EMAILS` | +| GET | `/emails/allowance` | `READ_EMAILS` | | GET | `/emails/:id/track` | `READ_EMAILS` | | PATCH | `/emails/:id/track` | `WRITE_EMAILS` | | POST | `/emails/:id/track/verify` | `WRITE_EMAILS` | @@ -43,11 +44,15 @@ All paths below are relative to the versioned base URL `https://api.warmbly.com/ |--------|------|----------------| | GET | `/campaigns` | `READ_CAMPAIGNS` | | GET | `/campaigns-overview` | `READ_CAMPAIGNS` | +| POST | `/campaigns-estimate` | `READ_CAMPAIGNS` | | POST | `/campaigns` | `WRITE_CAMPAIGNS` | | GET | `/campaigns/:id` | `READ_CAMPAIGNS` | | PATCH | `/campaigns/:id` | `WRITE_CAMPAIGNS` | | DELETE | `/campaigns/:id` | `WRITE_CAMPAIGNS` | | POST | `/campaigns/:id/duplicate` | `WRITE_CAMPAIGNS` | +| GET | `/campaigns/:id/attachments` | `READ_CAMPAIGNS` | +| POST | `/campaigns/:id/attachments` | `WRITE_CAMPAIGNS` | +| DELETE | `/campaigns/:id/attachments/:attachmentId` | `WRITE_CAMPAIGNS` | | GET | `/campaigns/:id/segments` | `READ_CAMPAIGNS` | | PUT | `/campaigns/:id/segments` | `WRITE_CAMPAIGNS` | | GET | `/campaigns/:id/advanced` | `READ_CAMPAIGNS` | @@ -201,6 +206,8 @@ The `/unibox/drafts` endpoints hold autosaved compose drafts, scoped to the call `POST /emails/:id/hold` keeps a mailbox out of campaign sending until `POST /emails/:id/release` puts it back; warmup is unaffected and the automatic rest logic never releases a hold. `release` is also the manual exit for a mailbox that is `resting` automatically. Both are bodyless and idempotent, so they take no `Idempotency-Key`. See [holding a mailbox yourself](/guides/mailboxes/#holding-a-mailbox-yourself). +`GET /emails/allowance` reports how many mailboxes the workspace holds (`used`), how many it may hold (`allowance`, `null` for unlimited), `remaining`, and the `basis` of the number: `fair_use` (the plan's daily sends divided by `sends_per_mailbox`), `plan`, `override` (an approved request), `free`, or `unlimited`. `pending_request` is the open limit-increase request for mailboxes, if any. Every connect path refuses with `mailbox_allowance_reached` once `remaining` is `0`. See [mailbox allowance](/guides/mailboxes/#mailbox-allowance). + `PATCH /emails/:id` accepts `save_to_sent` (boolean) on SMTP/IMAP mailboxes: when true, which is the default, the worker files a copy of each outbound message in the mailbox's Sent folder. It has no effect on Gmail and Outlook mailboxes, whose APIs file their own copy. See [keeping a copy of sent mail](/guides/mailboxes/#keeping-a-copy-of-sent-mail). `GET /unibox` and `GET /unibox/thread` return message previews: each row carries `snippet`, a one-line summary, not the message body. Read a full message with `GET /unibox/:id`, which returns `body_plain` plus `body_html`. The HTML is sanitized before it leaves the API (scripts, event handlers, embedded frames, and unsafe URL schemes are removed), so it is safe to render, and links carry `target="_blank"` with `rel="noopener"`. `body_truncated` is `true` on the rare message whose stored body could not be read, where `body_plain` falls back to the snippet. @@ -259,6 +266,9 @@ Changing Advisor settings (`PATCH /advisor/settings`) is JWT only, alongside the | GET | `/api-keys/:id` | `API_KEYS` | | PATCH | `/api-keys/:id` | `API_KEYS` | | DELETE | `/api-keys/:id` | `API_KEYS` | +| DELETE | `/api-keys/self` | none | + +`DELETE /api-keys/self` revokes the key the request was made with, and is the one route here that needs no scope. A credential must always be able to end itself: requiring `API_KEYS` to sign out would leave a read-only key on a laptop someone is handing back live, which is what [`warmbly auth logout`](/api/cli/) promises to prevent. A JWT caller gets a `400`: there is no key in that request to end, only a session, which `POST /auth/logout` ends. ### OAuth apps @@ -283,6 +293,9 @@ Registering and managing the OAuth apps your workspace owns. The flow itself (au |--------|------|----------------| | GET/PATCH | `/outreach/settings` | `WRITE_CAMPAIGNS` | | POST | `/deliverability/events` | `WRITE_CAMPAIGNS` | +| GET | `/suppressions` | `READ_CONTACTS` | +| POST | `/suppressions` | `WRITE_CONTACTS` | +| DELETE | `/suppressions/:id` | `WRITE_CONTACTS` | | GET | `/tasks/dlq` | `SEND_CAMPAIGNS` | | POST | `/tasks/dlq/:id/replay` | `SEND_CAMPAIGNS` | | GET/POST/PATCH/DELETE | `/webhooks[/:id]` | `WEBHOOKS` | @@ -323,11 +336,14 @@ These never accept an API key. They depend on a human-bound session: billing flo - `POST /auth/login`, `/auth/login/confirm`, `/auth/register`, `/auth/register/confirm`, `/auth/refresh`, `/auth/reset-password`, `/auth/reset-password/confirm` - `GET /auth/config` (public deployment capabilities: which sign-in methods this backend has enabled, whether a login code step follows, whether signups are open, whether the instance still needs claiming) +- `GET /auth/instance` (JWT only: the running Warmbly version of a self-hosted instance and whether a newer release exists, for the dashboard's version pill; a hosted deployment answers `self_hosted: false` and nothing else) `POST /auth/register` accepts an optional `invite` field carrying an invitation token. On a deployment running `DISABLE_REGISTRATION=invite_only` it is what permits the signup, and the account is created inside the inviting organization rather than in a new one. The token must resolve to a live invitation whose email equals the submitted address, otherwise the request is refused with `invitation_invalid`. Omitting it on a closed deployment returns `registration_invite_only` or `registration_closed`. See [error codes](/api/error-codes/#registration-and-invitation-refusals). `GET /auth/config` gained two fields: `invites_required` (boolean, true when an invitation token is needed to create an account) and `docs_url` (string, the deployment's link to the accounts and access documentation, for a client to surface next to a refusal). +`GET /auth/config` also carries `websocket_url` and `app_url` (both strings, each omitted when the instance has none). They are the realtime gateway a developer client connects to and the dashboard origin a client sends someone to. Both are served here because on a self-hosted instance the host layout is whatever the operator chose, and there is no other way to discover it: the [CLI](/api/cli/) reads them for `warmbly events tail` and `warmbly browse`. + `GET /auth/config` also carries `billing_enabled` (boolean). It is `false` when the deployment runs with `BILLING_PROVIDER=none`, which is the self-host default: every feature is unlocked server-side, so the dashboard shows the workspace as self-hosted instead of on a free trial and hides the billing and referral pages. `self_hosted` alone does not imply this, because a self-hosted install may still run Stripe. - `POST /auth/setup` (first-run claim: exchanges the one-time token printed at boot for the owner account. Refused once any account exists) - `GET /auth/providers`, `POST /auth/apple`, `POST /auth/google` (native-app social sign-in) @@ -337,6 +353,7 @@ These never accept an API key. They depend on a human-bound session: billing flo - `POST /auth/logout`, `POST /auth/logout-all`, `GET /auth/me`, `PATCH /auth/me/onboarding` - `POST /auth/me/avatar`, `DELETE /auth/me/avatar` - `POST /emails/onboarding/oauth/start`, `POST /emails/onboarding/oauth/finish`, `POST /emails/onboarding/smtp-imap` +- `POST /emails/onboarding/smtp-imap/bulk` (up to `50` SMTP/IMAP rows in `accounts`, answered `200` with a per-row `status` of `connected`, `skipped` or `failed` and a `code`; rows past the workspace's [mailbox allowance](/guides/mailboxes/#mailbox-allowance) fail with `mailbox_allowance_reached` before any credential is dialled. Naturally retry-safe: an already connected mailbox is `skipped`, so it takes no `Idempotency-Key`) - `POST /emails/onboarding/oauth/reauth/:id`, `PUT /emails/onboarding/smtp-imap/:id` (reconnect an existing mailbox after a credential change; JWT permission `MANAGE_EMAILS`) - `GET /oauth/authorize/details`, `POST /oauth/authorize` (the consent flow: a human approves a third-party app) - `GET /oauth/authorized-apps`, `DELETE /oauth/authorized-apps/:id` (apps the user has authorized) @@ -347,6 +364,7 @@ These never accept an API key. They depend on a human-bound session: billing flo - All of `/organization/*` (create, switch, members, invitations, transfer ownership, avatar, danger zone) - `GET /website-tracking/settings`, `PATCH /website-tracking/settings`, `POST /website-tracking/settings/rotate-key` (the [website tracking](/guides/website-tracking/) snippet's consent mode, location precision, allowed hosts and retention; JWT permission `MANAGE_SETTINGS`. The rotate is bodyless and safe to repeat, each call issues a new key) - All of `/subscription/*` (checkout, portal, cancel, change-plan, preview-change, enterprise-inquiry, discounts, referrals, etc.) +- All of `/auth/cli/*` except the two handshake routes below (`GET /auth/cli/codes/:code`, `POST /auth/cli/codes/:code/approve`, `POST /auth/cli/codes/:code/deny`: the browser half of `warmbly auth login`, where a signed-in member reviews the code a CLI is showing and authorizes it. Approving mints an ordinary API key, so it requires the `MANAGE_API_KEYS` organization permission and is session-only: an API key must not be able to mint another one this way) - All of `/pool-link/*` and `/cloud-link/*` (the self-hosted warmup pool link: approving an instance's code, listing and unlinking instances, and on a self-hosted instance the connect flow and mailbox enrollment). `POST /pool-link/codes` and `POST /pool-link/poll` are public and per-IP rate limited: they are the device-code handshake an instance uses before it has a token, and `/pool-link/instance/*` accepts only an instance token. `/pool-link/instance/oauth/*`, `/pool-link/instance/mailboxes/:id/token`, `/pool-link/instance/workspace-mailboxes` and `/pool-link/instance/mailboxes/adopt` are the cloud-managed mailbox surface (Google and Microsoft sign-in on Warmbly's OAuth apps, brokered access tokens); their instance-side counterparts are `/cloud-link/oauth/*` and `/cloud-link/workspace-mailboxes/*` - All of `/admin/*` @@ -425,6 +443,7 @@ External MCP servers whose tools the assistant can use (see [Connect MCP tools]( ## Public - `GET /health` +- `POST /auth/cli/code`, `POST /auth/cli/poll` (the [CLI](/api/cli/) device-code handshake, per-IP rate limited. Public by necessity: the CLI has no credential until the flow completes. `POST /auth/cli/code` returns `device_code`, `user_code`, `verification_uri`, `verification_uri_complete`, `expires_in` and `interval`; polling returns `{"status":"pending"}` until a member decides, then `{"status":"approved"}` carrying the minted key exactly once, or `{"status":"denied"}`. An unknown or expired `device_code` is a `404`, so a poller cannot probe for live handshakes) - `POST /webhooks/github/releases` (HMAC-SHA256 signature) - `POST /webhook/stripe` (Stripe signature) - `POST /webhook/campaign`, `/webhook/email`, `/webhook/user-email` (Google OIDC token from Cloud Tasks) diff --git a/docs/content/docs/api/error-codes.mdx b/docs/content/docs/api/error-codes.mdx index 24b7539e..cc00b000 100644 --- a/docs/content/docs/api/error-codes.mdx +++ b/docs/content/docs/api/error-codes.mdx @@ -26,10 +26,10 @@ All errors follow this structure: | Code | Error | Description | |------|-------|-------------| -| 400 | Bad Request | Invalid request syntax or parameters | +| 400 | Bad Request | Invalid request syntax or parameters, or a quota that would be passed (`storage_limit_reached`) | | 401 | Unauthorized | Missing or invalid authentication | | 402 | Payment Required | Out of AI credits (`insufficient_credits`) | -| 403 | Forbidden | Authenticated but lacks permission | +| 403 | Forbidden | Authenticated but lacks permission, or the workspace's mailbox allowance is full (`mailbox_allowance_reached`) | | 404 | Not Found | Resource doesn't exist | | 409 | Conflict | Resource already exists | | 422 | Unprocessable | Validation failed | @@ -317,6 +317,41 @@ A `503` whose `code` is `mailbox_provider_not_configured` is not transient and r - Or connect the mailbox over SMTP and IMAP instead, which needs no configuration - Full walkthrough: [connect mailboxes](/development/deployment-guide/#connect-mailboxes) +#### `mailbox_allowance_reached` + +A `403` whose `code` is `mailbox_allowance_reached` comes from every path that connects a mailbox: `POST /emails/onboarding/oauth/start`, `POST /emails/onboarding/oauth/finish`, `POST /emails/onboarding/smtp-imap`, and per row inside `POST /emails/onboarding/smtp-imap/bulk`. It is not a permission problem: the workspace holds its whole [mailbox allowance](/guides/mailboxes/#mailbox-allowance), which on a paid plan is one mailbox for every send a day the plan includes, and `10` on a free workspace. Nothing was connected. + +```json +{ + "error": "Forbidden", + "message": "This workspace holds 15000 of its 15000 mailboxes. Request an increase, or move to a plan with more daily sends.", + "code": "mailbox_allowance_reached", + "request_id": "4bbbd1b2-8f86-47dd-8a7f-9476501ad20e" +} +``` + +**How to fix:** +- Read `GET /emails/allowance` first: `remaining` says how many connects will succeed, and `pending_request` whether an increase is already asked for +- Submit a limit-increase request for `max_email_accounts` via `POST /organization/:orgId/limit-requests`, or move to a plan with more daily sends. An approved request raises the allowance immediately; retry the connect then +- Reconnecting an existing mailbox never returns this code + +#### `storage_limit_reached` + +A `400` whose `code` is `storage_limit_reached` comes from `POST /campaigns/:id/attachments` and from a campaign duplicate that would copy attachments. The workspace's attachment storage, summed across every campaign, would pass its quota. The check and the write happen together under a per-workspace lock, so two uploads racing for the last of the quota cannot both get in. Nothing was stored. + +```json +{ + "error": "Bad Request", + "message": "Storage limit reached: 51190 MB of 51200 MB used, 12 MB to add. Remove attachments or upgrade your plan.", + "code": "storage_limit_reached", + "request_id": "4bbbd1b2-8f86-47dd-8a7f-9476501ad20e" +} +``` + +**How to fix:** +- `GET /organization/current/limits` reports `storage.used_bytes` and `storage.limit_bytes`, and `storage.over_quota` when a plan change left the workspace above the quota. Existing attachments keep sending either way +- Delete attachments you no longer need (`DELETE /campaigns/:id/attachments/:attachmentId`), or move to a paid plan for the larger quota + #### `mailbox_worker_unreachable` A `503` whose `code` is `mailbox_worker_unreachable` comes from `DELETE /emails/{id}`. Disconnecting a mailbox has to reach the machine that syncs it before the record goes, because once the record is gone nothing can tell that machine to stop. When the instruction cannot be delivered, nothing is removed and the mailbox is left exactly as it was. diff --git a/docs/content/docs/api/meta.json b/docs/content/docs/api/meta.json index 03161192..29d522e0 100644 --- a/docs/content/docs/api/meta.json +++ b/docs/content/docs/api/meta.json @@ -6,6 +6,7 @@ "index", "authentication", "sdks", + "cli", "oauth", "permissions", "endpoints", diff --git a/docs/content/docs/api/realtime.mdx b/docs/content/docs/api/realtime.mdx index c9977b93..9b0f033e 100644 --- a/docs/content/docs/api/realtime.mdx +++ b/docs/content/docs/api/realtime.mdx @@ -41,7 +41,7 @@ After connecting, join one or more topics with a `phx_join` message: | `account:` | One mailbox's sync and warmup events | Requires `manage_emails` | | `bulk:` | Progress of one bulk operation | Import/export progress; only the user who started the operation receives its events | -Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). +Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `CAMPAIGN_IDLE` (a continuous campaign ran out of leads and is waiting), `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). `EMAIL_OPENED` and `EMAIL_CLICKED` also carry `occurred_at` (when the tracking service saw it; `timestamp` is the publish time), `machine` (true for a mail client prefetch or a security scanner's fetch), and `client`, `device_type`, `country_code` and `city` when the consumer could tell. `AI_DRAFT_READY` fires when the [inbox agent](/guides/inbox-agent/) drafts a suggested reply awaiting review; it carries `thread_id` and `draft_id` and requires `access_unibox`. diff --git a/docs/content/docs/api/reference/account-org.mdx b/docs/content/docs/api/reference/account-org.mdx index 6f6003d9..4aebc31e 100644 --- a/docs/content/docs/api/reference/account-org.mdx +++ b/docs/content/docs/api/reference/account-org.mdx @@ -470,7 +470,7 @@ Auth: Session only (not available to API keys). `DELETE /auth/me/avatar` -Clears the profile image. Returns `204 No Content`. +Clears the profile image and deletes the stored file on a best-effort basis. Returns `204 No Content`. Auth: Session only (not available to API keys). @@ -1456,7 +1456,7 @@ Auth: Session only (not available to API keys). Requires a selected organization `DELETE /organization/avatar` -Clears the workspace image. Owner only. Returns `204 No Content`. +Clears the workspace image and deletes the stored file on a best-effort basis. Owner only. Returns `204 No Content`. Auth: Session only (not available to API keys). Requires a selected organization (owner only). diff --git a/docs/content/docs/api/reference/analytics.mdx b/docs/content/docs/api/reference/analytics.mdx index f53d5e38..a09387ab 100644 --- a/docs/content/docs/api/reference/analytics.mdx +++ b/docs/content/docs/api/reference/analytics.mdx @@ -29,6 +29,7 @@ Returns the main dashboard overview for the active organization: aggregate stats "total_opens": 612, "machine_opens": 88, "total_clicks": 143, + "machine_clicks": 6, "total_replies": 57, "total_bounces": 9, "open_rate": 49.35, @@ -239,6 +240,7 @@ Returns a single campaign's performance summary plus per-sequence-step stats. Th "unique_opens": 410, "machine_opens": 52, "unique_clicks": 99, + "machine_clicks": 4, "replies": 41, "bounces": 5, "unsubscribes": 3, @@ -258,11 +260,18 @@ Returns a single campaign's performance summary plus per-sequence-step stats. Th "replies": 28, "bounces": 3 } - ] + ], + "engagement": { + "countries": [{ "key": "US", "opens": 120, "clicks": 31 }, { "key": "DE", "opens": 44, "clicks": 9 }], + "clients": [{ "key": "Gmail", "opens": 98, "clicks": 20 }, { "key": "Outlook", "opens": 51, "clicks": 12 }], + "devices": [{ "key": "desktop", "opens": 140, "clicks": 37 }, { "key": "mobile", "opens": 24, "clicks": 3 }] + } } ``` -`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients); human opens are `unique_opens` minus `machine_opens`. +`engagement` groups the campaign's human opens and clicks by country (ISO code), mail client or browser, and device type (`desktop`, `mobile`, `tablet`), counting distinct contacts per bucket; a click counts as an open. Each list holds the busiest eight; an empty `key` is unknown. Country needs the GeoLite2 database on the consumer, otherwise every country row is unknown. + +`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients, opens inside ten seconds of the send); human opens are `unique_opens` minus `machine_opens`. `machine_clicks` counts steps whose only clicks were automated (a security gateway walking the links); those are not part of `unique_clicks` or `total_clicks`, which only ever count a person's click. ## Get campaign daily stats diff --git a/docs/content/docs/api/reference/campaigns.mdx b/docs/content/docs/api/reference/campaigns.mdx index 3148172f..bb5682f6 100644 --- a/docs/content/docs/api/reference/campaigns.mdx +++ b/docs/content/docs/api/reference/campaigns.mdx @@ -18,6 +18,7 @@ Search and page through the organization's campaigns. **Scope** `READ_CAMPAIGNS` | `q` | query | string | Free-text filter on campaign name. Optional. | | `folder` | query | string | Restrict to a single folder id. Optional. | | `status` | query | string | Status bucket filter: `draft`, `active`, `paused` (matches every paused variant), or `completed`. Any other value returns `400`. Optional. | +| `kind` | query | string | `sequence` or `one_time`. Any other value returns `400`. Optional. | | `cursor` | query | string | Opaque cursor from the previous page's `pagination.next_cursor`. Optional. | | `limit` | query | string | Page size. Optional. | @@ -35,6 +36,7 @@ A `data` plus `pagination` envelope. `next_cursor` is the campaign id to resume "name": "Q3 outbound", "description": "", "status": "active", + "kind": "sequence", "stop_on_reply": true, "open_tracking": true, "link_tracking": true, @@ -65,8 +67,14 @@ A `data` plus `pagination` envelope. `next_cursor` is the campaign id to resume "esp_match_mode": "off", "max_new_leads_per_day": 0, "prioritize_new_leads": false, + "continuous": false, + "idle_since": null, "tracking_domain": "", "tracking_domain_verified": false, + "utm_tracking": true, + "utm_source": "", + "utm_medium": "", + "utm_campaign": "", "updated_at": "2026-06-10T12:00:00Z", "created_at": "2026-06-01T09:00:00Z" } @@ -83,7 +91,7 @@ A `data` plus `pagination` envelope. `next_cursor` is the campaign id to resume `GET /campaigns-overview` -Status-bucket counts plus per-folder totals for the organization, used to drive campaign browsing UIs. `paused` sums every paused variant (`paused`, `paused_no_accounts`, `paused_trial_expired`). The path has no campaign id, so it lives beside `/campaigns` rather than under it. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. +Status-bucket counts plus per-folder totals for the organization, used to drive campaign browsing UIs. `paused` sums every paused variant (`paused`, `paused_no_accounts`, `paused_trial_expired`); `one_time` counts campaigns of that kind whatever their status. The path has no campaign id, so it lives beside `/campaigns` rather than under it. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. ### Response @@ -94,12 +102,45 @@ Status-bucket counts plus per-folder totals for the organization, used to drive "paused": 2, "draft": 4, "completed": 3, + "one_time": 2, "folders": [ { "folder_id": "6b9c1c8e-1f2a-4d3b-8c7e-9a0b1c2d3e4f", "total": 5 } ] } ``` +## Estimate a send + +`POST /campaigns-estimate` + +Project an audience against a sender pool before a campaign exists: how many contacts the segments resolve to, how many mailboxes would send, the pool's daily ceiling under the campaign limit, and the day the last send is expected to land. Nothing is written, so it needs no `Idempotency-Key`. The dashboard's one-time email wizard shows this on its last step. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. + +### Request body + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `segment_ids` | string[] | yes | Segments making up the audience (at most 20). A contact in several of them is counted once. | +| `email_tag_ids` | string[] | no | Mailbox tags that resolve the pool. Omit or send `[]` for every active mailbox in the workspace. | +| `daily_limit` | integer | no | Per-mailbox campaign cap to apply (defaults to `50`). Each mailbox counts the smaller of this and its own cap. | +| `days` | integer (0-127) | no | Weekday bitmask of sending days, bit 0 = Monday. Defaults to weekdays. | +| `timezone` | string | no | IANA timezone the days are counted in. Defaults to UTC. | +| `start_date` | string (RFC 3339) | no | When sending begins. Omit for now. | + +### Response + +`sending_days` and `estimated_finish_at` are `null` when the audience is empty, the pool has no capacity, or the send would take longer than two years. Today only contributes what the pool has not already sent (`remaining_today`). + +```json +{ + "recipients": 1000, + "mailboxes": 4, + "daily_capacity": 200, + "remaining_today": 140, + "sending_days": 5, + "estimated_finish_at": "2026-09-09T00:00:00+02:00" +} +``` + ## Create a campaign `POST /campaigns` @@ -112,10 +153,15 @@ Create a campaign. Only `name` is required, every other field is optional and ap | --- | --- | --- | --- | | `name` | string | yes | Campaign name. | | `description` | string | no | Free-text description. | +| `kind` | string | no | `sequence` (default) or `one_time`. A one-time email accepts at most one entry in `steps` here and refuses further email steps later; it is otherwise a normal campaign. Fixed at creation. | | `stop_on_reply` | boolean | no | Stop sending to a contact once they reply. | | `open_tracking` | boolean | no | Insert the open pixel. | -| `link_tracking` | boolean | no | Rewrite links through the tracking ticket service. | -| `text_only` | boolean | no | Send plain text only. | +| `link_tracking` | boolean | no | Rewrite links through the tracking ticket service. Each link gets its own ticket, so clicks are attributed per link. | +| `utm_tracking` | boolean | no | Tag every link with `utm_source`, `utm_medium`, `utm_campaign` and a per-link `utm_content` at send time. Default `false`. Values already on a link are kept. | +| `utm_source` | string | no | Overrides the default `warmbly`. Empty means the default. Up to 128 characters. | +| `utm_medium` | string | no | Overrides the default `email`. Empty means the default. | +| `utm_campaign` | string | no | Overrides the default, the campaign name as a slug. Empty means the default. | +| `text_only` | boolean | no | Send plain text only: no HTML part, and open and click tracking are off regardless of their flags. | | `daily_limit` | integer | no | Per-campaign daily send cap. | | `unsubscribe_header` | boolean | no | Add the RFC 8058 one-click unsubscribe header. | | `risky_emails` | boolean | no | Allow sending to risky/unverified addresses. | @@ -127,6 +173,7 @@ Create a campaign. Only `name` is required, every other field is optional and ap | `days` | integer (0-127) | no | Legacy weekday bitmask (superseded by `schedule_windows`). | | `start_time` | string | no | Legacy daily start (`HH:MM`). | | `end_time` | string | no | Legacy daily end (`HH:MM`). | +| `schedule_windows` | array | no | Per-day sending windows, 7 arrays indexed by weekday (Sunday = 0) of `{start, end}` minute-of-day intervals. When non-empty it supersedes `days`, `start_time` and `end_time`. | | `email_tag_ids` | string[] | no | Mailbox tag ids that resolve the sender pool (tags strategy). | | `folder_ids` | string[] | no | Folder ids to file the campaign under. | | `sender_strategy` | string | no | `tags` (default) or `explicit`. | @@ -139,6 +186,7 @@ Create a campaign. Only `name` is required, every other field is optional and ap | `esp_match_mode` | string | no | `off`, `prefer`, or `strict`. | | `max_new_leads_per_day` | integer | no | New-lead throttle, `0` is unlimited. | | `prioritize_new_leads` | boolean | no | Prefer new leads in each send window. | +| `continuous` | boolean | no | Keep running for new leads: out of leads, the campaign stays `active` and waits instead of finishing. Linking a segment turns it on. Default `false`. | | `tracking_domain` | string | no | Campaign-scoped tracking domain (honored only once verified). | | `steps` | object[] | no | Initial sequence steps in order (see create sequence input below). They are connected in order: each step routes unconditionally to the next, waiting that step's `wait_after` days. The first step's `wait_after` defaults to `0`, follow-ups to `3`. | | `variants` | object[] | no | A/B variants for the first step (same shape as create A/B variant). | @@ -200,7 +248,9 @@ Patch any subset of campaign fields. Omitted fields are left unchanged. The expl ### Request body -Every field is optional. Scalar fields use nullable pointers, so any field you send is applied. `start_date` and `end_date` additionally accept an explicit `null` to clear the stored date: a null `start_date` means "start now" and a null `end_date` means "run open-ended". Changing any schedule field (`start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`) on an active campaign reschedules its next send immediately, so clearing a future start date takes effect right away. Notable fields: `name`, `description`, `status`, `stop_on_reply`, `open_tracking`, `link_tracking`, `text_only`, `daily_limit`, `unsubscribe_header`, `risky_emails`, `cc`, `bcc`, `start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`, `email_tags`, `folders`, `contact_order_by`, `contact_order_dir`, `contact_order_field`, `sender_strategy`, `rotation_mode`, `ramp_enabled`, `ramp_start`, `ramp_increment`, `ramp_ceiling`, `esp_match_mode`, `max_new_leads_per_day`, `prioritize_new_leads`, `tracking_domain`. +Every field is optional. Scalar fields use nullable pointers, so any field you send is applied. `start_date` and `end_date` additionally accept an explicit `null` to clear the stored date: a null `start_date` means "start now" and a null `end_date` means "run open-ended". Changing any schedule field (`start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`) on an active campaign reschedules its next send immediately, so clearing a future start date takes effect right away. Notable fields: `name`, `description`, `status`, `stop_on_reply`, `open_tracking`, `link_tracking`, `text_only`, `daily_limit`, `unsubscribe_header`, `risky_emails`, `cc`, `bcc`, `start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`, `email_tags`, `folders`, `contact_order_by`, `contact_order_dir`, `contact_order_field`, `sender_strategy`, `rotation_mode`, `ramp_enabled`, `ramp_start`, `ramp_increment`, `ramp_ceiling`, `esp_match_mode`, `max_new_leads_per_day`, `prioritize_new_leads`, `continuous`, `tracking_domain`, `utm_tracking`, `utm_source`, `utm_medium`, `utm_campaign`. + +A campaign with `continuous` set stays `active` when it runs out of leads and carries `idle_since` while it waits; the timestamp clears once it has something to send. A continuous campaign can be started with no leads at all. Adding a lead to a `completed` campaign by any path (this API, a linked segment, an automation) restarts it through the same launch checks as starting it by hand; a refused restart is written to the campaign's activity log. ```json { @@ -519,7 +569,7 @@ An `ABWinnerAnalysis` object. `GET /campaigns/:id/attachments` -List the campaign's attachments. Each entry carries a short-lived presigned download `url`. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. +List every attachment of the campaign. Each entry carries a short-lived presigned download `url` and its `step_id`: a file scoped to a step is sent by that step alone, and a `null` `step_id` means every step of the campaign sends it. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. | Parameter | In | Type | Description | | --- | --- | --- | --- | @@ -550,13 +600,13 @@ A `data` array of attachment objects (no pagination wrapper). `POST /campaigns/:id/attachments` -Upload a file to attach to the campaign (or one step). Sent as `multipart/form-data`, not JSON. **Scope** `WRITE_CAMPAIGNS` · **Org permission** `manage_campaigns`. +Upload a file to attach to the campaign, or to one of its steps. Sent as `multipart/form-data`, not JSON. **Scope** `WRITE_CAMPAIGNS` · **Org permission** `manage_campaigns`. | Parameter | In | Type | Description | | --- | --- | --- | --- | | `id` | path | uuid | Campaign id. | | `file` | form (multipart) | file | Required. The file to upload (max 15 MB). Executable and script types are rejected. | -| `step_id` | form (multipart) | uuid | Optional. Scope the attachment to one sequence step. | +| `step_id` | form (multipart) | uuid | Optional. Scope the attachment to one sequence step of this campaign, which is then the only step that sends it. Omit it to attach the file to every step. A step of another campaign returns `404`. | ### Response @@ -616,7 +666,9 @@ A `PreflightReport` object. `POST /campaigns/:id/test-email` -Send a one-off preview of a sequence step to a chosen recipient through a chosen mailbox. Defaults to the first step when `step_id` is omitted. **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns`. +Send a one-off test of a sequence step to a chosen recipient through a chosen mailbox. The message is assembled the way a real send is: merge fields and spintax resolve for the contact, the files that step sends are attached (the campaign's unscoped attachments plus that step's own), the mailbox signature and the campaign's opt-out footer are appended, and a plain-text campaign ships without an HTML part. The subject is prefixed with `[TEST]`. Opens and clicks on a test are never tracked, and its opt-out link names no contact, so clicking it suppresses nobody. Defaults to the first step when `step_id` is omitted. **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns`. + +The mailbox may be any mailbox of the organization, not only one the caller connected, and it does not have to be in the campaign's sender pool. An API key with an `allowed_email_accounts` list can only test from those mailboxes. Passing `contact_id` additionally requires `READ_CONTACTS` (org permission `view_contacts`), since the rendered copy reads that contact's fields. | Parameter | In | Type | Description | | --- | --- | --- | --- | @@ -629,31 +681,45 @@ Send a one-off preview of a sequence step to a chosen recipient through a chosen | `account_id` | uuid | yes | Sending mailbox id. | | `recipient` | string (email) | yes | Where to send the test. | | `step_id` | uuid | no | Step to render and send, defaults to the first step. | +| `contact_id` | uuid | no | Contact of the organization to render the copy for, including its custom fields. Omitted renders for a placeholder contact (`Test Recipient` at `Test Company`, with the recipient's address and no custom fields). | ```json { "account_id": "5c7d...", "recipient": "me@example.com", - "step_id": "7e3b..." + "step_id": "7e3b...", + "contact_id": "9a2c..." } ``` ### Response +`contact_id` is present only when one was given. + ```json { "message": "test email sent", "recipient": "me@example.com", - "subject": "Quick question, Alex", - "account_id": "5c7d..." + "subject": "Quick question, {{.FirstName}}", + "account_id": "5c7d...", + "step_id": "7e3b...", + "contact_id": "9a2c..." } ``` +### Errors + +| Status | Code | When | +| --- | --- | --- | +| `404` | `not_found` | The campaign, step, mailbox or contact does not belong to the caller's organization. | +| `403` | `forbidden` | `contact_id` given without contact read permission, or `account_id` outside the API key's allowed mailboxes. | +| `400` | `bad_request` | The campaign has no steps. | + ## Start a campaign `POST /campaigns/:id/start` -Start (activate) the campaign so it begins sending real mail. Works from `draft`, any paused status, or `completed` (a campaign closed by a passed end date resumes once the date is extended or cleared; one with nothing left to send re-completes with a `400` explaining why). **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns`. +Start (activate) the campaign so it begins sending real mail. Works from `draft`, any paused status, or `completed` (a campaign closed by a passed end date resumes once the date is extended or cleared; one with nothing left to send re-completes with a `400` explaining why, unless it is `continuous`, in which case it starts and waits for leads with `idle_since` set). **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns`. | Parameter | In | Type | Description | | --- | --- | --- | --- | @@ -787,7 +853,7 @@ Return the segments linked to the campaign as live audience sources. **Scope** ` ### Response -A `data` array of link objects (no pagination wrapper). +A `data` array of link objects (no pagination wrapper). The counts are evaluated when you ask: `contact_count` is how many contacts the segment matches now, `lead_count` how many of them are leads of this campaign, and `held_out_count` how many are not leads because they were removed from the campaign by hand (see [replace linked segments](#replace-linked-segments)). ```json { @@ -798,6 +864,8 @@ A `data` array of link objects (no pagination wrapper). "color": "#0ea5e9", "description": "Replied or clicked in the last 30 days", "contact_count": 412, + "lead_count": 409, + "held_out_count": 3, "linked_at": "2026-06-10T12:00:00Z" } ] @@ -828,7 +896,7 @@ Atomically replace the campaign's linked segments with the supplied set (up to 2 ### Response -The resulting links plus how many leads the call enrolled. +The resulting links plus how many leads the call enrolled. `added` is `0` when every member was already a lead, when the segments match no contacts yet, or when the only members are held out; the per-link counts tell these apart. The links and the enrolment are written in one transaction, so a failed enrolment returns an error and changes nothing rather than `200` with `added: 0`. ```json { @@ -839,6 +907,8 @@ The resulting links plus how many leads the call enrolled. "color": "#0ea5e9", "description": "Replied or clicked in the last 30 days", "contact_count": 412, + "lead_count": 412, + "held_out_count": 0, "linked_at": "2026-06-10T12:00:00Z" } ], @@ -993,7 +1063,7 @@ Delete a sequence step. **Scope** `WRITE_CAMPAIGNS` · **Org permission** `manag `POST /campaign-template-preview` -Render subject and body templates against a sample (or supplied) contact exactly as the send path would, and report parse errors plus any unresolved `{{...}}` tokens. No side effects, no campaign id. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. +Render subject and body templates for a contact exactly as the send path would, and report parse errors plus any unresolved `{{...}}` tokens. With `campaign_id` and `account_id` the preview also goes through the rest of the send assembly: the plain-text rule, the mailbox signature and the campaign's opt-out footer are applied in send order, and the response names the sender and lists the attachments the send carries. Tracking pixels and link rewriting are left out. No side effects. **Scope** `READ_CAMPAIGNS` · **Org permission** `view_campaigns`. Passing `contact_id` additionally requires `READ_CONTACTS` (org permission `view_contacts`), and an API key with an `allowed_email_accounts` list can only name those mailboxes in `account_id`. ### Request body @@ -1002,29 +1072,46 @@ Render subject and body templates against a sample (or supplied) contact exactly | `subject` | string | no | Subject template. | | `body_html` | string | no | HTML body template. | | `body_plain` | string | no | Plain-text body template. | -| `contact` | object | no | Override fields on the built-in sample contact: `first_name`, `last_name`, `email`, `company`, `phone`, and a `custom_fields` map of string to string. | +| `contact_id` | uuid | no | A contact of the organization to render for, with its custom fields. Omitted uses the built-in sample contact. | +| `contact` | object | no | Override fields on the contact being rendered for (the sample or the one from `contact_id`): `first_name`, `last_name`, `email`, `company`, `phone`, and a `custom_fields` map of string to string. | +| `campaign_id` | uuid | no | Campaign of the organization whose opt-out footer, plain-text setting and attachments apply. The opt-out link in the preview names no contact. | +| `step_id` | uuid | no | The step being previewed, so `attachments` lists what that step sends. Without it only the campaign-wide files are listed. | +| `account_id` | uuid | no | Mailbox of the organization whose signature is appended (when signature sync is on) and which is reported as `from`. | ```json { "subject": "Hi {{first_name}} at {{company}}", - "body_plain": "Hey {{first_name}}, I saw {{company}} is hiring. {{unknown_token}}", - "contact": { "first_name": "Sam", "company": "Globex" } + "body_html": "

Hey {{first_name}}, I saw {{company}} is hiring. {{unknown_token}}

", + "contact_id": "9a2c...", + "campaign_id": "8f1d...", + "account_id": "5c7d..." } ``` ### Response -A `TemplatePreview` object. `errors` lists template parse errors that would block sending, `unresolved` lists literal tokens left after render. Both are omitted when empty. +A `TemplatePreview` object. `errors` lists template parse errors that would block sending, `unresolved` lists literal tokens left after render. `from` is present when `account_id` was given, `attachments` when `campaign_id` was given and there are files on the send (the campaign-wide ones plus `step_id`'s own). All four are omitted when empty. ```json { "subject": "Hi Sam at Globex", - "body_html": "", - "body_plain": "Hey Sam, I saw Globex is hiring. {{unknown_token}}", - "unresolved": ["{{unknown_token}}"] + "body_html": "

Hey Sam, I saw Globex is hiring. {{unknown_token}}



Best, Ana

Don't want these emails? Unsubscribe

", + "body_plain": "Hey Sam, I saw Globex is hiring. {{unknown_token}}\n\nBest, Ana\n\nDon't want these emails? https://app.example.com/u/...", + "unresolved": ["{{unknown_token}}"], + "from": { "name": "Ana Silva", "email": "ana@globex.com" }, + "attachments": [ + { "id": "3b0e...", "filename": "deck.pdf", "size": 482113, "mime_type": "application/pdf" } + ] } ``` +### Errors + +| Status | Code | When | +| --- | --- | --- | +| `404` | `not_found` | `contact_id`, `campaign_id` or `account_id` does not belong to the caller's organization. | +| `403` | `forbidden` | `contact_id` given without contact read permission, or `account_id` outside the API key's allowed mailboxes. | + ## Generate copy with the writing assistant `POST /generation/write` diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index 2f7aea2f..f69031b0 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -537,7 +537,9 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts` } ``` -When the contact is suppressed, `suppression` is an object: `{ "reason", "source", "expires_at", "created_at" }` where `source` is `bounce`, `complaint`, or `unsubscribe`. +`engagement.total_opened` and `engagement.last_opened_at` count opens by a person; opens a mail client or security gateway fetched automatically are left out, as they are in campaign analytics. A person's click counts as an open too; an automated click does not add to either field. + +When the contact is suppressed, `suppression` is an object: `{ "id", "kind", "value", "reason", "source", "expires_at", "created_at" }`. `kind` is `email` when the contact's own address is on the list or `domain` when its whole domain is, `value` is the matching entry, and `source` is `bounce`, `complaint`, `unsubscribe`, `manual`, or `import`. `id` is the suppression entry, which `DELETE /suppressions/:id` lifts; see [deliverability and ops](/api/reference/deliverability-ops/#list-the-suppression-list). ## Update a contact @@ -662,23 +664,46 @@ Returns a `data` array plus a `pagination` envelope. `GET /contacts/:id/timeline` -Returns the merged activity feed for a contact: sends, opens, clicks, replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or a category). Requires a selected organization (org-scoped events would otherwise be hidden), so a request with no organization returns `400`. +Returns the merged activity feed for a contact: sends, opens, clicks (one per link, naming the link), replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or a category). Requires a selected organization (org-scoped events would otherwise be hidden), so a request with no organization returns `400`. Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts` | Parameter | In | Type | Description | | --- | --- | --- | --- | | `id` | path | UUID | Contact ID. | -| `limit` | query | integer | Page size, 1 to 200 (default 50). | -| `before` | query | string (RFC 3339 nano) | The `at` timestamp of the oldest event from the previous page. | +| `limit` | query | integer | Page size, 1 to 200 (default 50). Anything else is a `400`. | +| `cursor` | query | string | Opaque pagination cursor from `pagination.next_cursor`. A malformed cursor is a `400`. | +| `before` | query | string (RFC 3339 nano) | Deprecated. Returns the events strictly older than this timestamp, which can skip events that share an instant with the page boundary; use `cursor`. A value that is not an RFC 3339 timestamp is a `400`. Ignored when `cursor` is set. | ### Response -Returns a `data` array and a `has_more` flag (not a cursor envelope). Paginate by passing the oldest event's `at` as `before`. +Returns a `data` array and the standard `pagination` envelope. Paginate by passing `pagination.next_cursor` back as `cursor` until `has_more` is `false`. The cursor is the exact position of the last event on the page (its time, source and row), so events that share a timestamp, common when a send, its open and a note land in the same second, are never skipped or repeated across pages. The top-level `has_more` mirrors `pagination.has_more` and is kept for clients written before the envelope. `pagination.total` is always `null`: the feed is merged from several tables and is never counted. ```json { "data": [ + { + "type": "email_clicked", + "at": "2026-06-09T11:42:00Z", + "email_account_id": "e1...", + "email_account_email": "rep@yourco.com", + "campaign_id": "c1...", + "campaign_name": "Q3 Outbound", + "step_id": "s1...", + "step_name": "Intro", + "subject": "Quick question", + "machine": false, + "link": { + "id": "7c0f...", + "url": "https://yourco.com/pricing?utm_source=warmbly&utm_medium=email&utm_campaign=q3_outbound&utm_content=pricing", + "label": "Pricing", + "utm_source": "warmbly", + "utm_medium": "email", + "utm_campaign": "q3_outbound", + "utm_content": "pricing", + "user_agent": "Mozilla/5.0 ..." + } + }, { "type": "email_replied", "at": "2026-06-09T11:02:00Z", @@ -710,13 +735,18 @@ Returns a `data` array and a `has_more` flag (not a cursor envelope). Paginate b "user_id": "u1..." } ], - "has_more": false + "has_more": false, + "pagination": { "total": null, "next_cursor": null, "has_more": false } } ``` -`type` is one of `email_sent`, `email_opened`, `email_clicked`, `email_replied`, `email_bounced`, `reply_received`, `deliverability`, `suppressed`, `note`, `meeting_booked`, `meeting_rescheduled`, `meeting_canceled`, `contact_created`, `campaign_added`, `campaign_removed`, `category_added`, or `category_removed`. Fields not relevant to an event type are omitted. +`type` is one of `email_sent`, `email_opened`, `email_clicked`, `email_replied`, `email_bounced`, `reply_received`, `deliverability`, `suppressed`, `note`, `meeting_booked`, `meeting_rescheduled`, `meeting_canceled`, `contact_created`, `campaign_added`, `campaign_removed`, `category_added`, `category_removed`, `form_submitted`, or `page_hit`. Fields not relevant to an event type are omitted. -Lifecycle events carry the name of what changed as it was at the time (`campaign_name`, or `category_id` plus `category_title`), so a later rename or deletion does not rewrite history. A `contact_created` event carries `source` (`manual`, `campaign`, `import`, `sheet_sync`, `api`, `ai_assistant`, or `unknown` for contacts that predate attribution) and `source_detail` (the file, campaign, sheet or API key name). The same values are on the contact itself as `source`, `source_detail` and `first_seen_at`, and never change after creation. +`email_opened` and `email_clicked` carry `machine`: `true` when an automated fetcher did it rather than the person (a mail privacy proxy, a fetch inside ten seconds of the send, several links followed within seconds). Per-link `email_clicked` events carry `machine_reason` (`prefetch`, `instant` or `burst`), and per-event `email_opened` rows carry it too (`prefetch` or `instant`); an open summarised from the lead alone carries only the flag. Both kinds carry an `origin` object when the event was logged: `client` when the user agent names a mail client or image proxy, `device_type`, `os`, `browser`, `browser_version`, and `country_code`, `region`, `city` when the consumer could resolve them. Opens appear once per event, so a contact who opened from two devices has two rows. Automated clicks are on the feed for the record but never count the step as clicked. An `email_clicked` event carries `link` with the link's `id`, `url`, `label` (its anchor text), the `utm_*` parameters the URL carried, and the `user_agent`; every link in an email is tracked on its own, so each link clicked is its own event. Clicks recorded before per-link attribution have no `link`. + +A `form_submitted` event carries `form_id` and `form_name`. A `page_hit` event is a page view on your own site from a browser tied to the contact through an email-link ticket (see [Website tracking](/guides/website-tracking/)); `subject` is the page title, or its path when the page has none, and `page_hit` carries the full view: `url`, `path`, `title`, `referrer`, `referrer_domain`, `landing` (the first view of a session), the `utm_*` parameters, `device_type`, `os`, `browser`, `browser_version`, `device_brand`, `language`, `timezone`, `screen_width`, `screen_height`, and `country_code`, `region`, `city` when known. + +Lifecycle events carry the name of what changed as it was at the time (`campaign_name`, or `category_id` plus `category_title`), so a later rename or deletion does not rewrite history. A `contact_created` event carries `source` (`manual`, `campaign`, `import`, `sheet_sync`, `api`, `form`, `automation`, `ai_assistant`, or `unknown` for contacts that predate attribution) and `source_detail` (the file, campaign, sheet, form, automation or API key name). The same values are on the contact itself as `source`, `source_detail` and `first_seen_at`, and never change after creation. ## Get a contact's campaign state diff --git a/docs/content/docs/api/reference/deliverability-ops.mdx b/docs/content/docs/api/reference/deliverability-ops.mdx index c783b674..eb24d0a9 100644 --- a/docs/content/docs/api/reference/deliverability-ops.mdx +++ b/docs/content/docs/api/reference/deliverability-ops.mdx @@ -1,6 +1,6 @@ --- title: Deliverability and ops -description: Ingest deliverability events, replay dead-lettered tasks, manage warmup routing rules, reply templates, and org-level outreach settings. +description: Ingest deliverability events, manage the suppression list, replay dead-lettered tasks, manage warmup routing rules, reply templates, and org-level outreach settings. --- This group covers the operational control surface for sending: posting deliverability signals (bounces, complaints, unsubscribes) back into the platform, listing and replaying dead-lettered tasks, defining premium-pool warmup routing preferences, managing reply templates, and reading or updating organization-wide advanced outreach settings. All routes are organization-scoped, so the caller must have an active organization selected (API keys are always bound to one organization). @@ -75,10 +75,18 @@ Returns the `AdvancedOutreachSettings` object directly (not wrapped in an envelo "show_intent_summary": true, "show_dlq_stats": true }, + "unsubscribe": { + "mode": "text", + "text": "If this isn't relevant, just reply and let me know and I won't email you again.", + "link_intro": "Not the right person, or not interested?", + "link_text": "Unsubscribe" + }, "custom": {} } ``` +The `unsubscribe` block is the workspace default for the opt-out appended after the signature of every campaign email: `mode` is `text` (a reply-to-opt-out sentence, the default), `link` (a sentence with the recipient's signed unsubscribe link) or `off`; a campaign's own `unsubscribe_mode` (`inherit` by default) overrides it. Copy fields are single lines of at most 300 characters and fall back to the defaults when blank. See the [unsubscribe guide](/guides/unsubscribe/). + `preflight.min_content_score` is stored clamped to `1`-`100`. A value outside that range is corrected on write rather than rejected, so a floor above `100` cannot flag every campaign permanently. Set `preflight.check_content_score` to `false` to turn the check off; the score never blocks or delays a send either way. See [Content checks](/guides/campaigns/). @@ -171,6 +179,94 @@ Auth: **Scope** `WRITE_CAMPAIGNS` · **Org permission** `send_campaigns` `202 Accepted` with no body. The event is queued for processing. +## List the suppression list + +`GET /suppressions` + +Pages the workspace suppression list, newest first: every address and domain no campaign will email, whatever put it there. Entries whose `expires_at` has passed are not returned. + +Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts` + +| Parameter | In | Type | Description | +| --- | --- | --- | --- | +| `q` | query | string | Substring filter on the address or domain. | +| `limit` | query | integer | Page size, 1 to 200. Default 50. | +| `cursor` | query | string | Opaque cursor from the previous page's `pagination.next_cursor`. | + +### Response + +```json +{ + "data": [ + { + "id": "5a0c...", + "organization_id": "9b1e...", + "email": "dana@acme.com", + "kind": "email", + "reason": "clicked the unsubscribe link", + "source": "unsubscribe", + "campaign_id": "c1...", + "metadata": { "via": "link" }, + "created_at": "2026-09-01T10:12:00Z", + "updated_at": "2026-09-01T10:12:00Z" + }, + { + "id": "7d2f...", + "organization_id": "9b1e...", + "email": "competitor.io", + "kind": "domain", + "reason": "Competitor", + "source": "manual", + "metadata": { "added_by": "4e5f..." }, + "created_at": "2026-08-20T09:00:00Z", + "updated_at": "2026-08-20T09:00:00Z" + } + ], + "pagination": { "next_cursor": null, "has_more": false } +} +``` + +`kind` is `email` or `domain`; a domain entry keeps the bare host in `email` and matches every address at it. `source` is `bounce`, `complaint`, `unsubscribe`, `manual` (added one at a time) or `import` (added as a batch). An invalid `limit` or `cursor` returns `400`. + +## Add to the suppression list + +`POST /suppressions` + +Adds addresses and domains. A value containing `@` is an address; a bare host, with or without a leading `@`, is a domain. Values that are neither are reported in `skipped` rather than failing the request, and a value already on the list is updated in place, so the call is safe to repeat. At most 5000 entries per request. + +Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts` + +### Request + +```json +{ + "entries": [ + { "value": "dana@acme.com", "reason": "Asked us by phone" }, + { "value": "acme.com" }, + { "value": "@partner.io" } + ], + "reason": "Existing customers" +} +``` + +`reason` on the request applies to every entry without its own. Adding an address also switches off the matching contact's `subscribed` flag. + +### Response + +```json +{ "added": 3, "skipped": [] } +``` + +## Remove from the suppression list + +`DELETE /suppressions/:id` + +Lifts one entry, so campaigns can email the address (or every address at the domain) again. Removing an address entry also restores the matching contact's `subscribed` flag. The removal is written to the audit log with the entry's value, kind and source, since lifting an opt-out the recipient made themselves is the action a compliance review looks for. + +Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts` + +Returns `204` on success and `404` when the entry does not exist in this organization. + ## List task dead letters `GET /tasks/dlq` diff --git a/docs/content/docs/api/reference/webhooks.mdx b/docs/content/docs/api/reference/webhooks.mdx index 8386afa0..134003ac 100644 --- a/docs/content/docs/api/reference/webhooks.mdx +++ b/docs/content/docs/api/reference/webhooks.mdx @@ -458,11 +458,11 @@ The full catalog is available at `GET /webhooks/event-types`. Below is the compl | `campaign.deleted` | A campaign is deleted. | | | `campaign.email_sent` | A campaign email is dispatched by a worker. | yes | | `campaign.email_delivered` | A campaign email is accepted by the recipient's mail server. | yes | -| `campaign.email_opened` | A tracked open is recorded. | yes | -| `campaign.email_clicked` | A tracked link click is recorded. | yes | +| `campaign.email_opened` | A person opens a tracked email. Automated opens (mail privacy proxies, fetches inside seconds of the send) do not fire it. | yes | +| `campaign.email_clicked` | A person clicks a tracked link. The payload carries the `url` and the link's `link_label`. Automated clicks (security gateways walking the links) do not fire it. | yes | | `campaign.email_bounced` | A campaign email bounces. | | | `campaign.reply_received` | A human reply lands for a campaign thread. | | -| `campaign.unsubscribed` | A recipient clicks the unsubscribe link or requests removal. | | +| `campaign.unsubscribed` | A recipient opts out. `source` is `one_click` (the mail client's button), `link` (the link in the email), `reply` (a reply asking to stop) or `action` (a sequence's unsubscribe step). | | | `campaign.deliverability_warning` | The campaign's rolling bounce or complaint rate enters the early-warning band. | | | `campaign.action` | A sequence flow `notify` action node fires. | | @@ -507,10 +507,10 @@ The full catalog is available at `GET /webhooks/event-types`. Below is the compl | Event | Description | Firehose | | --- | --- | --- | -| `contact.created` | A contact is created. | | +| `contact.created` | A contact is created. Carries `contact_id`, `contact_email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `custom_fields`, `source`, `source_detail`, `campaign_ids`, `category_ids` and `created_at`. Fires for contacts added by hand, through the API, by a form or by an automation; file imports, sheet syncs and a single API request adding more than 100 contacts are bulk arrivals and do not raise it. | | | `contact.updated` | A contact's fields change. | | | `contact.deleted` | A contact is deleted. | | -| `form.submitted` | A hosted form received a submission. Carries the form id and name, the submission id, the answers, and the contact id when one was created or matched. | | +| `form.submitted` | A hosted form received a submission. Carries the form id and name, the submission id, the answers under `data`, the mapped contact columns (`contact_email`, `first_name`, `last_name`, `company`, `phone`), `source_url`, and the contact id when one was created or matched. | | ### CRM diff --git a/docs/content/docs/development/bare-metal.mdx b/docs/content/docs/development/bare-metal.mdx index 92bb78ab..f4bb4706 100644 --- a/docs/content/docs/development/bare-metal.mdx +++ b/docs/content/docs/development/bare-metal.mdx @@ -333,6 +333,9 @@ PHX_HOST=ws.example.com PORT=4000 CHECK_ORIGIN=true +# ── GeoIP (the variable is required by the backend; the file is optional) ── +GEODB_PATH=/var/lib/warmbly/GeoLite2-City.mmdb + # ── Platform email (resets, invitations, digests) ──────────────── MAIL_TRANSPORT=smtp SMTP_HOST=smtp.example.com @@ -357,7 +360,7 @@ A few lines differ from the compose defaults on purpose: - `API_HOST`, `TRACKING_HOST` and the realtime `PORT` sit on `127.0.0.1`, so only nginx reaches them. The realtime service binds all interfaces regardless, so keep `4000` closed at the firewall - `TRUSTED_PROXIES` and `TRACKING_TRUSTED_PROXIES` name the proxy, so rate limits and audit records see the visitor's address instead of `127.0.0.1` - `CHECK_ORIGIN=true` makes the websocket refuse browsers that are not on `PHX_HOST`'s origin list -- `GEODB_PATH` is left unset. It only adds a city to sessions and audit rows; set it to a MaxMind `GeoLite2-City.mmdb` if you have one +- `GEODB_PATH` points at a file that need not exist. The backend refuses to start without the variable, but a missing database only means sessions, audit rows and email opens and clicks carry no city; drop a MaxMind `GeoLite2-City.mmdb` at that path if you have one Every other variable, with its default, is in the [configuration reference](/development/configuration/), and [`deploy/config/env.example`](https://github.com/warmbly/warmbly/blob/main/deploy/config/env.example) is the annotated template this file was cut down from. Mail relay, OAuth clients, single sign-on and the AI provider are configured exactly as in the [self-hosting guide](/development/deployment-guide/#platform-email); only the way values reach the process differs. @@ -562,16 +565,20 @@ Because the worker is not in a container, the admin panel's SSH-driven day-two a ## Upgrading -Rebuild the artifacts that changed, then restart. Migrations are forward-only and apply on backend boot, so bring the backend up before the rest: +`scripts/upgrade-bare-metal.sh` is the build steps above in order: build every artifact this host runs as the checkout's owner, then run the root-owned installer `warmbly-install-release` through sudo, which installs them, restarts the backend first (migrations are forward-only and apply on its boot), then the rest, keeping each frontend's `config.js` across the copy. Install that one script root-owned once, then run the upgrade as the user who owns the checkout: ```bash -cd /opt/warmbly/src && sudo git pull # or checkout a newer tag -# repeat the build steps for the services that changed, then: -sudo systemctl restart warmbly-backend -sudo systemctl restart warmbly-forms warmbly-consumer warmbly-tracking warmbly-realtime warmbly-worker +sudo install -o root -g root -m 0755 /opt/warmbly/src/deploy/systemd/warmbly-install-release.sh /usr/local/sbin/warmbly-install-release +echo "$USER ALL=(root) NOPASSWD: /usr/local/sbin/warmbly-install-release" | sudo tee /etc/sudoers.d/warmbly-upgrade >/dev/null +sudo chmod 0440 /etc/sudoers.d/warmbly-upgrade +cd /opt/warmbly/src && scripts/upgrade-bare-metal.sh --pull # git pull --ff-only, build, install, restart ``` -Frontend rebuilds overwrite `config.js` when you copy `dist/` over, so rewrite it afterwards (or keep the two files somewhere and copy them back). A build script that does all of it in order is worth writing the second time you upgrade. +The sudoers line is what lets the script call the installer without a password prompt; it allows that one root-owned command and nothing else. + +To pin a release instead of following the branch, check the tag out first (`git checkout vX.Y.Z`) and run it without `--pull`. + +The admin panel can run the same script for you. The version pill in its top bar shows when a newer release exists, and with the updater unit installed, **Update and restart** in it pulls, runs the script and reports back. [Updates](/development/updates/#without-docker) has the unit, the sudoers line it needs and the backend variable that points at it. ## Backups diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index f8b042be..a30fa2f8 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -141,6 +141,7 @@ TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12 | `DISABLE_PASSWORD_LOGIN` | Turns off email and password entirely, for single sign-on only deployments | `false` | yes | | `SSO_AUTO_PROVISION` | `true` lets a verified identity provider assertion create an account regardless of `DISABLE_REGISTRATION` | `false` | yes | | `AUTH_IP_RATE_LIMIT` | Unauthenticated auth requests allowed per source IP per 15 minutes | `60` | yes | +| `CLI_AUTH_IP_RATE_LIMIT` | CLI sign-in handshake requests allowed per source IP per 15 minutes. Its own budget, because one `warmbly auth login` polls around 200 times and must not exhaust the allowance above | `500` | yes | | `WARMBLY_BOOTSTRAP_EMAIL` | First owner's address, read only while the users table is empty | unset | yes | | `WARMBLY_BOOTSTRAP_PASSWORD_HASH` | Argon2 PHC string for that owner. Preferred over the plaintext form | unset | yes | | `WARMBLY_BOOTSTRAP_PASSWORD` | Plaintext convenience form. Warns at boot, and leaves a password in your process environment | unset | yes | @@ -305,7 +306,7 @@ Redis holds rate limit counters, the organization key cache, the realtime bridge |---|---|---|---| | `GEODB_PATH` | Path to a GeoLite2 City database | none, and the backend refuses to start without the variable | yes | -The variable must be set on the backend in every environment. The file itself is optional: a missing file at that path means sessions and audit rows are recorded without a city, and nothing else changes. +The variable must be set on the backend in every environment. The file itself is optional: a missing file at that path means sessions and audit rows are recorded without a city, and nothing else changes. The consumer reads the same variable, optionally, to put a country and city on each email open and click; without it those records carry client and device only. ## Workers @@ -381,6 +382,20 @@ Delayed sends run through the local poller, so the backend must be running for s | `SENTRY_DSN` | Error reporting. Optional in every environment, including `prod` | unset | | `APNS_KEY` or `APNS_KEY_PATH`, `APNS_KEY_ID`, `APNS_TEAM_ID`, `APNS_TOPIC` | Mobile push on backend and consumer. Partial configuration disables push with a warning, never a crash | unset | +## Updates + +| Variable | What it does | Default | Restart needed | +|---|---|---|---| +| `UPDATE_CHECK_ENABLED` | Polls GitHub Releases for a newer Warmbly and shows it in the admin panel's top bar and on Setup and health | `true` | yes | +| `UPDATE_CHECK_INTERVAL` | How often the release check runs. Minimum `5m` | `30m` | yes | +| `UPDATE_CHANNEL` | `stable` follows releases; `dev` also offers prereleases | `stable` | yes | +| `RELEASES_GITHUB_REPO` | The `owner/repo` whose releases count as Warmbly versions. Point a fork's instance at the fork | `warmbly/warmbly` | yes | +| `RELEASES_GITHUB_TOKEN` | Optional GitHub token; only raises the API rate limit | unset | yes | +| `UPDATER_URL` | The host-side updater that applies an update (pull, rebuild, restart). Unset, or `none`, leaves the panel report-only | `http://updater:8095` under compose | yes | +| `UPDATER_TOKEN` | The bearer token the backend presents to the updater | `INTERNAL_API_TOKEN` | yes | + +The updater's own variables (`UPDATER_MODE`, `UPDATER_COMMAND`, `UPDATER_REPO_DIR` and the rest) are on [Updates](/development/updates/#configuration); it is a separate process and its environment is not on the configuration page. + ## Forms service The public face of hosted forms (`cmd/forms`): it serves the React (TanStack) form app built from `forms/`, the per-form page shells, the embed loader and public submissions, on its own port so form traffic never touches the API origin. It reads its own environment; none of these appear in the admin panel except `FORM_IP_RATE_LIMIT`. @@ -406,6 +421,7 @@ The Rust open and click service. It reads its own environment, so these have to | `TRACKING_RATE_LIMIT_PER_MIN` | Counted pixel and click requests per source per minute. Over budget, pixels are still served but not counted, and click redirects get `429` | `300` | | `TRACKING_PAGEHIT_RATE_LIMIT_PER_MIN` | Website page views accepted per source per minute, on top of the shared budget above. Over budget, the snippet gets `429` | `60` | | `TRACKING_TRUSTED_PROXIES` | CIDRs the tracking service accepts a forwarded client address from. Empty trusts nothing and uses the socket peer, which is correct for a directly exposed service; set it behind a reverse proxy or the per-source rate limits and the location stored with page views are caller-controlled. Same convention as the backend's `TRUSTED_PROXIES` | empty | +| `TRACKING_IP_HASH_KEY` | Secret the source-address token in tracking events is keyed with. The token names one source for deduplication, rate limits and the click burst rule; keyed, it cannot be turned back into the address by enumeration | `INTERNAL_API_TOKEN` | | `TRACKING_CLIENT_IP_HEADER` | The one header a trusted proxy sets with the client address. No other header is read, so a caller cannot smuggle an address past a generic proxy in `CF-Connecting-IP`. For `x-forwarded-for` the proxy-appended last entry is used; set `cf-connecting-ip` behind Cloudflare | `x-forwarded-for` | | `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs an image built with `CARGO_FEATURES=kafka` | `nats` | | `NATS_URL`, `NATS_SUBJECT_PREFIX` | JetStream address and subject prefix. The publish subject is `.` | `nats://localhost:4222`, `warmbly` | @@ -456,11 +472,24 @@ These are the only settings a browser can change, and no environment variable ow | `sync.backfill_messages` | integer, 1 to 100000 | `5000` | The most messages that import stores per mailbox | | `sync.daily_messages_per_mailbox` | integer, 1 to 100000 | `2000` | New (live) messages one mailbox may store per UTC day. Over it, mail waits for the next day; replies to the mailbox's own sends have a separate budget of the same size | | `sync.daily_messages_per_org` | integer, 1 to 2000000 | `25000` | New plus imported messages one organization may store per UTC day | +| `retention.engagement_event_days` | integer, 1 to 3650 | `365` | How long the per-event open and click logs (client, device, approximate location) are kept. Campaign progress keeps its own summary that outlives them, so counts, filters and branching never change | +| `retention.form_event_days` | integer, 1 to 3650 | `180` | How long form funnel events (views, starts, field-level drop-off) are kept. Funnel reports range up to 90 days, so anything shorter shortens the report too | +| `retention.audit_log_days` | integer, 1 to 3650 | `90` | How long the audit trail is kept. It carries IP addresses, user agents and change payloads, so this is also how long that data is held | | `deliverability.enforce_domain_auth` | boolean | `true` | Whether a sending domain that fails SPF or DMARC stops cold campaign sending and warmup sending from every mailbox on it. Off keeps the check running and still shows the state and the Advisor card, it just never blocks | | `deliverability.auth_grace_hours` | integer, 1 to 720 | `72` | How long a domain must stay failing before the gate applies. The clock starts when the background check first sees the failure, so this is also how much warning the owner gets | The four `sync.*` values are read by the backend when a mailbox is loaded onto a worker (on connect, on reassignment, and by the reconciler's periodic republish), so a change reaches every mailbox within a few minutes without a restart. The fixed pacing numbers around them (burst per five minutes, hourly, backfill pace, the flood threshold and the chronic-overage rule) are compiled constants listed under **Instance > Limits**; see [Mailboxes](/guides/mailboxes/#what-gets-synced) for how the budgets behave. +The three `retention.*` values are read by the pruning sweeps on every pass, so shortening one takes effect on the next sweep rather than at the next restart. Deletion is permanent and there is no grace period: what already sits outside a shortened window goes on that sweep. See [data control](/development/data-control/#what-is-kept-and-for-how-long) for what each log holds and what a shorter window costs. + +An unattended install can seed the whole document before anyone signs in, with `WARMBLY_SETTINGS_BOOTSTRAP` holding the same partial JSON the admin API takes: + +```bash +WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"audit_log_days":30}} +``` + +It is applied only while the settings row has never been written, so from the first save in the panel onwards the panel is authoritative and the variable is a no-op. That is what [`install.sh --wizard`](/development/install/) writes when you answer its retention questions. + The two `deliverability.*` values are read on every scheduling pass and every warmup send, so turning the gate off releases blocked mailboxes within the 30 second cache window. Turning it back on does not stop anything retroactively: a domain still has to spend its whole grace window failing first. Only a sustained failure gates, so a domain reading `unknown` (never checked, DNS could not answer, or a special-use domain that cannot resolve) always sends. Manual sends and unibox replies are never gated; see [domain authentication](/guides/deliverability/#domain-authentication). Changing them is audited, and every value is validated and clamped server side on write as well as on read, so a row written by an older version still resolves. @@ -476,6 +505,8 @@ Changing them is audited, and every value is validated and clamped server side o ## See also +- [Install](/development/install/) for the one-command install that writes all of this for you +- [Data control](/development/data-control/) for where each store lives and what each retention window governs - [First run](/development/first-run/) for claiming a fresh instance - [Accounts and access](/development/accounts-and-access/) for who may sign in and how to invite people - [Instance health](/development/instance-health/) for the checks that read these values back diff --git a/docs/content/docs/development/data-control.mdx b/docs/content/docs/development/data-control.mdx new file mode 100644 index 00000000..c506cb8b --- /dev/null +++ b/docs/content/docs/development/data-control.mdx @@ -0,0 +1,214 @@ +--- +title: Data control +description: Where every store lives on disk, what each retention window governs, how backups work, and how to move an instance to another host. +--- + +Self-hosting Warmbly means holding mailbox credentials, message bodies and contact records on your own disk. This page is the whole answer to where that data is, how long it stays, and how you get it off this machine. + +Three questions, and they have different answers: + +| Question | Answer | +|---|---| +| Where is it? | [Six stores](#where-the-data-sits), each pointed at by one variable | +| How long does it stay? | [Retention windows](#what-is-kept-and-for-how-long), all editable in the admin panel | +| How do I move it? | [`warmblyctl backup`](#backups) for the instance, [workspace export](/guides/workspace-export-import/) for one workspace | + +## Where the data sits + +Every store's location is one variable in `.env`. Compose reads a source starting with `/` as a bind mount and anything else as a named volume, so the same variable covers both and there is no second compose file. + +| Variable | Holds | Default | +|---|---|---| +| `WARMBLY_PG_DATA` | Postgres: organizations, users, mailboxes (credentials sealed), contacts, campaigns, the audit trail | `/postgres` | +| `WARMBLY_BLOBS` | Message bodies, attachments, avatars and logos | `/blobs` | +| `WARMBLY_NATS_DATA` | The event bus's JetStream state | `/nats` | +| `WARMBLY_REDIS_DATA` | Cache and rate-limit counters. Disposable | `/redis` | +| `WARMBLY_WORKER_STATE` | A worker's own id and sync cursors. Disposable | `/worker` | +| `WARMBLY_UPDATER_STATE` | The last update job's log. Disposable | `/updater` | + +An [install from `install.sh`](/development/install/) points all six under one data root, so `/opt/warmbly/data` is the whole of it and you can `rsync` that path. A clone-and-build install keeps Docker named volumes unless you set the same variables. + +Only the first two carry anything you cannot rebuild. The other four are state a fresh container reconstructs, which is why a backup does not include them. + +### Encryption + +Two keys, and they are not interchangeable. + +| Key | Opens | Read by | +|---|---|---| +| `CREDENTIALS_ENCRYPTION_KEY` | Mailbox SMTP and IMAP credentials | Backend and workers, without an organization context | +| `KMS_LOCAL_MASTER_KEY` | The per-organization data keys, which in turn open everything else | Backend and consumer | + +Both are unrecoverable. A database backup without them restores an instance whose mailboxes authenticate against nothing, and there is no way back from that. They are in `.env`, and `install.sh` also writes them to `keys-backup.txt` next to the install, which is on the same disk as the database and therefore not a backup either. Copy them somewhere else. + +### External stores + +Postgres, Redis and blob storage each accept an external target, set at install time or by editing `.env`: + +```bash +PRIMARY_DB=postgres://user:pass@db.internal:5432/warmbly?sslmode=require +REDIS=redis://cache.internal:6379 +BLOB_PROVIDER=s3 +BLOB_BUCKET=warmbly +AWS_ENDPOINT_URL_S3=https://.r2.cloudflarestorage.com +``` + + +A remote worker writes bodies to its own disk, so the dashboard on the control-plane host finds nothing. Any deployment with workers on more than one machine needs S3-compatible blob storage. This is the one storage choice that is not just a preference. + + +## What is kept, and for how long + +Every window below lives in the database, not the environment, and is edited under **Instance > Instance settings** in the admin panel. A sweep runs a few times a day and reads the current value on every pass, so a change takes effect without a restart. + +### Mailbox import and sync + +| Setting | Default | Range | What it governs | +|---|---|---|---| +| `sync.backfill_days` | 90 | 1 to 730 | How far back the initial import reaches when a mailbox is connected, newest first | +| `sync.backfill_messages` | 5,000 | 1 to 100,000 | The most messages that import stores per mailbox | +| `sync.daily_messages_per_mailbox` | 2,000 | 1 to 100,000 | New mail one mailbox may store per UTC day | +| `sync.daily_messages_per_org` | 25,000 | 1 to 2,000,000 | New plus imported mail across one workspace per UTC day | + +Mail over a daily budget is deferred, never dropped: the provider cursor is held and the mail is re-offered on the next pass. Replies to the mailbox's own outreach ride a separate budget of the same size and keep landing regardless. + +### Event history + +| Setting | Default | What it holds | +|---|---|---| +| `retention.engagement_event_days` | 365 | Per-event open and click logs: client, device, approximate location | +| `retention.form_event_days` | 180 | Form funnel events: views, starts, field-level drop-off | +| `retention.audit_log_days` | 90 | The audit trail: actor, IP address, user agent, change payload | + +Each is between 1 and 3,650 days. These are the three settings a retention or privacy policy applies to, because each window is also how long the personal data in that log is held. + +None of them change a number anyone reads. Campaign progress keeps its own summary of opens and clicks that outlives the per-event log, so counts, filters and branching are unaffected by shortening any of these. What gets shorter is what a contact's timeline can show, how far a funnel report reaches, and how far back an admin can audit. + + +There is no grace period and no copy. Take a backup first if you are not sure. + + +The **minimal retention** preset in the admin panel and in the installer sets all three to 30 days. + +### Set them at install time + +The installer writes the answers into `.env` as one document, applied on the first boot of a fresh database: + +```bash +WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"audit_log_days":30}} +``` + +It is read only while the settings row has never been written. From the first save in the admin panel onwards the panel is authoritative, so leaving the line in `.env` never undoes a later edit. + +## Backups + +`warmblyctl backup` writes one bundle holding the three things that only restore together: + +- the database, as a `pg_dump` +- the blob root, when blobs are on the filesystem +- the encryption keys, unless you pass `--no-keys` + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz \ + && docker compose -p warmbly exec -T backend rm -f /data/blobs/warmbly.tar.gz +``` + +`/data/blobs` is a hand-off, not a destination: it is the one path the container and the host both see. The `&&` matters twice over. `backup` leaves its own output out of the archive, but a bundle left there is swept into the next run, so it has to be deleted; and a `cp` that failed must not be followed by deleting the only copy that exists. + +The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. Treat the file as you would the instance itself. + +An install from `install.sh --wizard` can schedule this for you: `backup.sh` next to the install, a systemd timer, a retention count, and an optional `aws s3 cp` to somewhere off the host. A backup that only exists on the machine it backs up is not one. + +### Restore + +On the destination host, with the same keys in place: + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + +The restore empties the schema and replays the dump, so it replaces everything currently on that instance and asks you to type `restore` first. + +Before it does anything it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` against the destination's and refuses to continue when they differ, printing the two lines to put in `.env`. That check is the point of the command: without it a restore looks like it worked and every mailbox fails to authenticate days later, with no error that names the cause. + +## Moving an instance + +Two ways, and they answer different questions. + +### The whole instance, to a new host + +Every workspace, every user, the platform admins, the API keys. + + + + + +### Install Warmbly on the new host + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --host +``` + + + + + +### Put the old keys in the new `.env` + +Copy `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` from the old install, then recreate the containers so they take: + +```bash +docker compose -p warmbly up -d +``` + + + + + +### Restore the bundle + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + + + + + +### Check it + +```bash +docker compose -p warmbly exec backend warmblyctl status +``` + +Mailboxes should be connected, not needing a reconnect. If they need one, the keys did not match. + + + + + +The `rsync` alternative works too and is sometimes simpler, as long as the stack is **stopped** first: a running Postgres data directory copied file by file is not a consistent snapshot and can restore as a corrupt cluster. Stop it, copy the data root and the `.env` to the new host, start it there. It moves the same bytes; the bundle exists because it is the version that survives a different host layout, a different Postgres, a live instance, and a partial copy. + +### One workspace, to another instance + +[Workspace export and import](/guides/workspace-export-import/) moves a single organization's data between two running instances, re-sealing its secrets for the destination's keys. That is the per-customer tool; the bundle here is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance. + +## Outbound calls + +A self-hosted Warmbly makes no outbound call of its own except one, and it is off with a single setting. + +| Call | When | Turn it off | +|---|---|---| +| GitHub releases API | Every 30 minutes, to tell the admin panel a newer version exists | `UPDATE_CHECK_ENABLED=false` | + +Everything else is you: mail through the mailboxes you connect, DNS lookups for the domains you check, and whatever integrations you configure. There is no telemetry, no phone-home, and no license check. + +## See also + +- [Install](/development/install/): the wizard that asks all of this up front +- [Configuration](/development/configuration/): every variable and every database-backed setting +- [warmblyctl](/development/warmblyctl/): `backup`, `restore`, and the operator commands +- [Workspace export and import](/guides/workspace-export-import/): the per-workspace story diff --git a/docs/content/docs/development/deployment-guide.mdx b/docs/content/docs/development/deployment-guide.mdx index de9191b1..6520ed29 100644 --- a/docs/content/docs/development/deployment-guide.mdx +++ b/docs/content/docs/development/deployment-guide.mdx @@ -5,6 +5,15 @@ description: A step-by-step guide to running the whole Warmbly platform yourself Warmbly self-hosts with **no cloud account of any kind**: no AWS, no GCP, no Stripe, no Kafka. One `docker-compose.yml` runs everything on local, open-source pieces, and every external dependency is an environment variable you can flip later. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` +That pulls the published release images and skips everything on this page: no clone, no compiler, under two minutes. Add `--wizard` and it asks where your data lives, what is kept and for how long, and how it is backed up. See [Install](/development/install/). + +This page is the build-from-source path, and the reference for every setting either path writes. Take it when you want to run modified code, a platform we publish no image for, or a build you produced yourself. + + Three commands give you a working install. Everything after them is optional, and nothing below is needed before you have seen it running. /warmbly/`, which works on any | Trigger | Images | Tags | |---------|--------|------| -| Push to `main` | backend, consumer, worker, tracking, realtime | `:`, `:dev` | +| Push to `main` | backend, consumer, worker, forms, updater, tracking, realtime | `:`, `:dev` | | Tag `vX.Y.Z` | all of the above plus web and admin | `:vX.Y.Z`, `:vX.Y`, `:vX`, `:prod` | +Every app service in `docker-compose.yml` carries both an `image:` and a `build:` key, so the same file serves both paths. `docker compose pull && docker compose up -d` runs the published images and compiles nothing; `docker compose up --build` still builds this checkout. Pin a release with `WARMBLY_TAG=v1.4.2` in `.env`, or point `WARMBLY_IMAGE_PREFIX` at your own registry. + To move a worker fleet onto a new release, update `WORKER_IMAGE` and use "Pull latest and restart" on each worker. Control-plane migrations are forward-only, so prefer rolling forward over rolling back. ## Upgrading and backups +The admin panel tells you when a newer version exists: the version pill in its top bar turns amber, and **Update and restart** in it pulls the checkout, rebuilds, restarts what changed and reconnects once the backend is back. `make up` starts the updater that makes the button work. The whole flow, what it does and how to run it without Docker, is on [Updates](/development/updates/). + +By hand it is the same two steps: + ```bash -git pull && make up # migrations apply on backend boot +make upgrade # git pull --ff-only, then make up; migrations apply on backend boot ``` Upgrading is safe with data in place: migrations are forward-only and apply on backend boot. ### Backing up -Back up three things. All three, or the backup is not a backup: +One command writes all three of the things that only restore together: + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz \ + && docker compose -p warmbly exec -T backend rm -f /data/blobs/warmbly.tar.gz +``` + +That bundle holds the database, the blob root and the encryption keys, and `warmblyctl restore` puts it back on another host, refusing to run when that host's keys are not the ones the bundle was sealed with. [Data control](/development/data-control/#backups) covers scheduling it and moving an instance. + +By hand, if you would rather assemble it yourself, it is these three and no fewer: ```bash docker compose -p warmbly exec -T postgres pg_dump -U warmbly warmbly_dev > backup.sql diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index 83c04b8b..e1364756 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -74,13 +74,16 @@ Produced by the Rust tracking service when a pixel loads or a tracked link is cl "event_type": "EMAIL_OPENED", "task_id": "uuid", "original_url": "https://example.com/page", + "link_id": "uuid", "timestamp": "2026-01-29T12:00:00Z", "user_agent": "Mozilla/5.0...", "ip_hash": "sha256..." } ``` -`event_type` is `EMAIL_OPENED` or `EMAIL_CLICKED`; `original_url` is set only for clicks; IPs are stored as hashes, never raw. The struct is `events.TrackingEvent` in `internal/events/schemas.go`, mirrored in `tracking/src/events.rs`. +`event_type` is `EMAIL_OPENED` or `EMAIL_CLICKED`; `original_url` and `link_id` (the click ticket, which names the link's stored destination and anchor text) are set only for clicks; IPs are stored as hashes, never raw. The struct is `events.TrackingEvent` in `internal/events/schemas.go`, mirrored in `tracking/src/events.rs`. + +The consumer classifies each event before it counts. An open is automated when the user agent is a mail privacy proxy or missing, or when it arrives within `TrackingMachineWindowSeconds` of the step's dispatch; it is recorded with `opened_machine` and upgraded by a later human open. A click is automated for a missing user agent, for arriving inside the same window, or when the same source clicked another link of the same email within `TrackingClickBurstSeconds`; every click is logged per link in `email_link_clicks` with its reason, and only a human click stamps `clicked_at`, fires instant actions, or emits a webhook. A burst is only recognisable from its second click, so a human click's stamp and log row are written at once but its effects (evidence, instant actions, webhook, realtime event) run after the burst window plus a second, on the classification the click has by then; a burst recognised meanwhile relabels the earlier click and withdraws the stamp when no human click remains. A consumer restart inside the window loses only those deferred effects, and routing still follows the clicked branch at the next step boundary. Website page views do not ride this topic. The tracking service forwards each accepted view to the backend's internal API (`POST /api/v1/internal/page-hits`) instead, because the backend is where the user agent and IP are turned into device and location, and the IP must not sit in a durable stream on the way there. diff --git a/docs/content/docs/development/first-run.mdx b/docs/content/docs/development/first-run.mdx index 23a02060..7b1db1c7 100644 --- a/docs/content/docs/development/first-run.mdx +++ b/docs/content/docs/development/first-run.mdx @@ -7,12 +7,18 @@ A fresh instance has no accounts and is described as **unclaimed**. Claiming it ## Claim the instance +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +The [installer](/development/install/) starts the stack and prints the claim link when it finishes. From a clone it is the same two steps and a build: + ```bash git clone https://github.com/warmbly/warmbly && cd warmbly make up ``` -`make up` builds the images, starts the stack, waits for the backend to answer, then runs `make claim`, which prints a single-use link and ends on the command that reissues it: +Both print a single-use link when they finish. `make claim`, or `warmblyctl setup-link` inside the backend container, prints a fresh one at any time while the instance is still unclaimed: ``` No accounts exist yet. Open this link to claim the instance and diff --git a/docs/content/docs/development/install.mdx b/docs/content/docs/development/install.mdx new file mode 100644 index 00000000..bedfda43 --- /dev/null +++ b/docs/content/docs/development/install.mdx @@ -0,0 +1,318 @@ +--- +title: Install +description: One command installs Warmbly from published release images, with a wizard that asks where your data lives, what is kept and for how long, and how it is backed up. +--- + +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +No clone, no compiler, no toolchain. The script checks Docker, creates one directory, generates every secret, pulls the pinned release images, starts the stack and prints the link that claims your instance. On a clean box with Docker already installed it finishes in under two minutes. + +Add `--wizard` and it asks the questions a config reference cannot ask for you. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --wizard +``` + +## What it installs + +| | | +|---|---| +| Where | `/opt/warmbly`, or wherever you point `--dir` | +| What | `docker-compose.yml`, a `.env` at 0600, `keys-backup.txt`, and the data root | +| Version | The newest release, resolved once and pinned in `.env`. Never `latest` | +| Images | `ghcr.io/warmbly/warmbly/*`, multi-arch for amd64 and arm64 | +| Services | Backend, consumer, worker, dashboard, admin, Postgres, Redis, NATS, and (unless you say core only) tracking, realtime and forms | + +Nothing is built on your machine. The clone-and-build path still exists and is still supported; see [self-hosting](/development/deployment-guide/) for when you want it. + +## Before you run it + +Piping a script into a shell is a fair thing to be uneasy about, and the objection is only answerable if the script is versioned, inspectable and checksummed. All three are: + +```bash +curl -fsSLO https://warmbly.com/install.sh +curl -fsSLO https://warmbly.com/install.sh.sha256 +sha256sum -c install.sh.sha256 +less install.sh +sh install.sh +``` + +The script itself is written to deserve that: `set -eu`, every line inside a function, and `main "$@"` on the very last line, so a download that is cut off half way executes nothing at all. It never asks to be piped into `sudo`; it asks for elevation only for the specific steps that need it, and names them. + +You also do not have to run it to see what it would do: + +```bash +sh install.sh --dry-run +``` + +That prints the exact `.env` and compose file it would write, and touches nothing. + +### And after it runs + +Every release publishes an `images.json` asset listing the manifest digest of each image in it. Once the pull finishes the installer compares what actually landed on the machine against that file, and stops before starting anything if they differ. You can check the same thing yourself at any time: + +```bash +docker image inspect ghcr.io/warmbly/warmbly/backend:v1.4.2 --format '{{index .RepoDigests 0}}' +``` + +An install pointed at your own registry with `--registry` has no manifest to check against, and the installer says so rather than pretending it verified something. + +## Requirements + +| You need | Notes | +|---|---| +| Linux or macOS, amd64 or arm64 | On Windows, run it inside WSL | +| Docker 20.10+ with Compose v2 | The script offers to install Docker on Linux, and never does so quietly | +| ~2 GB disk | The images, not a build cache. There is no build | +| 4 GB RAM | The running stack idles near 300 MB | +| Free ports | 8080, 5173, 5174, and 3000, 4000, 8090 with every component | + +Port `3000` is the one that actually collides: it is the default for a lot of other self-hosted software. The installer checks every port before it writes anything and tells you which one is taken. + +## The wizard + +`--wizard` walks nine steps. Every answer has a default that matches the fast path, so pressing enter through the whole thing produces exactly the install the one-liner produces. + +Every menu takes the keys you already use: + +| Keys | Does | +|---|---| +| `↑` `↓`, `j` `k`, `ctrl-n` `ctrl-p` | Move | +| `ctrl-d` `ctrl-u` | Half a screen | +| `gg`, `G` | First, last | +| `1` to `9` | Jump straight to that option | +| `enter`, `space`, `l` | Take it | +| `q`, `esc`, `ctrl-c` | Leave without installing | + +The review screen at the end takes `enter` to install, `e` to go back into a section, and `q` to quit. + +Steps append rather than replacing each other, so the whole run stays scrollable and nothing above the installer is touched. `--clear` redraws the screen at each step instead, if you prefer that. + + + + + +### Where it lives + +One directory holds the compose file, the `.env`, and, unless you move it in step three, every store this instance writes to. Default `/opt/warmbly`. + + + + + +### How it is reached + +The hostname every link is built from: the dashboard URL, the unsubscribe link in campaign mail, the tracking pixel. Then how it is served: + +| Choice | What it does | +|---|---| +| **Plain HTTP** | Ports published as they are. Correct for localhost or a private network | +| **Bundled Caddy** | Adds a Caddy service with automatic HTTPS, one hostname per surface, and binds everything else to loopback | +| **Behind your proxy** | Binds every service to `127.0.0.1` and sets `TRUSTED_PROXIES`, which is silently wrong on every proxied install that was configured by hand | + +`TRUSTED_PROXIES` matters more than it looks. Without it every request appears to come from the proxy, so per-IP rate limits and click deduplication stop distinguishing between people. + +The bundled Caddy option serves six hostnames under the domain you give it, and the installer prints the DNS records at the end: + +``` +app.example.com admin.example.com api.example.com +ws.example.com track.example.com forms.example.com +``` + + + + + +### Where the data sits + +The question the rest of the install cannot answer for you, and the reason the wizard exists. + +| Choice | What you get | +|---|---| +| **Under the install directory** | `/opt/warmbly/data` holds Postgres, blobs, NATS and worker state | +| **A path you choose** | An external disk or a mount you already snapshot | +| **Docker named volumes** | Managed by Docker, portable only through `docker volume` | + +A real path is the default, because it is the choice that makes backup and migration obvious: with the stack stopped, moving the instance to another host is `rsync` of that path plus the `.env`. Do not copy it while Postgres is running; a live data directory is not a consistent snapshot. To move an instance without stopping it, use [`warmblyctl backup`](/development/data-control/#backups). Compose reads a volume source starting with `/` as a bind mount and anything else as a named volume, so this is one variable per store and no second compose file. + +Then, behind one prompt: an external Postgres or Redis (the bundled container is left out entirely when you bring your own), and where message bodies, attachments and avatars go. + + +A worker running on another host writes to its own disk. Once you scale workers out, blob storage has to be S3-compatible or bodies go missing. The wizard offers MinIO, R2, B2 and AWS. + + + + + + +### Keys and secrets + +Five secrets are generated: `AUTH_SECRET`, `INTERNAL_API_TOKEN`, `SECRET_KEY_BASE`, `KMS_LOCAL_MASTER_KEY` and `CREDENTIALS_ENCRYPTION_KEY`. They are written 0600, `APP_ENV` is set to `prod`, and no published default is ever reachable through this path. + +The last two are unrecoverable. Every mailbox credential and every per-organization key is sealed with them, so a database backup that does not travel with them restores an instance whose mailboxes authenticate against nothing. The wizard prints both, writes them to `keys-backup.txt` next to the install, and waits until you type that you have a copy somewhere else. + + + + + +### What is kept, and for how long + +Two groups, both editable afterwards in **Instance > Instance settings**: + +| Setting | Default | What it governs | +|---|---|---| +| Import window | 90 days | How far back the initial import reaches when a mailbox is connected | +| Import cap | 5,000 | The most messages that import stores per mailbox | +| Daily per mailbox | 2,000 | New mail one mailbox may store per UTC day | +| Daily per organization | 25,000 | New plus imported mail across one workspace per UTC day | +| Opens and clicks | 365 days | Per-event logs, with the client, device and approximate location of each | +| Form funnel events | 180 days | Views, starts, field-level drop-off | +| Audit log | 90 days | Who did what, from which IP address and user agent | + +The last three are also how long that personal data is held. A "minimal retention" preset sets every event window to 30 days. + + + + + +### Backups + +A scheduled `warmblyctl backup`: a target directory, a frequency, how many to keep, and whether the encryption keys travel in the bundle. It writes `backup.sh` next to the install and, on a systemd host, a service and a timer. + +Including the keys makes the bundle as sensitive as the instance itself. Leaving them out means keeping `keys-backup.txt` somewhere else, or the bundles cannot be restored at all. + + + + + +### Who gets in + +The first owner, either as a printed claim link or unattended from `WARMBLY_BOOTSTRAP_EMAIL` plus a hash from `warmblyctl hash-password`. Then the registration posture, and platform mail: an SMTP relay, or skip it and have password resets and login codes printed to the backend log. + + + + + +### Footprint + +Everything, or core only (no tracking pixel, no websockets, no hosted forms). Ports are checked for collisions before anything is written. Last, the update check, which is one outbound call to the GitHub releases API and the only outbound call this instance makes on its own. There is no telemetry in Warmbly. + + + + + +### Review + +Every answer on one screen. Enter installs, `e` goes back to any section, `q` leaves without writing anything. + + + + + +## See it first + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --demo +``` + +The real wizard and the real review, with the pull, the container creation and the health wait played rather than run. It writes no file, pulls no image, creates no container and needs no Docker, so it answers "what does this actually do" without a server to try it on. + +It takes about half a minute, paced so the phases are readable; a real install is several minutes, most of it the pull and the first boot's migrations. `WARMBLY_DEMO_FAST=1` collapses it to under ten seconds. From a checkout: `make installer-demo`. + +## For agents + +An agent should never drive the wizard: it reads raw keypresses and needs a +terminal. Every answer is also a flag, so `--yes` with flags is silent, +deterministic and idempotent, and `--print-env` and `--dry-run` show what a run +would produce without producing it. + +The repository ships a `warmbly-install` skill under `skills/` covering exactly +that, plus backups and moving an instance. Install it the way your agent +expects, for example `cp -r skills/warmbly-install ~/.claude/skills/`. + +## Unattended + +Every answer is a flag and a `WARMBLY_*` environment variable, so the same script provisions from Ansible or cloud-init: + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --yes \ + --dir /opt/warmbly \ + --host warmbly.acme.com \ + --tls caddy \ + --data-root /mnt/data/warmbly \ + --blobs s3 \ + --retention-preset minimal \ + --backup-dir /mnt/backups/warmbly \ + --version v1.4.2 +``` + +| Flag | Variable | Default | +|---|---|---| +| `--dir` | `WARMBLY_DIR` | `/opt/warmbly` | +| `--host` | `WARMBLY_HOST` | `localhost` | +| `--tls` | `WARMBLY_TLS` | `none` (`caddy`, `proxy`) | +| `--data-root` | `WARMBLY_DATA_ROOT` | `/data` (or `volumes`) | +| `--blobs` | `WARMBLY_BLOBS` | `filesystem` (`s3`) | +| `--database-url` | `WARMBLY_DATABASE_URL` | bundled Postgres | +| `--redis-url` | `WARMBLY_REDIS_URL` | bundled Redis | +| `--components` | `WARMBLY_COMPONENTS` | `full` (`core`) | +| `--version` | `WARMBLY_VERSION` | newest release | +| `--channel` | `WARMBLY_CHANNEL` | `stable` (`dev`) | +| `--registry` | `WARMBLY_IMAGE_PREFIX` | `ghcr.io/warmbly/warmbly` | +| `--backup-dir` | `WARMBLY_BACKUP_DIR` | none | +| `--backup-keep` | `WARMBLY_BACKUP_KEEP` | `14` | +| `--retention-preset` | | `default` (`minimal`) | +| `--no-update-check` | `WARMBLY_UPDATE_CHECK` | on | + +`--print-env` emits the `.env` and exits. `--dry-run` prints every file it would write and exits. `--demo` walks the whole thing and installs nothing. + +## Re-running it + +The installer is idempotent. A second run: + +- adopts the existing `.env` and never regenerates a secret, because a new `AUTH_SECRET` signs everyone out and a new `CREDENTIALS_ENCRYPTION_KEY` is permanent data loss +- keeps the existing data root, because pointing an install at a new path is not a move, it is an empty instance next to a full one +- keeps any keys you added to `.env` yourself, in a clearly marked block at the end +- regenerates `docker-compose.yml`, keeping the previous one as `docker-compose.yml.bak` +- refuses a directory it did not create, unless you pass `--force` + +Put your own compose changes in a `docker-compose.override.yml`, which compose merges on top and the installer never touches. + +## Updating + +The version pill in the admin panel, then **Update and restart**. On an install created this way the updater runs in image mode: it writes the new tag into `.env`, pulls, and recreates the containers whose image changed. Nothing is built. See [updates](/development/updates/). + +By hand, from the install directory: + +```bash +docker compose -p warmbly pull && docker compose -p warmbly up -d +``` + +## Removing it + +```bash +sh install.sh --uninstall +``` + +Stops and removes the containers and leaves every byte of data where it is. Adding `--purge-data` deletes the data root and the install directory as well, and asks you to type a phrase first, because that also destroys the only keys that could have opened a backup of it. + +## When it goes wrong + +| Symptom | Cause | +|---|---| +| `Docker is installed but not answering` | The daemon is not running, or your user is not in the `docker` group yet. `sudo systemctl start docker`, then log out and back in | +| `Something already listens on 3000` | Another self-hosted tool. Change the published port in `docker-compose.yml` before starting | +| `Could not pull the release images` | The tag does not exist, or this host cannot reach `ghcr.io`. Check [releases](https://github.com/warmbly/warmbly/releases) | +| The API never answers | The first boot applies every migration. `docker compose -p warmbly logs -f backend` | +| No claim link printed | The database already has accounts, so there is nothing to claim. See [first run](/development/first-run/) | + +More in [troubleshooting](/development/troubleshooting/) and [instance health](/development/instance-health/). + +## Next + +- [First run](/development/first-run/): claiming the instance, the first owner, demo data +- [Data control](/development/data-control/): where every store lives, what each retention window governs, backups, and moving an instance +- [Configuration](/development/configuration/): every environment variable and every database-backed setting +- [Self-hosting](/development/deployment-guide/): the full guide, including building from source diff --git a/docs/content/docs/development/instance-health.mdx b/docs/content/docs/development/instance-health.mdx index cbcacd7d..24355e84 100644 --- a/docs/content/docs/development/instance-health.mdx +++ b/docs/content/docs/development/instance-health.mdx @@ -61,6 +61,7 @@ Two neighbouring endpoints round out the surface, both under **Instance** in the |---|---|---| | `GET /admin/instance/config` | `manage_settings` | Every configuration entry with its resolved value, source and restart requirement. Sensitive keys return a fingerprint, never a value | | `GET /admin/instance/limits` | `view_analytics` | The effective sending, warmup and rate limits this build compiles in | +| `GET /admin/instance/update` | `view_analytics` | The running build, the newest release, and the updater's state. Backs the version pill in the top bar; see [Updates](/development/updates/) | ## Security and secrets @@ -164,6 +165,16 @@ Platform mail is sent from a domain that differs from the dashboard's. Mailbox p `AUTH_LOGIN_CODE=always` was demoted to `new_device` at boot because the transport does not deliver. Otherwise nobody could ever complete a login. See [login codes](/development/accounts-and-access/#login-codes). +## Updates + +### update_available + +A newer Warmbly exists: a release on the configured channel is newer than the running build, or the checkout is behind its branch. The message names both versions and how to apply it. With the updater running, the version pill in the top bar has an **Update and restart** button; without it, `git pull && make up`. See [Updates](/development/updates/). + +### updater_unreachable + +`UPDATER_URL` is set but nothing answers there, so the update button cannot work. Start the updater (the `updater` compose profile, or the systemd unit on a bare-metal host), fix the address, or remove the variable to update by hand. See [enabling the updater](/development/updates/#enabling-the-updater). + ## Accounts and access ### registration_mode @@ -241,7 +252,7 @@ A reachable realtime service that still leaves the dashboard dead has a differen | Service | Needs | |---|---| | backend | The five secrets, `PRIMARY_DB`, `REDIS`, the provider switches, the public URLs, `EMAIL_ADDRESS`, `EMAIL_NAME` and `GEODB_PATH` | -| consumer | The same shared block. It writes to Postgres, so it needs `PRIMARY_DB` and both encryption keys, and it needs the mail identity or it silently sends nothing | +| consumer | The same shared block. It writes to Postgres, so it needs `PRIMARY_DB` and both encryption keys, and it needs the mail identity or it silently sends nothing. `GEODB_PATH` is optional and only adds a location to opens and clicks | | worker | No database. The event bus, `REDIS`, both encryption keys, `ENCRYPTED_KEYS_BACKEND_URL` plus the worker token, and the `BOX_*` OAuth clients | | tracking | The event bus, plus `BACKEND_INTERNAL_URL` and `INTERNAL_API_TOKEN`. It exits at boot without either of those two | | realtime | `JWT_SECRET` equal to the backend's `AUTH_SECRET`, plus `SECRET_KEY_BASE` and `DATABASE_URL`. It refuses to boot without all three. `REDIS_URL`, `PHX_HOST` and the connection limits have defaults | diff --git a/docs/content/docs/development/meta.json b/docs/content/docs/development/meta.json index 07a4934f..38ae6bba 100644 --- a/docs/content/docs/development/meta.json +++ b/docs/content/docs/development/meta.json @@ -4,13 +4,17 @@ "root": true, "pages": [ "---Self-hosting---", + "install", "deployment-guide", "bare-metal", "first-run", "accounts-and-access", "warmblyctl", + "data-control", "configuration", "instance-health", + "operator-notifications", + "updates", "troubleshooting", "---Development---", "local-development", diff --git a/docs/content/docs/development/operator-notifications.mdx b/docs/content/docs/development/operator-notifications.mdx new file mode 100644 index 00000000..7e641869 --- /dev/null +++ b/docs/content/docs/development/operator-notifications.mdx @@ -0,0 +1,79 @@ +--- +title: Operator notifications +description: Send instance events to Discord, Slack, a signed webhook, or an email address. +--- + +Your instance can tell you when something happens on it: a new signup, a worker going offline, a workspace asking for more capacity. Configure the destinations in the admin panel under **Configuration > Notifications**. + +These are operator alerts, not customer alerts. They are instance-wide and go to you. The per-workspace event delivery your customers configure is [Webhooks](/guides/webhooks/), which is a separate, queued and retried system. + +## Channels + +A channel is one destination. You can add up to 25. + +| Type | What to paste | Notes | +| --- | --- | --- | +| Discord | An incoming webhook URL | Server settings, then Integrations, then New Webhook | +| Slack | An incoming webhook URL | Create a Slack app with an incoming webhook | +| Webhook | Any HTTPS endpoint | Receives the event as JSON, optionally signed | +| Email | An address | Needs a working platform mail transport | + +Each channel picks which events it wants. Leave every event unchecked and it receives all of them, including ones added in later versions. + +Use **Test** on a channel to deliver a sample alert immediately. A test ignores the channel's on/off switch and its event selection, so it always sends, and it reports the transport error directly when delivery fails. + +### URL safety + +Webhook URLs must be HTTPS and resolve to a publicly routable host. Inline credentials are rejected. This is the same posture customer webhooks use, and it exists so a notification channel cannot be pointed at your internal network. + +To point a channel at a host on your own network, set `WARMBLY_ALLOW_UNSAFE_WEBHOOK_URLS=true`. That also permits plain HTTP. Only do this on a deployment where you control the network. + +### Signing + +A `webhook` channel with a signing secret sends: + +``` +X-Warmbly-Signature: t=,v1= +X-Warmbly-Event: +``` + +`v1` is `HMAC-SHA256(secret, "." + body)`, the same scheme as customer webhooks, so an existing verifier works unchanged. The body is: + +```json +{ + "event": "worker.offline", + "title": "Worker stopped responding", + "summary": "No healthy replacement of the same tier was available…", + "severity": "urgent", + "fields": [{ "label": "Worker", "value": "…" }], + "link": "https://app.example.com", + "timestamp": "2026-01-01T00:00:00Z" +} +``` + +### Credentials + +A chat webhook URL is a bearer credential: anyone holding it can post to that channel. The admin API never returns one in full. A saved channel reads back with its URL reduced to a host and the tail of its path, and its signing secret replaced by a placeholder. Sending those values back unchanged means "keep what is stored", so saving an unrelated toggle cannot wipe a credential. To change a URL or secret, type the new value in full. + +## Events + +| Event | Fires when | +| --- | --- | +| Enterprise inquiry submitted | Someone asks for enterprise pricing from the plan chooser | +| Limit increase requested | A workspace asks for more capacity than its plan allows | +| Payment failed | Stripe could not collect an invoice | +| Workspace created | A new organization is created | +| User registered | A new account finishes signing up | +| Worker went offline | A worker stops heartbeating and no healthy replacement of its tier is available | +| Warmup ban appealed | A blocked mailbox asks to rejoin the warmup pool | +| Workspace risk escalated | Risk scoring moves a workspace into a different posture | + +A self-hosted instance is offered the subset that means something without billing, so the panel never lists an event that cannot fire there. + +Worker and risk alerts are deduplicated: a worker that stays down does not re-alert for six hours, and a risk alert only fires on an actual change of posture. + +## Delivery behaviour + +Delivery is best effort and deliberately so. An alert never blocks or fails the request that produced it, and a chat server that is slow or unreachable is not allowed to slow the product down. There is no retry queue. If a channel is down when an event fires, that alert is lost. + +If alerts stop arriving, use **Test** first: it is synchronous and tells you exactly what the transport said. diff --git a/docs/content/docs/development/troubleshooting.mdx b/docs/content/docs/development/troubleshooting.mdx index 3e146b05..46582860 100644 --- a/docs/content/docs/development/troubleshooting.mdx +++ b/docs/content/docs/development/troubleshooting.mdx @@ -64,6 +64,7 @@ Newer builds return the invite-only refusal with its own machine code, `registra |---|---| | `no space left on device`, often from a random service mid-compile | Docker is out of disk. Free space with `docker builder prune -af` and `docker image prune -af`, check the host has about 10 GB free, then re-run `make up` | | `failed to authorize: ... EOF` while pulling a base image | A transient registry blip. Re-run `make up`; completed layers are cached | +| `ERR_PNPM_ABORTED_REMOVE_MODULES_DIR_NO_TTY` while building the `forms` image | A `forms/node_modules` from a native `make forms` or `make dev` was shipped into the Docker build context and overwrote the image's own install, and pnpm will not recreate it without a terminal. Current checkouts keep every `node_modules` out of the context and run pnpm in CI mode; on an older checkout, `rm -rf forms/node_modules` and re-run `make up` | | `failed to xattr /path/._something: operation not permitted` on macOS | The checkout is on a filesystem without native extended attributes (exFAT, NTFS or a network share), so macOS writes `._*` sidecar files that BuildKit cannot read. Run `dot_clean -m .` then `find . -name '._*' -delete` and re-run. Cloning to an APFS volume avoids it | ## The stack is up but something is wrong diff --git a/docs/content/docs/development/updates.mdx b/docs/content/docs/development/updates.mdx new file mode 100644 index 00000000..2e4519e1 --- /dev/null +++ b/docs/content/docs/development/updates.mdx @@ -0,0 +1,195 @@ +--- +title: Updates +description: How a self-hosted instance learns about a new Warmbly version, what the indicator in the admin panel means, and how Update and restart pulls, rebuilds and restarts the stack for you. +--- + +A self-hosted Warmbly checks for a newer version on its own and shows it in the admin panel. Applying it is one button when the updater runs next to the stack: the new version is fetched, the services that changed are recreated, and the panel reconnects once the backend answers again. Nothing in flight is lost: sending and syncing pause for the restart and resume from the database on the other side. + +How "fetched" happens depends on how the instance was installed, and the updater has a mode for each: + +| Mode | For | What an update is | +|---|---|---| +| `image` | An install from [`install.sh`](/development/install/) | Pin the new tag in `.env`, `docker compose pull`, recreate. Nothing is built | +| `compose` | A git clone | `git pull`, `docker compose build`, recreate | +| `command` | Anything else | `git pull`, then a build-and-restart script of your own | + +## The indicator + +The top bar of the **admin panel** (`:5174`) carries a version pill next to the environment label. It reads the running backend's build identity, so it shows exactly what is deployed, not what the checkout says. + +| Pill | Means | +|---|---| +| `v1.4.0` (grey) | Up to date. Click it for the details and a manual check | +| `Update to v1.5.0` (amber) | A newer release exists on the configured channel | +| `Update available` (amber) | The checkout is behind its branch (an install that tracks `main`) | +| `Updating` (blue, spinning) | An update job is running; the pill follows it from any tab | +| `Restarting` (blue, spinning) | The backend is coming back after an update | + +The same facts sit at the top of **Instance > Setup and health**, and an available update is also a finding there (`update_available`, info severity), so a page that only lists findings still tells you. + +## In the dashboard + +Every member of a self-hosted workspace sees the same version pill in the **dashboard** header, next to the plan badge: the running version in grey, or "Update to vX.Y.Z" in amber with a pulsing dot once a newer release exists. It is there so nobody has to ask which version the server runs, and so the people who cannot update still know one is waiting. + +Who can act on it follows platform admin access, not workspace roles: + +- **Members** see a badge. Its tooltip names the version and says to ask a platform admin. +- **Platform admins** click it and get the update dialog: the running and available versions with the release notes, the checkout and updater state, a "Check now" button, and **Update and restart**. That button leads to a confirmation pane that spells out what the update does (pull, rebuild and restart, sending pauses and resumes, migrations apply, the tab reconnects) before anything runs. + +While the update runs the dialog shows a progress bar, the step list with the live step highlighted, and the log behind a toggle. When the backend goes away for the restart the dialog says it is reconnecting and keeps polling; the pill in the header turns into a spinner so the job stays visible with the dialog closed, and a reload picks it back up. When the new backend answers, the dialog shows the result and reloads the dashboard after a short countdown, or, if the dialog was closed, a toast reports the new version and every list refreshes. + +The dashboard reads `GET /auth/instance` for the version (any member) and the admin endpoints below for the rest (platform admins only, the same permission gates as the admin panel). + +## What counts as newer + +Two signals, and either one lights the pill: + +- **A release.** The newest release of `RELEASES_GITHUB_REPO` on `UPDATE_CHANNEL` (`stable`, or `dev` to include prereleases) is compared with the running build's version. A build stamped `v1.4.0-3-gabc1234` (three commits past the `v1.4.0` tag) is treated as newer than `v1.4.0` and older than `v1.4.1`, so an instance built from `main` is not nagged about the release it already contains. +- **Commits.** When the updater runs, it fetches the remote on `UPDATER_FETCH_INTERVAL` and reports how far the checkout is behind its branch. Any distance counts as an update on an install that tracks a branch. + +A build that carries no version (an image built without the build arguments, reported as `dev`) cannot be compared with a release, and only the commit distance applies. + +The version comes from the binary itself: the Dockerfiles and `make up` stamp the tag, commit and build time in (`internal/version`), CI does the same for published images, and `warmblyctl status` prints it as the first line. + +## Update and restart + +The button appears when the updater is reachable and something newer exists. It needs the `manage_settings` admin permission, and every press is an audit row (`upgrade` on `instance`). + +On an install from `install.sh`, in image mode: + +1. **Pin release.** The chosen tag is written into `WARMBLY_TAG` in the install's `.env`, so the pull, the recreate, and every later `docker compose up` you run by hand all resolve the same images. A pull that fails puts the old tag back rather than leaving `.env` pointing at images that do not exist. +2. **Pull images.** `docker compose pull`. +3. **Restart.** `docker compose up -d --no-build` for every service, minus the updater itself. +4. **Clean up.** `docker image prune -f`, unless `UPDATER_PRUNE=false`. +5. **Wait for backend.** The updater polls `/health` until it answers, for up to six minutes. + +On a git checkout, in compose mode: + +1. **Fetch.** `git fetch --tags --prune` on the checkout. +2. **Pull.** On a branch, a fast-forward merge of the remote branch. On a checkout pinned to a tag, a checkout of the newest release. A checkout with local modifications is refused rather than merged over; commit or stash them, or set `UPDATER_ALLOW_DIRTY=true`. +3. **Build.** `docker compose build` for every service, with the version stamped in. +4. **Restart.** `docker compose up -d --no-build` for every service the checkout defines plus everything running, minus the updater itself. Compose recreates only the containers whose image or configuration changed, so Postgres, Redis and NATS stay up, and the backend applies migrations as it boots. +5. **Clean up.** `docker image prune -f`, unless `UPDATER_PRUNE=false`. +6. **Wait for backend.** The updater polls `/health` until it answers, for up to six minutes, and marks the job as succeeded or failed. + +If the updater's own image changed, it recreates itself last, after the outcome is already on disk, so the new updater reports the finished job. + +The dialog streams the log. Close it and the pill keeps following the job; reload the page and the pill picks the job up again. When the backend answers with a new build, a toast says which version is now running and every list refreshes. + +Migrations are forward-only, so an update never needs to be rolled back to be safe with data in place. Read [backing up](/development/deployment-guide/#backing-up) anyway: a backup before an update is the one you are glad to have. + + +The updater mounts `/var/run/docker.sock`, which is root on the host. That is why on a clone it lives behind the `updater` compose profile, and why in every install it answers only to a bearer token the backend holds, on the compose network, with no published port. The backend gates the button on an admin permission and audits it. If any of that is more trust than the host should extend, leave the profile off and update by hand. + + +## Enabling the updater + +### An install from install.sh + +Already done. The installer writes the updater into the generated compose file in image mode, pointed at the install directory, sharing `INTERNAL_API_TOKEN` with the backend. Nothing to configure. + +To update by hand instead, from the install directory: + +```bash +docker compose -p warmbly pull && docker compose -p warmbly up -d +``` + +To leave the button out entirely, delete the `updater` service from `docker-compose.yml` and set `UPDATER_URL=none` in `.env`. + +### Docker Compose from a clone + +`make up` starts the stack with the `updater` profile, so a stock install already has the button. For a plain `docker compose up`, put this in `.env`: + +```bash +COMPOSE_PROFILES=updater +``` + +The service mounts the checkout at its own host path (`WARMBLY_REPO_DIR`, `$PWD` by default) so the compose file's relative paths resolve the same way inside the container, and shares `INTERNAL_API_TOKEN` with the backend as `UPDATER_TOKEN`. Nothing else to configure. + +Files git writes as root are given back to the checkout's owner after each pull, so your own `git pull` on the host keeps working. + +To update by hand instead: `make upgrade` (a fast-forward pull, then `make up`). To leave the button out entirely, start with `make up UPDATER=false` and set `UPDATER_URL=none` in `.env`; the panel then only reports. + +### Without Docker + +On a [bare-metal install](/development/bare-metal/) the updater is a systemd unit that runs `scripts/upgrade-bare-metal.sh` after pulling. The script builds every artifact this host runs as the checkout's owner, then hands off to one privileged installer, `warmbly-install-release`, which copies the artifacts into `/opt/warmbly`, keeps each frontend's `config.js`, restarts the backend first and the rest once it answers. + +```bash +cd /opt/warmbly/src +go build -o out/updater ./cmd/updater && sudo install -m 0755 out/updater /opt/warmbly/bin/ +sudo install -o root -g root -m 0755 deploy/systemd/warmbly-install-release.sh /usr/local/sbin/warmbly-install-release +sudo install -m 0644 deploy/systemd/warmbly-updater.service /etc/systemd/system/ +printf 'UPDATER_TOKEN=%s\n' "$(grep ^INTERNAL_API_TOKEN= /etc/warmbly/warmbly.env | cut -d= -f2-)" | sudo tee /etc/warmbly/updater.env >/dev/null +sudo chmod 0600 /etc/warmbly/updater.env +sudo systemctl daemon-reload && sudo systemctl enable --now warmbly-updater +``` + +The unit runs as the user who owns the checkout (`deploy` as shipped; edit it). The only root step is the installer, so that user needs exactly one sudoers line, and nothing broader: + +```bash +echo "deploy ALL=(root) NOPASSWD: /usr/local/sbin/warmbly-install-release" | sudo tee /etc/sudoers.d/warmbly-upgrade >/dev/null +sudo chmod 0440 /etc/sudoers.d/warmbly-upgrade +``` + +The installer is root-owned and not writable by that user, takes no arguments, reads only from fixed paths under the checkout and refuses symlinks there, so owning the checkout does not become owning the host. The binaries it installs run as the unprivileged `warmbly` user either way. + +Then point the backend at the updater in `warmbly.env` and restart it: + +```bash +UPDATER_URL=http://127.0.0.1:8095 +``` + +The same script works by hand: `scripts/upgrade-bare-metal.sh --pull`. + +## Configuration + +Backend: + +| Variable | What it does | Default | +|---|---|---| +| `UPDATE_CHECK_ENABLED` | Polls GitHub Releases and shows a newer version in the panel | `true` | +| `UPDATE_CHECK_INTERVAL` | How often the release check runs; minimum `5m` | `30m` | +| `UPDATE_CHANNEL` | `stable` follows releases; `dev` also offers prereleases | `stable` | +| `RELEASES_GITHUB_REPO` | The `owner/repo` whose releases are Warmbly versions. Point a fork's instance at the fork | `warmbly/warmbly` | +| `RELEASES_GITHUB_TOKEN` | Optional; only raises the GitHub API rate limit | unset | +| `UPDATER_URL` | The updater. Unset, or `none`, leaves the panel report-only | `http://updater:8095` under compose | +| `UPDATER_TOKEN` | Bearer token presented to the updater | `INTERNAL_API_TOKEN` | + +Updater: + +| Variable | What it does | Default | +|---|---|---| +| `UPDATER_TOKEN` | The token it accepts; falls back to `INTERNAL_API_TOKEN`. Refuses to start without one | unset | +| `UPDATER_MODE` | `image` pins a release tag and pulls; `compose` rebuilds a checkout; `command` runs `UPDATER_COMMAND` | `compose` | +| `UPDATER_COMMAND` | The build-and-restart command for `command` mode | unset | +| `UPDATER_REPO_DIR` | The checkout, or in image mode the install directory holding the compose file and `.env` | working directory | +| `UPDATER_REMOTE` | The git remote to fetch and pull from | `origin` | +| `UPDATER_COMPOSE_PROJECT` | The `-p` the stack was started with | `warmbly` | +| `UPDATER_COMPOSE_PROFILES` | Extra profiles on every compose call, so its own image is rebuilt | `updater` | +| `UPDATER_BACKEND_HEALTH_URL` | Polled after the restart until it answers | `http://backend:8080/health` | +| `UPDATER_FETCH_INTERVAL` | How often it fetches for the commits-behind count | `30m` | +| `UPDATER_PRUNE` | `docker image prune -f` after a successful update | `true` | +| `UPDATER_ALLOW_DIRTY` | Update over local modifications in the checkout | `false` | +| `UPDATER_STATE_DIR` | Where the last job is kept across restarts | `/var/lib/warmbly-updater` | +| `UPDATER_ADDR` | Listen address | `:8095` | + +## Endpoints + +| Endpoint | Auth | Returns | +|---|---|---| +| `GET /admin/instance/update` | platform admin, `view_analytics` | Running build, latest release, whether an update is available, updater and job state. `?log=1` includes the job log | +| `POST /admin/instance/update/check` | platform admin, `manage_settings` | Runs both checks now and returns the state | +| `POST /admin/instance/update/apply` | platform admin, `manage_settings` | Starts an update job (`{"target": "latest"}` or a release tag) and returns it. The backend restarts as part of it | + +The updater's own API (`GET /status`, `POST /check`, `POST /update`) is bearer-authenticated and meant for the backend only. + +## Findings + +Two [instance health](/development/instance-health/) checks belong to this page: + +- `update_available` (info): a newer version exists, with the version and how to apply it. +- `updater_unreachable` (warning): `UPDATER_URL` is set but nothing answers there, so the button cannot work. Start the updater, fix the address, or remove the variable to update by hand. + +## Workers on other machines + +This page is about the control plane. Remote workers installed from the panel keep their own daily self-update timer and the **Pull latest and restart** action on the worker's page; see [day-2 operations](/development/deployment-guide/#day-2-operations). diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index c4657bf7..8cb2d462 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -3,6 +3,12 @@ title: warmblyctl description: The CLI for a Warmbly instance. The operator commands for accounts, health and recovery, and the API commands that let scripts and AI agents drive campaigns, contacts, mailboxes and the inbox with an API key. --- + +`warmblyctl` is for running an instance: accounts, health, recovery, backups. It reads the database directly and is meant to be run inside the backend container. + +If you want to use the product from your terminal (campaigns, contacts, mailboxes, the inbox), you want [`warmbly`](/api/cli/): a signed-in, multi-host client you install on your own machine, with `warmbly auth login` instead of an exported key. + + `warmblyctl` is the CLI for a Warmbly instance, and it has two halves with two trust models. The operator commands answer two questions: what state is this install in, and how do I get back in. They read and write the database directly, so they keep working when the sign-in page does not. Their authorization is container or host access, the same trust model as Sentry's `createuser`, Gitea's `admin user create` and authentik's `ak changepassword`, and it is the right one when the identity system is the thing that is broken. @@ -11,6 +17,8 @@ The [API commands](#the-api-commands) drive a running instance over its public R The CLI never serves HTTP. The API commands are a client of the already-gated public API, which adds no new surface to your instance. +Those API commands predate the [`warmbly` CLI](/api/cli/) and still work exactly as documented below. For day-to-day product work prefer `warmbly`: it signs in for you, holds a credential per host, prints tables rather than raw JSON, and runs on your laptop rather than inside a container. + ## Running it The binary ships inside the backend image at `/usr/local/bin/warmblyctl`, so it is on the path in every runtime. Running it inside the backend is the documented path because the environment there is already correct. @@ -55,10 +63,12 @@ A command that would set a password refuses on a non-TTY unless you passed `--pa | `APP_URL` | every printed link and sign-in hint | Links are built against `https://app.warmbly.com`, which is the hosted service and not your instance | | `KMS_PROVIDER` and its key | `org export`, `org import` | The command stops: a workspace's sealed values cannot be opened, so an archive would be useless | | `CREDENTIALS_ENCRYPTION_KEY` | `org export`, `org import` | A warning, and mailbox credentials are neither read nor written. Everything else still moves | +| `CREDENTIALS_ENCRYPTION_KEY`, `KMS_LOCAL_MASTER_KEY` | `backup`, `restore` | `backup` warns and leaves them out of the bundle. `restore` refuses when this host's differ from the bundle's | +| `BLOB_FS_ROOT` | `backup`, `restore` | Defaults to `/data/blobs`, which is where the compose stack mounts it | | `WARMBLY_API_KEY` | every API command | The command stops and explains where a key comes from | | `WARMBLY_API_URL` | every API command | Falls back to the instance's own `API_PUBLIC_URL`, then the hosted service | -Inside the backend container all of these are already set, except the API key, which is yours. Outside it, export what the command needs, and match `AUTH_SECRET` to the backend's exactly. +`backup` and `restore` also need `pg_dump` and `psql`, which the backend image ships for exactly this reason. Inside the backend container all of these are already set, except the API key, which is yours. Outside it, export what the command needs, and match `AUTH_SECRET` to the backend's exactly. ## The operator commands @@ -73,6 +83,8 @@ Inside the backend container all of these are already set, except the API key, w | [`user revoke-admin`](#user-revoke-admin) | Takes platform admin away from an account | | [`user disable-2fa`](#user-disable-2fa) | Clears an account's authenticator enrolment | | [`hash-password`](#hash-password) | Prints an argon2 hash for unattended provisioning | +| [`backup`](#backup) | Writes the whole instance to one restorable bundle | +| [`restore`](#restore) | Restores a bundle onto this instance, replacing everything on it | | [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size | | [`org export`](#org-export) | Writes a whole workspace to a portable archive file | | [`org import`](#org-import) | Applies an archive to a workspace on this instance | @@ -282,6 +294,57 @@ It prompts when a terminal is attached and reads the pipe when one is not, so th An argon2 PHC string contains `$` characters. Docker Compose reads those as interpolation, so a bare hash in `.env` silently loses part of itself. Wrap it in single quotes there, and double every `$` if you paste it into `docker-compose.yml` directly. +## backup + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz \ + && docker compose -p warmbly exec -T backend rm -f /data/blobs/warmbly.tar.gz +``` + +`/data/blobs` is the one path the container and the host both see, so it is the hand-off. Move the bundle out and delete it: `backup` excludes its own output from the archive, but a bundle left behind is swept into the next one. The delete is chained with `&&` so a failed copy cannot take the only copy with it. + +Writes one bundle holding the three things that only restore together: a `pg_dump` of the database, the blob root (message bodies, attachments, avatars), and the encryption keys. + +That combination is the whole reason this is a command rather than a documented list of steps. A dump alone restores an instance whose every mailbox credential decrypts to nothing, because the ciphertext is in the database and the key that opens it is in `.env`. The blob root alone restores bodies nothing points at. + +| Flag | Does | +|---|---| +| `--out` | Where to write it. Defaults to `warmbly-backup-.tar.gz` in the working directory | +| `--no-keys` | Leaves the encryption keys out. The bundle is then not restorable on its own | +| `--no-blobs` | Database only | +| `--force` | Overwrite an existing output file | + +The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. Treat the file as you would the instance itself: encrypted at rest, off the host, and not in a shared drive. + +Blobs travel only on the filesystem provider. An instance storing them in S3 keeps them in the bucket, and the bundle says so rather than pretending to be complete. + + +[`install.sh --wizard`](/development/install/) writes a `backup.sh` and a systemd timer that runs exactly this, keeps the last N bundles and can copy each one off the host. + + +## restore + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + +Empties the schema and replays the bundle into it, so it replaces every organization, user, campaign and mailbox currently on the instance. It prints what the bundle holds and asks you to type `restore` before it does; `--yes` skips that for scripts. + +Before anything is written it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` against this host's, and refuses to continue when they differ, printing the two lines to put in `.env` first. That check is the point of the command: without it a restore looks like it worked, and every mailbox fails to authenticate days later with no error that names the cause. + +| Flag | Does | +|---|---| +| `--file` | The bundle. Required | +| `--yes` | Skip the typed confirmation | +| `--no-blobs` | Restore the database only, leaving the blob root alone | +| `--force` | Restore even though the keys differ. Accepts losing every stored mailbox credential | + +Blobs are unpacked over the blob root additively: files the bundle knows about are overwritten, anything else is left alone. + +[Data control](/development/data-control/#moving-an-instance) walks the whole move to a new host. + ## org list ```bash @@ -344,6 +407,8 @@ Run it with `--dry-run` first. The report is the same preflight the dashboard sh The whole import runs in one transaction: if any part fails, nothing lands and the workspace is untouched. Members are matched to accounts on this instance by email address, and anyone without one has their rows reassigned to the workspace owner, named in the report before you commit. An archive carries no password material, so it can never create an account here. +`org export` and `org import` are the per-workspace tool and [`backup`](#backup) is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance. + Billing history, plan overrides, worker placement, mailbox sync checkpoints and warmup pool membership are exported for the record but never applied: each belongs to the instance rather than to the workspace. [Export and import](/guides/workspace-export-import/) has the full table. ## The API commands @@ -405,7 +470,7 @@ Paths are relative to `/v1`. `--data` takes a JSON literal, `-` for stdin, or `@ ### For AI agents -The repository ships skills under `skills/` (`warmbly-api` for the product, `warmbly-ops` for instance administration) that teach a coding agent these commands, the conventions above, and the sending-safety rules. Install them the way your agent expects, for example `cp -r skills/warmbly-api ~/.claude/skills/` for Claude Code, or point the agent at the `SKILL.md` directly. An agent given a scoped key and those skills can operate a workspace end to end without touching the database. +The repository ships skills under `skills/` (`warmbly-cli` for the [`warmbly` CLI](/api/cli/), `warmbly-api` for the same product surface through `warmblyctl`, `warmbly-ops` for instance administration, `warmbly-install` for standing an instance up and moving it) that teach a coding agent these commands, the conventions above, and the sending-safety rules. Install them the way your agent expects, for example `cp -r skills/warmbly-api ~/.claude/skills/` for Claude Code, or point the agent at the `SKILL.md` directly. An agent given a scoped key and those skills can operate a workspace end to end without touching the database. ## When Redis is down diff --git a/docs/content/docs/guides/analytics.mdx b/docs/content/docs/guides/analytics.mdx index 0dd4ae50..59a7a95f 100644 --- a/docs/content/docs/guides/analytics.mdx +++ b/docs/content/docs/guides/analytics.mdx @@ -20,8 +20,10 @@ Four rules govern the counts: - **Tracking must be on.** Opens and clicks need open or link tracking enabled on the campaign, and a tracking host on the install. A [custom tracking domain](/guides/mailboxes/#custom-tracking-domain) per mailbox is optional; without one they go through the shared host and still count. With tracking off they stay at zero while sends and replies still count. - **Replies are human replies.** Out-of-office and autoresponders never count, never stamp the contact as replied, and never trip stop-on-reply. -- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that open every link are served normally but never counted. Otherwise one corporate scanner would "click" every link seconds after delivery. -- **Auto-opens are labeled, not hidden.** Privacy proxies like Apple Mail Privacy Protection still count (they confirm delivery) but are tagged and shown separately (`12 auto`). A later real open upgrades them to human. **Auto-opens never trigger opened-based branches or automations.** +- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that announce themselves are served normally but never counted. The ones that do not announce themselves are caught by what they do: an open or click inside ten seconds of the send (nobody reads that fast; the clock starts when the send is handed to the worker, before the mail has even been delivered) and clicks on two or more links of one email from the same source within five seconds (a scanner walking the message). Otherwise one corporate scanner would "click" every link seconds after delivery. +- **Auto-opens are labeled, not hidden.** Privacy proxies like Apple Mail Privacy Protection, and instant opens, still count (they confirm delivery) but are tagged and shown separately (`12 auto`). A later real open upgrades them to human. **Auto-opens never trigger opened-based branches or automations.** +- **A click is an open.** Opens need the mail client to load images, and many clients block them. A click by the person proves the email was read, so it counts as an open as well, and a lead can never read "clicked, not opened". +- **Auto-clicks are kept but never counted.** A click classified as automated is logged on the contact's activity with the link it hit and the rule that caught it, and the campaign overview shows how many steps had only automated clicks (`3 auto`). It never makes the step clicked, never fires a clicked branch or automation, and never sends a webhook. A person clicking the same link later counts normally. A burst is only recognisable from its second click, so a click that looks human waits for the burst window plus one second (six seconds) before it fires anything (clicked branches, automations, webhooks, the live feed): if a second link follows inside that window, both clicks are relabeled, the stamp is withdrawn and nothing fires. ## Workspace dashboard @@ -42,6 +44,14 @@ Each campaign reports its own totals: contacts, sent, pending, unique opens and **Per-step stats** break sent, opens, clicks, replies, and bounces down by sequence step, which is how you find the touch pulling replies and the follow-ups that are dead weight. Daily stats drive the trend chart, and hourly stats show when sends land and get engagement. Campaigns can also be compared side by side on the same metrics. +**Who engaged, and from where** lists the countries, mail clients and devices people opened and clicked from, counting each contact once per bucket. Only human events feed it, so a security scanner's data centre never tops the country list. + +### What is recorded per open and click + +Every open and every click is kept as its own record, repeats included, with what the request said about itself: the mail client or image proxy when the user agent names one (Gmail, Apple Mail, Outlook), otherwise the browser, operating system and device type; the country, region and city resolved from the source network; for clicks the exact link; and whether it was the person or an automated fetch, with the rule that caught it. The address itself never leaves the tracking service: it publishes the network the address belongs to (the last IPv4 octet zeroed, an IPv6 address cut to its first 48 bits), which is enough for a city-level lookup and cannot single out a host, plus a keyed token of the full address used to tell one source from another, which cannot be turned back into the address. The consumer resolves the network to a location and stores neither. Location needs the GeoLite2 database on the consumer (`GEODB_PATH`); without it the country, region and city stay empty and everything else is still recorded. These records are kept for a year, then pruned; the counts and the first open and click per step live on the lead itself and are not affected. + +These records back the engagement rows on a contact's [activity timeline](/guides/contacts-crm/#activity-timeline) and the per-campaign breakdown above. + ## Per-mailbox | Signal | What it tells you | diff --git a/docs/content/docs/guides/automations.mdx b/docs/content/docs/guides/automations.mdx index d3db6889..badeb860 100644 --- a/docs/content/docs/guides/automations.mdx +++ b/docs/content/docs/guides/automations.mdx @@ -34,6 +34,8 @@ Triggers, conditions, and actions only take effect after **Save** (card position | Trigger | Fires when | | --- | --- | | Reply received | A contact replies to a campaign email | +| Contact created | A new contact is added by hand, through the API, by a form or by an automation (bulk file imports and sheet syncs stay quiet, see below) | +| Form submitted | A hosted [form](/guides/forms/) receives a submission | | Meeting booked / rescheduled / canceled | A Calendly or Cal.com meeting changes | | Email bounced | A campaign email bounces | | Unsubscribed | A contact unsubscribes | @@ -46,6 +48,8 @@ Each trigger carries its own event data (a reply carries contact email, reply in **Campaign action** never fires by itself; it runs only from a campaign step, still evaluating your conditions. +**Contact created** fires once per new contact, with the contact's fields and its first-touch source, so a condition can branch on `source` (a form, the API, an automation) or on any field. A contact that already existed and was only updated does not fire it. Bulk arrivals are deliberately silent: a file import or a Google Sheets sync of ten thousand rows would otherwise run ten thousand flows and flood every webhook, so those paths never raise it, and neither does a single API request adding more than 100 contacts. + ### Inbound webhook Pick the trigger and **save**, and the editor shows a unique URL. Anything that can send HTTP `POST`s JSON to it. @@ -93,6 +97,8 @@ Slack, Discord, and webhook actions take an optional message template. Slack and | Built-in action | What it does | | --- | --- | +| Create or update contact | Makes a contact from the event's fields, or enriches the one with that email, then tags it and enrols it in a campaign. See [Lead intake](#lead-intake) | +| Add to campaign | Enrols the event's contact in a campaign; a finished campaign restarts through the usual launch checks (a refusal is noted in its activity log and the lead waits), one waiting for leads picks up straight away | | Add / remove a tag | Adds or removes a contact category | | Label the email | Applies inbox labels to the replied-on conversation | | Create a task | Assigned to the workspace owner | @@ -105,6 +111,26 @@ Slack, Discord, and webhook actions take an optional message template. Slack and Three constraints worth knowing: **Move the deal stage** does nothing if the contact has no open deal in that pipeline, **Unsubscribe** needs an event carrying a campaign, and **Label the email** works only on a Reply received automation, since it needs a thread to label. +**Add to campaign** and every other contact action need a contact to act on: the event must carry `contact_id` or `contact_email`, which every Warmbly trigger does. On an inbound webhook, put **Create or update contact** first and the contact it writes becomes the event's contact for the steps after it. + +### Lead intake + +Any system that can send an HTTP request can create contacts in Warmbly without an API client: an **Inbound webhook** trigger followed by a **Create or update contact** action. + +1. Pick the Inbound webhook trigger, save, and copy the URL. +2. Add **Create or update contact**. Each field is a template over the JSON the caller sends: an email of `{{.email}}`, a first name of `{{.first_name}}`, a custom field `team_size` set to `{{.answers.team_size}}`. Pick the tags and the campaign the lead lands in. +3. Point the sender at the URL. Zapier's *Webhooks by Zapier*, Make's *HTTP* module, n8n's *HTTP Request* node, a Typeform or Tally webhook, or your own code. + +The action matches an existing contact by email, so re-sending the same lead updates it instead of duplicating it. A blank rendered value never erases a field the contact already has. **If the contact already exists** decides whether an existing contact is enriched and enrolled (the default) or left alone. New contacts carry `automation` as their first-touch source, with the automation's name as the detail. + + +Meta, LinkedIn and TikTok deliver lead-form submissions to Zapier, Make and n8n in real time. Either route them through the Warmbly app there with its **Create or Update Contact** and **Add to Campaign** actions, or send them to an inbound webhook automation as above. Both end the same way: a contact with the ad's answers as custom fields, tagged and enrolled in the campaign that follows up from your mailbox. See [Zapier](/guides/zapier/#lead-forms-and-other-lead-sources), [Make](/guides/make/#lead-forms-and-other-lead-sources) and [n8n](/guides/n8n/). + + +Lead-form campaigns produce leads in bursts, and the follow-up still goes out from your mailboxes under their daily caps and spacing. A day of two hundred leads on one mailbox capped at fifty is a four-day send; the campaign's recipients-versus-capacity estimate shows it. Add mailboxes rather than raising caps. + +A flow that creates a contact can be the reason a **Contact created** flow runs. That is allowed, and bounded: an event raised by an automation's own action carries how many hops led to it, and after five hops no further automation runs, so two flows cannot feed each other forever. + **Fire event** is the inverse of the inbound webhook: your app subscribes over the websocket with an API key holding `REALTIME_SUBSCRIBE` and receives `{ name, payload }`, so you get events without hosting a public URL. See [Realtime events](/api/realtime/). ### The on-error branch diff --git a/docs/content/docs/guides/billing.mdx b/docs/content/docs/guides/billing.mdx new file mode 100644 index 00000000..fd74506a --- /dev/null +++ b/docs/content/docs/guides/billing.mdx @@ -0,0 +1,54 @@ +--- +title: Plans and billing +description: How plans unlock features, how to upgrade from anywhere in the dashboard, and what changes when you switch. +--- + +Every hosted workspace starts free. The free workspace warms up to 10 mailboxes and links self-hosted instances. Sending from Warmbly Cloud, the unified inbox, contacts, the CRM, automations and integrations turn on with a paid plan. + +## Plans + +| Plan | Monthly | Annual (per month) | Sends per day | Includes | +| --- | --- | --- | --- | --- | +| Starter | $29 | $23 | 150 | Unlimited warmup, unlimited mailboxes, unified inbox, team invitations, bulk contact operations | +| Grow | $89 | $71 | 3,000 | Everything in Starter | +| Business | $329 | $263 | 15,000 | Everything in Grow, sending kept apart from other customers, webhooks, advanced outreach | +| Enterprise | Custom | Custom | 15,000+ | Everything in Business, dedicated support | + +Annual billing is 20% off the monthly price. The same lineup is on the [pricing page](https://warmbly.com/pricing). + +Unlimited mailboxes means exactly that, under a fair-use allowance of one mailbox for every send a day the plan includes: Grow holds `3,000`, Business `15,000`, and either can be raised on request. See [mailbox allowance](/guides/mailboxes/#mailbox-allowance). + +## Upgrading from a locked feature + +When you open something your plan does not include, the dashboard opens a full-screen plan chooser instead of sending you to settings. It names the feature, highlights the plan that unlocks it, and lets you pick monthly or annual billing. + +- **No subscription yet.** Choosing a plan opens Stripe Checkout. When you come back, you land on the page you left with the feature unlocked. +- **Already on a paid plan.** Choosing a higher plan switches you in place. The change is prorated and takes effect immediately. +- **Enterprise.** Opens the billing portal so you can reach us about custom volume. + +Promo codes can be applied in the chooser before checkout. + +Only the workspace owner can change the plan. Other members see the same comparison with a note to ask the owner. Which roles can do what is covered in [Team and roles](/guides/team-roles/). + +## Managing your subscription + +**Settings > Billing** is owner only and splits into four tabs, each with its own link. + +**Overview** is the control panel for the subscription. + +- The plan card shows the plan, its status, the price, and the renewal or end date. Change plan opens the same chooser the locked features use. +- Cancel plan schedules the cancellation for the end of the paid period. The plan stays fully active until then, and Resume clears it with one click. Nothing is lost either way: the workspace returns to the free tier and keeps its mailboxes, warmup and settings. +- Usage and limits shows live counts against the limits the server actually enforces, for mailboxes, sends today, contacts, campaigns, team members, attachment storage, sends this period, and API calls. A meter turns amber past 70% and red past 90%. A meter with no cap means that limit is unmetered on your plan. The mailbox meter says where its number comes from (fair use, an approved request, or the free allowance), and the storage meter says so when a plan change left the workspace over its quota: existing attachments keep sending, and new uploads wait until you are back under. +- Banners appear only when something needs a decision: a failed payment, a scheduled cancellation, or a trial about to end. + +Need a single limit raised without moving plan? Request an increase from **Settings > Limits**, linked from the usage section, or straight from the connect dialog when it is mailboxes you need. An operator reviews it and the approved limit takes effect on the workspace, including the daily send allowance the sender enforces. + +**Plans** compares the lineup as full plan cards, with the best value plan highlighted. On a paid plan each card also prices the switch before you commit, showing what is due today and when the next bill lands. Promo codes and the codes this workspace has redeemed live here too. + +**AI and credits** covers the credit balance, top-ups and usage. See [AI credits](/guides/ai-credits/). + +**Payment** links out to the Stripe billing portal. Cards and the billing email are held by Stripe and never touch Warmbly, so they are read and changed there. Invoices and PDF receipts live in the portal too. + +## Self-hosted deployments + +A self-hosted instance running without a billing provider has every feature unlocked and no billing page. See [Warmbly Cloud](/guides/warmbly-cloud/) for how a self-hosted instance links to a hosted workspace. diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index b0a22e91..22adfbcd 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -11,10 +11,24 @@ You need at least one active mailbox and a contact list. Warm new mailboxes befo ## Create a campaign -**Campaigns** > **New campaign** opens a four-step wizard: **Basics** (name, description), **Schedule** (timezone, sending days, hours), **Sending** (mailbox tags, daily limit per mailbox, stop on reply, open and click tracking, unsubscribe header), and **First email** (subject, body, optional follow-ups). Follow-ups added here are connected in order, each waiting the number of days you set after the previous step; open the Steps tab to rearrange, branch, or change the waits. Only a name is required; the first email can be skipped and written in the full editor on the Steps tab, and everything else can be changed later from settings. Closing a wizard with unsaved edits asks before discarding them. +**Campaigns** > **New campaign** opens a wizard that first asks what you are sending: a **sequence** or a [one-time email](#one-time-emails). For a sequence the steps are **Basics** (name, description), **Schedule** (timezone, sending days, hours), **Sending** (mailbox tags, daily limit per mailbox, stop on reply, open and click tracking, UTM parameters, unsubscribe header), and **First email** (subject, body, optional follow-ups). Follow-ups added here are connected in order, each waiting the number of days you set after the previous step; open the Steps tab to rearrange, branch, or change the waits. Only a name is required; the first email can be skipped and written in the full editor on the Steps tab, and everything else except the campaign type can be changed later from settings. The choice between a sequence and a one-time email is fixed at creation: switching means creating a new campaign. Closing a wizard with unsaved edits asks before discarding them. The campaign starts as a **draft**. Nothing sends until you start it. +## One-time emails + +A one-time email is a campaign with a single message and no follow-ups: an announcement, an update, a single ask to a segment. It is still a campaign underneath, so it sends through your mailbox pool with the same rotation, per-mailbox daily caps, spacing, suppression, tracking and analytics as a sequence. What changes is the flow and the wording. + +Pick **One-time email** on the wizard's first step. The steps are then **Basics**, **Email** (subject and body, both required), **Audience** (one or more [segments](/guides/segments/); every current member becomes a lead, and a contact in several of them is emailed once), **Sending** (mailbox tags, daily limit, tracking, unsubscribe header) and **Send**. The last step chooses **Send now** or **Schedule for later** with a date and time, sets the sending days and window, and shows an estimate: how many recipients the segments resolve to, how many mailboxes will send, the pool's daily ceiling, and the day the last send is expected to land. The estimate is the earliest finish under the daily caps, not a promise. A thousand contacts on one mailbox at the default cap is a multi-week send, and the wizard says so before you confirm. + +Confirming creates the campaign, links the segments (so the campaign keeps enrolling contacts who join them later, exactly like [linking a segment](/guides/segments/#linking-a-segment-to-a-campaign)), and starts it. A scheduled one waits for its date, then sends inside the window. If the start is refused, for example because the segments are empty or the list fails the [launch check](#the-launch-check-on-your-list), the email is kept as a draft and its page explains why. + +In the campaigns list a one-time email carries a **One-time** badge and reads **draft**, **scheduled** (started, waiting for its date), **sending** or **sent**; the **Type** filter shows only sequences or only one-time emails. The Overview tab reports opens, clicks, replies and bounces the same way it does for a sequence. Adding a second email step to a one-time email is refused: create a sequence when you want a reminder later. + +## Plain text only + +**Settings** > **Plain text only** sends the campaign without an HTML part. Recipients get the plain-text body with the mailbox's plain-text signature, and because open pixels and wrapped links need HTML, open and click tracking are off for that campaign whatever their toggles say. Test emails respect it too, so what lands in your own inbox is what recipients get. + ## Senders and rotation Pick mailboxes in **Sending accounts** three ways, and the first two combine: @@ -49,6 +63,8 @@ Warmbly is mailbox-first: safe volume is the sum of each mailbox's budget, not o The campaign daily limit applies as a minimum against the cold cap, so it can lower a mailbox's volume but never raise it above the mailbox's own daily cap (default `50`/day). +Only an email actually handed to a sending worker counts against a mailbox's daily budget and its minimum gap. The scheduler's own wake-ups, a step it deferred because its slot was not due yet, and action or wait steps never do, so a campaign started outside its sending window still has its whole budget when the window opens. When every mailbox on a campaign has used its budget for the day, the campaign stays active and waits for the next day; the activity log notes it once per day. It also waits, rather than pausing, when every mailbox is outside its own hours, resting, or held back by warmup health. + Anything above `50`/day per cold mailbox needs positive reputation signals and a low complaint rate behind it. Adding mailboxes is safer than forcing a few to send more. @@ -57,11 +73,17 @@ Anything above `50`/day per cold mailbox needs positive reputation signals and a **Lead flow** throttles new contacts with **Max new leads per day** (`0` for unlimited, up to `1000`). At the cap, follow-ups to in-flight contacts continue and new leads resume tomorrow. You can prioritize new leads over follow-ups, and choose whether to attempt addresses verification flagged risky. +**Keep running for new leads** turns a campaign into an ongoing one: when it runs out of leads it stays **active** and shows **waiting for leads** instead of finishing, then sends the sequence to each lead as they arrive from a linked segment, a form, the API, Zapier, Make, n8n or an automation. Linking a segment turns it on; turn it off in the campaign's preferences to have the campaign finish once every lead is done. A campaign with this on can be started with no leads yet. An end date still ends it. + ### Adding leads The **Leads** tab takes contacts four ways. **From contacts** opens a picker over the workspace contact list: search by name, email or company, filter by category, tick individual people or **Select loaded**, or **Select all matching** to add everyone the search returns (up to 1,000 per batch; repeat for the rest). Contacts already in the campaign are marked as leads and skipped. **Import** runs the file import wizard with this campaign preselected (its Options step can also pin the file into segments), **Sheet sync** attaches a Google Sheet, and **Add lead** creates a single new contact in the campaign. Any workspace member with contact access can add leads to any campaign in the workspace, whoever created it. -**Segments** on the same toolbar links [segments](/guides/segments/) to the campaign as a live audience, up to 20 per campaign. Linking enrols every current member immediately, and contacts who enter a linked segment later are enrolled on their own, within a couple of minutes. An active campaign wakes to send to them, a finished one restarts through the usual launch checks, and a paused one accumulates them for later. Enrolment is additive: a contact who leaves the segment keeps their lead row, and detaching a segment stops future enrolment without touching existing leads. See [linking a segment to a campaign](/guides/segments/#linking-a-segment-to-a-campaign). +**Segments** on the same toolbar links [segments](/guides/segments/) to the campaign as a live audience, up to 20 per campaign. Linking enrols every current member immediately and turns on **Keep running for new leads**, and contacts who enter a linked segment later are enrolled on their own, within a couple of minutes. An active campaign wakes to send to them (a campaign waiting for leads picks up straight away), a finished one restarts through the usual launch checks, and a paused one accumulates them for later. Enrolment is additive: a contact who leaves the segment keeps their lead row, and detaching a segment stops future enrolment without touching existing leads. See [linking a segment to a campaign](/guides/segments/#linking-a-segment-to-a-campaign). + +Every linked segment is shown in a **Linked segments** strip under the toolbar, so the audience feeding the list is always visible. Each chip names the segment and shows how many of its members are leads right now (`120/120`); clicking it filters the list to that segment's leads, and the arrow next to it opens the segment. A segment that matches no contacts says **no contacts** instead of a count, and one whose members were removed from the campaign by hand shows how many are **held out** with an **Add back** action, which re-enrols every current member and clears the removals. When a campaign has linked segments but no leads, the empty state explains why (the segments are empty, or their members were removed by hand) rather than offering to link a segment again. + +**Export** on the toolbar downloads the leads as CSV, XLSX or JSON. The scope is the campaign: **Every lead** exports the whole list, **Filtered** the leads matching the current filters, **Selected** the ticked rows. The **Campaign-ready** column preset adds each lead's status and whether they opened, clicked or replied. The same export lives on a segment page for its members. A lead can also be taken out again: the remove button on a lead's row, or **Remove from campaign** in the selection bar when several are ticked, drops the lead from this campaign without deleting the contact. The removal sticks: a linked segment will not re-enrol that contact automatically, even while they still match it, until you add them back yourself. Neither offers to delete the contact itself: inside a campaign the destructive action is leaving the campaign, and deleting a contact from the workspace is done from the contacts list. @@ -81,7 +103,9 @@ A lead is **Processing** only while steps remain, so a finished campaign reads a ### Who opened, clicked and replied -Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts only when a person opened the email. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement. +Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts when a person opened the email, or clicked a link in it. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement. Clicks are held to the same standard: a link followed within ten seconds of the send, or several links of one email followed within a few seconds of each other, is a security gateway scanning the message, not the recipient. Those clicks are kept on the contact's activity marked **auto**, counted apart on the campaign overview, and never make a lead **Clicked**, never fire a clicked branch or automation, and never send a webhook. See [Link tracking and UTM parameters](#link-tracking-and-utm-parameters). + +Opens depend on the recipient's mail client loading images. Outlook, many corporate setups and some privacy-minded clients block them, so a person can read an email and still show no open; the info icon on the column says as much. A click is stronger evidence, so a click by the person always counts as an open too. Open the contact's Activity tab to see each open and click with the mail client and location it came from. The chips above the list are filters. Click a status chip (**Processing**, **Done**, **Replied**, **Queued**, **Bounced**, **Unsub**) or an engagement chip (**Opened**, **Not opened**, **Clicked**, **Not clicked**, **Replied**, **Not replied**) to show only those leads; click it again to clear. One status and one engagement chip can be active at once and both must match. The numbers on the chips are campaign-wide totals, and filtering happens on the server, so a scope shows every matching lead however long the list is. **Not opened**, **Not clicked** and **Not replied** only cover leads that have been sent at least one email: a queued lead has not had the chance. The same filters live in the **Filters** sheet under **Lead status** and **Engagement**. @@ -106,6 +130,21 @@ A step counts as sent only once the sending worker has handed it to the mailbox No contact is emailed the same step twice. Each step is recorded as attempted before the send is handed to a worker, so a crash, a restart, or a database hiccup in the moment between the two cannot make the step look unsent and send it again. The trade-off is a step whose outcome is genuinely unknown, when the worker stops responding mid-send: after 30 minutes with no answer the step is treated as a failed attempt, appears in **Needs attention**, and is retried like any other failure. +## Link tracking and UTM parameters + +With **Link tracking** on, every link in the email body becomes its own tracked link, so Warmbly records which link was clicked, not only that one was. Open a contact's **Activity** tab and a click reads **Clicked Pricing**, with the campaign, step and sending mailbox on the row and, expanded, the link's text, its full URL, every UTM parameter it carried, the browser, and whether a person or a scanner followed it. The campaign's live feed names the link the same way, and the `campaign.email_clicked` webhook carries the URL and the link text. + +**UTM parameters** (on by default for campaigns created in the dashboard, off for campaigns created through the API unless you send `utm_tracking`) tag every link when the email is sent, so nobody has to add them by hand and clicks show up attributed in your web analytics: + +| Parameter | Value | +|-----------|-------| +| `utm_source` | `warmbly`, or the value you set on the campaign | +| `utm_medium` | `email`, or the value you set | +| `utm_campaign` | The campaign name as a slug (`q3_outbound`), or the value you set | +| `utm_content` | The link's own text as a slug (`pricing_page`); a link with no text, such as an image, is numbered `link_1`, `link_2` in order | + +A link that already carries one of these keeps the hand-written value; only the missing parameters are added. Existing query strings and fragments are left exactly as written, and the tags go on the real destination, so they survive the tracking redirect. Anchors, `mailto:` and `tel:` links, and anything that is not `http(s)` are never touched. UTM tagging works with link tracking off too: the tags are added and the link is otherwise left as written. Bare URLs in the plain-text part are tagged as well, numbered `link_1`, `link_2` since they have no text. Change the source, medium and campaign values under **Settings** > **Deliverability**; leave a field empty to use the default. + ## Scheduling Campaigns send only inside the **weekly sending windows** you define, in the campaign's timezone. Each day is independent, with different hours or several windows per day, set from presets like `Mon-Fri 9-5` or by dragging on the grid (which is Monday-first). @@ -122,9 +161,9 @@ Optional **campaign dates** bound when it may send. Leave both blank to run open The play and pause buttons work from the list row or the detail view. Starting moves the campaign to **active** and begins scheduling inside your windows and limits; pausing stops new scheduling immediately and resumes from where it left off. -A campaign can pause itself: **paused, no accounts** when it loses every sender, **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, **needs verification** when address verification has refused every remaining lead (the campaign offers to re-verify them or send anyway; see [address verification](/guides/deliverability/#address-verification)), and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes. Configured to do so, it also stops following up with a contact the moment they reply. +A campaign can pause itself: **paused, no accounts** when it loses every sender or no sender can send under its settings (a sending behaviour profile with no working days), **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, **needs verification** when address verification has refused every remaining lead (the campaign offers to re-verify them or send anyway; see [address verification](/guides/deliverability/#address-verification)), and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes, unless **Keep running for new leads** is on: then it stays active and shows **waiting for leads** until the next lead arrives, and only its end date finishes it. Configured to do so, it also stops following up with a contact the moment they reply. -A finished campaign can be started again: after extending or clearing its end date, or adding new leads, pressing play resumes it. If there is genuinely nothing left to send it finishes again immediately with a message saying so. +A finished campaign starts again on its own when a lead is added to it, whether by a linked segment, the API, a form, an integration or an automation, through the same launch checks as pressing play. If a check refuses the restart (the list is a bounce risk, the plan cannot send, too many campaigns are active), the activity log says so and the new leads wait until you fix the cause and press play. Pressing play also works after extending or clearing the end date. If there is genuinely nothing left to send it finishes again immediately with a message saying so. ## Duplicate and delete @@ -136,6 +175,8 @@ Every campaign has a **⋯** menu, on its row in the list and in the header of i ## Auto-pause guardrails +**Preferences** > **Opt-out line** chooses what this campaign appends after the signature: the workspace default from **Settings > Sending**, a reply-to-opt-out sentence, an unsubscribe link, or nothing. **Unsubscribe header** attaches the RFC 8058 headers alongside. See [unsubscribe and suppression](/guides/unsubscribe/). + **Preferences** > **Auto-pause** stops a campaign the moment its rates leave the band you set, instead of waiting for a mailbox provider to react. Off by default. | Rule | Direction | Default | Why | diff --git a/docs/content/docs/guides/collaboration.mdx b/docs/content/docs/guides/collaboration.mdx index ea170b83..53a115d8 100644 --- a/docs/content/docs/guides/collaboration.mdx +++ b/docs/content/docs/guides/collaboration.mdx @@ -34,6 +34,12 @@ Indicators are workspace-scoped. Teammates see only your name, avatar, current p **Events are permission-aware**: a member without inbox access never receives unibox events, and billing events reach only those who can manage billing. See [Team roles](/guides/team-roles/). +## Connection status + +A dot in the header reports the live connection: green **Live** when it is healthy, amber **Slow connection** or red **Poor connection** when heartbeats are taking longer than they should, amber **Reconnecting...** while it retries, and red **Disconnected** when it is down. The reading comes from the measured round trip to the realtime service, so amber means your connection genuinely is slow rather than merely busy. + +Stored data still catches up while it is down. Warmbly refreshes counts and lists on a slower cycle, and on reconnect it refetches everything that could have drifted. The events themselves are never replayed, so live-only signals such as presence and cursors resume from wherever things stand rather than backfilling what you missed. On a self-hosted install a permanently disconnected dot usually means the realtime service is not running; see [troubleshooting](/development/troubleshooting/). + ## Live cursors and chat On a shared page you see each other's cursors move, tagged with avatar and name, across the contacts table, deal board, unibox, settings, and everywhere else. On app-scrolling pages the cursor tracks content as either person scrolls. diff --git a/docs/content/docs/guides/contacts-crm.mdx b/docs/content/docs/guides/contacts-crm.mdx index 1ade6e79..8f1a53c1 100644 --- a/docs/content/docs/guides/contacts-crm.mdx +++ b/docs/content/docs/guides/contacts-crm.mdx @@ -100,16 +100,22 @@ Every contact records its first-touch source: how it entered the workspace, with | Import | Created by the file import wizard | The file name | | Sheet sync | Created by a Google Sheets sync source | The sheet | | API | Created through the API with an API key | The key's name | +| Form | Submitted a hosted [form](/guides/forms/) | The form | +| Automation | Written by an automation's **Create or update contact** action | The automation | | AI assistant | Created by the assistant on your behalf | | | Unknown | Created before Warmbly recorded sources | | Only new contacts get a source. An import or API call that matches an existing contact updates its fields and leaves the original source in place, so an address that arrived by hand and later turned up in a file still reads as manual. +A new contact is also an event. `contact.created` goes to your [webhooks](/guides/webhooks/) and can start an [automation](/guides/automations/#triggers) with the contact's fields and source, except for bulk arrivals, which stay silent: file imports, sheet syncs, and a single API request adding more than 100 contacts. + ## Activity timeline The **Activity** tab of a contact is one feed, newest first, of everything Warmbly knows about them: every campaign email sent, opened, clicked, replied to or bounced (with the campaign, step, subject and sending mailbox), replies with their classified intent, deliverability and suppression events, notes, meetings, and the contact's lifecycle: when it was created and how, and each time it joined or left a campaign or a category. -Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, categories and reasons, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it. +Opens appear once per event, not once per email: a second open from another device is its own row. Opens and clicks show the mail client or browser they came from and the city and country when known (see [what is recorded per open and click](/guides/analytics/#what-is-recorded-per-open-and-click)); expanded, the operating system, device and full location. A click names the link: the row reads **Clicked Pricing** and, expanded, shows the link's text, its full URL, the UTM source, medium, campaign and content it carried, and the browser. Every link in an email is tracked on its own, so two links clicked are two rows. Opens and clicks that came from a machine rather than the person (a mail privacy proxy, a security gateway that follows every link at delivery) carry an **auto** badge, and the expanded row says which rule caught them; see [Link tracking and UTM parameters](/guides/campaigns/#link-tracking-and-utm-parameters). + +Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, categories, reasons and, for clicks, the link's text, URL and UTM values, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it. At the top of the tab sits the campaign panel: for each campaign the contact is in, its flow with this contact's progress, the lead status, and what the scheduler will do next. See [Campaigns](/guides/campaigns/) for how the next action is worked out and what its states mean. @@ -130,9 +136,9 @@ Campaigns can create and advance deals automatically with **Create deal** and ** A suppressed contact receives no further mail. Unsubscribed contacts are skipped by every campaign automatically; toggle subscription on the Details tab, and filter the list by Subscribed or Unsubscribed. -Suppression also happens automatically on a hard bounce or spam complaint, and a sequence can run an **Unsubscribe** action. +Suppression also happens automatically when a recipient unsubscribes or replies asking to stop, on a hard bounce or spam complaint, and when a sequence runs an **Unsubscribe** action. The **Suppression list** tab holds every entry, takes addresses and whole domains by hand, and lets you lift an entry; a suppressed contact's Overview tab shows why and offers the same. -This is the safe default response to a bad signal: stop sending rather than keep collecting bounces and complaints. See [Deliverability](/guides/deliverability/) for the signals behind it. +This is the safe default response to a bad signal: stop sending rather than keep collecting bounces and complaints. See [unsubscribe and suppression](/guides/unsubscribe/) for the full picture and [Deliverability](/guides/deliverability/) for the signals behind it. ## Where to go next diff --git a/docs/content/docs/guides/deliverability.mdx b/docs/content/docs/guides/deliverability.mdx index 036b6064..efd23972 100644 --- a/docs/content/docs/guides/deliverability.mdx +++ b/docs/content/docs/guides/deliverability.mdx @@ -96,11 +96,11 @@ If a campaign has already been paused because every one of its mailboxes was gat ## Built-in protections -**One-click unsubscribe** (RFC 8058) puts a native Unsubscribe control in the recipient's mail client. It drives complaint rate down, because someone who can opt out cleanly rarely hits "mark as spam" instead, and Google's bulk-sender guidance effectively requires it at volume. Unsubscribes are suppressed automatically. +**An opt-out in every email.** Each campaign email ends with a way to stop hearing from you: by default a plain "just reply and let me know" sentence, whose replies are detected and honoured automatically, or an unsubscribe link if you prefer one. Campaigns can also attach the RFC 8058 `List-Unsubscribe` headers, which put a native Unsubscribe control in mail clients that show one. Either way, someone who can opt out cleanly rarely hits "mark as spam" instead, and every opt-out is suppressed automatically. See [unsubscribe and suppression](/guides/unsubscribe/). **Pre-send verification** checks every address before a campaign sends to it and skips the undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. There is nothing to run: new contacts are checked in the background within a minute of being added, each contact carries its verdict (a small mark next to the email), and Settings > Sending shows the whole workspace at a glance. See [address verification](#address-verification) below for how verdicts are produced and what to do when one is wrong. -**Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals. +**Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals. The list, with anything you add by hand (addresses or whole domains), is under **Contacts > Suppression list**. The list is held per workspace, and the check runs against the sending campaign's workspace before every send. If a campaign somehow has no workspace, that check cannot be applied, so the campaign is paused with the reason in its activity feed instead of sending unchecked. diff --git a/docs/content/docs/guides/expressions.mdx b/docs/content/docs/guides/expressions.mdx index 5f9ae333..8bd0b857 100644 --- a/docs/content/docs/guides/expressions.mdx +++ b/docs/content/docs/guides/expressions.mdx @@ -42,6 +42,8 @@ New reply from {{.contact_email}} on campaign {{.campaign_id}} ({{.intent}}) | Trigger | Variables | | --- | --- | | Reply received | `contact_email`, `contact_id`, `campaign_id`, `intent`, `confidence`, `subject`, `snippet` | +| Contact created | `contact_email`, `contact_id`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `source`, `source_detail`, `custom_fields` (a map: `{{.custom_fields.industry}}`), `campaign_ids`, `category_ids` | +| Form submitted | `contact_email`, `contact_id`, `first_name`, `last_name`, `company`, `phone`, `form_id`, `form_name`, `submission_id`, `source_url`, `campaign_id`, `data` (every answer by its key: `{{.data.team_size}}`) | | Meeting booked / rescheduled | `invitee_name`, `invitee_email`, `event_name`, `scheduled_for`, `join_url`, `source`, `contact_id` | | Meeting canceled | Same, without `join_url` | | Email bounced, Deliverability complaint | `contact_email`, `campaign_id`, `contact_id`, `event_type`, `provider`, `reason` | diff --git a/docs/content/docs/guides/forms.mdx b/docs/content/docs/guides/forms.mdx index e1b92f76..8c7a047e 100644 --- a/docs/content/docs/guides/forms.mdx +++ b/docs/content/docs/guides/forms.mdx @@ -58,7 +58,7 @@ The **Settings** tab controls what a submission does: - **Success message** is shown after submitting, or set a **redirect URL** to send the visitor to your own thank-you page instead. - **Add to categories** files every submitted contact under the categories you pick, for example "Website leads". -- **Add to campaign** enrolls new contacts as leads in the campaign you pick. Sending still follows the campaign's own schedule, limits and windows; a form never causes immediate mail. +- **Add to campaign** enrolls new contacts as leads in the campaign you pick. Sending still follows the campaign's own schedule, limits and windows; a form never causes immediate mail. Turn on the campaign's **Keep running for new leads** so it waits between submissions instead of finishing; a finished campaign restarts when a lead arrives, through the usual launch checks, and a refused restart is noted in its activity log while the lead waits. - **Spam protection** and **allowed embed domains** are covered below. ## Publishing and sharing @@ -109,6 +109,8 @@ The **Submissions** tab is a table of everyone who engaged, newest activity firs Each submitted contact carries `form` as its first-touch source, gets a "Submitted a form" entry on its activity timeline, and can be filtered in segments by source. The form's page views, submission count and conversion rate show on the Forms list. +A submission is also the **Form submitted** [automation trigger](/guides/automations/#triggers) and the `form.submitted` webhook event, carrying the form, the contact and every answer by its key, so a flow can tag the contact, open a task or post to Slack the moment it lands. + Submissions also fire the `form.submitted` [webhook](/guides/webhooks/), carrying the answers and the contact id, so you can forward leads to any external system. ## Analytics diff --git a/docs/content/docs/guides/integrations.mdx b/docs/content/docs/guides/integrations.mdx index b7835eed..e61039fa 100644 --- a/docs/content/docs/guides/integrations.mdx +++ b/docs/content/docs/guides/integrations.mdx @@ -72,7 +72,9 @@ URLs are validated before storage under the same policy as customer webhooks: th Integrations are most useful as the actions at the end of an [automation](/guides/automations/): post to Slack or Discord, upsert into a CRM, or fan out to Zapier, Make, n8n, or any signed webhook. -The most actionable events are available as triggers: reply received (with intent), email bounced, unsubscribed, meeting booked, rescheduled, or canceled, warmup health changed, and deliverability complaint. +The most actionable events are available as triggers: reply received (with intent), contact created, form submitted, email bounced, unsubscribed, meeting booked, rescheduled, or canceled, warmup health changed, and deliverability complaint. + +Data flows in as well as out. An automation's inbound webhook URL plus its **Create or update contact** action turns any push (a lead-form connector in Zapier, Make or n8n, a form tool's webhook, your own code) into a tagged, campaign-enrolled contact. See [Lead intake](/guides/automations/#lead-intake). Actions run in the background, so a slow third party never blocks the event that triggered it. Failures are recorded against the connection's health and visible in its recent activity rather than retried forever or silently lost. diff --git a/docs/content/docs/guides/mailboxes.mdx b/docs/content/docs/guides/mailboxes.mdx index 8d46eeac..d53d9c01 100644 --- a/docs/content/docs/guides/mailboxes.mdx +++ b/docs/content/docs/guides/mailboxes.mdx @@ -41,7 +41,56 @@ With two-factor authentication on, generate an app password in your provider's s Credentials and both connections are validated when you add the account, so wrong settings fail immediately rather than silently at send time. Tokens and credentials are sealed with envelope encryption before they touch storage. -**Limits**: `200` mailboxes per workspace by default (higher allowances come through a reviewed limit-increase request), and `5` new mailboxes per workspace per day as an abuse guardrail that resets daily. Spread large onboarding batches across days. +## Mailbox allowance + +Mailboxes are unlimited on every paid plan. What keeps that honest is a fair-use allowance derived from the plan's sending volume: one mailbox for every send a day the plan includes. + +| Plan | Sends per day | Mailboxes | +| --- | --- | --- | +| Free workspace | none | `10` | +| Starter | `150` | `150` | +| Grow | `3,000` | `3,000` | +| Business | `15,000` | `15,000` | +| Enterprise | custom | as many as the volume needs | + +That is deliberately far more than safe sending ever needs: at the recommended `30` to `50` sends a day per mailbox, a Business workspace fills its volume with a few hundred mailboxes and still has room for tens of thousands. The allowance exists so that it is never the reason to run a mailbox hotter. A plan whose daily sends are uncapped holds unlimited mailboxes, and a self-hosted instance without billing never counts. + +The connect dialog shows the allowance up front: a quiet count while there is room, a warning near the cap, and a clear full state that leads to the request flow instead of letting you type credentials that would be refused. When a connect is refused the answer is the same dialog, not an error toast. + +**Getting more.** Two paths, both in the dialog: + +- **Move to a bigger plan.** The dialog names the next plan and how many mailboxes it holds; the change is prorated and takes effect immediately. +- **Request an increase.** Keep your plan and ask for a higher allowance with a sentence on what you are sending. An operator reviews it, usually within a business day, and the new allowance applies to the workspace straight away. The dialog shows the open request while it is pending and lets you withdraw it. History lives under **Settings > Limits**. + +Nothing is ever removed for being over the allowance. A workspace that moves to a smaller plan keeps every mailbox sending and warming; it simply cannot add more until it is back under, or the allowance is raised. + +There is no daily cap on how many mailboxes you connect: a Business workspace can connect thousands in one afternoon, and the [bulk import](#connecting-many-mailboxes-at-once) exists for exactly that. + +## Connecting many mailboxes at once + +Pick **Bulk import from CSV** in the connect dialog to connect any number of SMTP and IMAP mailboxes from one file. Gmail and Outlook mailboxes sign in one at a time, because each needs its own consent. + +**The file.** One mailbox per row. `email`, `smtp_host` and `imap_host` are required, plus a password; everything else has a default. + +| Column | Default | +| --- | --- | +| `email` | required | +| `name` | derived from the address (`alex.rivera@` becomes Alex Rivera) | +| `smtp_host`, `imap_host` | required | +| `smtp_port`, `imap_port` | `587` and `993` | +| `smtp_user`, `imap_user` | the address, or a shared `username` column | +| `smtp_password`, `imap_password` | a shared `password` column | +| `smtp_security`, `imap_security` | inferred from the port (`465` and `993` are `tls`, `587` and `143` are `starttls`) | + +The dialog offers a template with these headers, and accepts the common spellings (`smtp_server`, `app_password`, `login`, and so on). + +**What happens.** The file is read in your browser and checked before anything is sent: rows missing something the connect needs are listed with the reason and left out of the run. The preview also says how many rows fit your allowance; if the file is larger, the first rows that fit are connected and the rest are reported as failed so you can request more and re-upload only those. + +The run then streams the rows to the server in small batches. Every credential is verified against its own server before it is saved, the same as a single connect, so a large file takes a few seconds per mailbox; the dialog shows live progress and the mailbox list behind it fills in as they land, for everyone in the workspace. You can stop after the batch in flight. Closing the tab loses nothing that was already connected. + +**When it is done** you see how many connected, how many were already here, and how many did not connect, with a reason per row. **Retry failed** runs those rows again without leaving the dialog, passwords still in memory. **Download failed rows** gives you those rows as you uploaded them plus an `error` column, with the password columns left out so no credential lands in a Downloads folder; add them back before uploading the fixed file. Re-uploading is always safe: a mailbox that is already connected is skipped, never doubled. + +The same endpoint is available to the API as `POST /emails/onboarding/smtp-imap/bulk`, up to `50` rows per call, answered per row. ## Reconnecting an account @@ -52,7 +101,7 @@ When the provider stops accepting a mailbox's stored credential (a password chan A successful reconnect stores the new credential, clears the authentication error, reactivates the mailbox on its existing worker, and it resumes syncing from where it stopped. Nothing else changes: settings, history, warmup progress, and campaign membership all stay. -Reconnecting never counts against the mailbox limit or the daily connect guardrail, so a workspace at its cap can still fix a broken mailbox. A mailbox whose sign-in is held by Warmbly Cloud is reconnected from your cloud workspace instead; the button says so if you try locally. +Reconnecting never counts against the mailbox allowance, so a workspace at its cap can still fix a broken mailbox. A mailbox whose sign-in is held by Warmbly Cloud is reconnected from your cloud workspace instead; the button says so if you try locally. ## What gets synced @@ -91,7 +140,9 @@ SMTP mailboxes get a **Keep a copy of sent mail** toggle, on by default. SMTP su Turn it off when your provider already saves its own copy of anything submitted over SMTP, which Gmail, Fastmail and Zoho do, or the folder ends up with two of every message. Gmail and Outlook mailboxes connected with OAuth never show the toggle: their APIs file the copy themselves. Warmup mail is never filed, since it would bury your real sent mail. -The same tab sets the **display name**, **reply-to** (empty uses the mailbox address), **signature** in plain text and HTML, and **tags** for grouping. +The same tab sets the **display name**, **reply-to** (empty uses the mailbox address), **signature** in plain text and HTML, and **tags** for grouping. A new display name is on the From header of the next message the mailbox sends, campaign, reply or warmup alike; nothing needs to be reconnected. + +The HTML signature is placed in a block of its own one line below the body, so it arrives without the stack of blank lines above it that Apple Mail and Outlook used to show. Put any extra spacing you want inside the signature itself. The plain-text signature follows the body after a single blank line. ### Custom tracking domain diff --git a/docs/content/docs/guides/make.mdx b/docs/content/docs/guides/make.mdx index 04680e6a..ea236cfc 100644 --- a/docs/content/docs/guides/make.mdx +++ b/docs/content/docs/guides/make.mdx @@ -51,6 +51,17 @@ Find Contact (by exact email), Find Deal, Find CRM Task, Find Campaign, Find Mai Pair a search with an action for idempotent flows: Find Contact, then Create or Update Contact. +## Lead forms and other lead sources + +Facebook and Instagram Lead Ads, LinkedIn Lead Gen Forms and TikTok Lead Generation all deliver new form submissions to Make in real time, which makes Make the shortest route from an ad to a follow-up sent from your own mailbox. + +1. **Trigger:** *Facebook Lead Ads: Watch Leads* (or the LinkedIn or TikTok equivalent), picking the Page and form. +2. **Module:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the categories. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. + +Prefer to keep the mapping inside Warmbly? Use the *HTTP: Make a request* module to POST the lead as JSON to an [inbound webhook automation](/guides/automations/#lead-intake) instead. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates, and the same flow can tag, notify Slack and open a task. + +The follow-up still runs through the campaign's mailboxes, so a burst of leads queues under each mailbox's daily cap and spacing rather than going out at once. Cold email tools do not ship a native Meta connector; this route is what their customers use too. + ## Good to know - **Sending respects your limits.** Send Email and Reply in Inbox honor each mailbox's daily cap and spacing, and support smart and scheduled send modes. diff --git a/docs/content/docs/guides/meta.json b/docs/content/docs/guides/meta.json index 662e64b1..7cd91f91 100644 --- a/docs/content/docs/guides/meta.json +++ b/docs/content/docs/guides/meta.json @@ -12,6 +12,7 @@ "sequences", "expressions", "deliverability", + "unsubscribe", "advisor", "---Contacts and inbox---", "contacts-crm", @@ -36,7 +37,9 @@ "integrations", "zapier", "make", + "n8n", "---Account and team---", + "billing", "analytics", "notifications", "security", diff --git a/docs/content/docs/guides/n8n.mdx b/docs/content/docs/guides/n8n.mdx new file mode 100644 index 00000000..02ff088f --- /dev/null +++ b/docs/content/docs/guides/n8n.mdx @@ -0,0 +1,37 @@ +--- +title: n8n +description: Connect Warmbly to n8n workflows, self-hosted or cloud, with the REST API and webhooks in both directions. +--- + +n8n has no Warmbly node yet, and it does not need one: its HTTP Request and Webhook nodes cover both directions, and every Warmbly endpoint is documented in the [API reference](/api/endpoints/). + +## Connecting + +Create a scoped [API key](/api/authentication/) in Settings > API keys with only the scopes the workflow needs (contacts and campaigns for lead intake; add inbox for replies). In n8n, add a **Header Auth** credential with the name `Authorization` and the value `Bearer `, and attach it to every HTTP Request node that calls Warmbly. + +## Warmbly to n8n + +To start a workflow from a Warmbly event, add a **Webhook** node, copy its production URL, and either: + +- register it as a [webhook endpoint](/guides/webhooks/) subscribed to the events you want (signed with HMAC so the workflow can verify the sender), or +- add an n8n connection on the Integrations page and use it as a **Send a webhook** action inside an [automation](/guides/automations/), which lets you filter and branch before the request leaves Warmbly. + +Both deliver the same JSON body: a delivery id, the event type and the full event data. + +## n8n to Warmbly + +Two routes, depending on where you want the field mapping to live. + +**Mapping in n8n.** An HTTP Request node calling `POST /api/v1/contacts` with the contact's fields, `categories` and `campaigns`. The write is an upsert by email, so re-running a workflow updates the contact instead of duplicating it. + +**Mapping in Warmbly.** Create an automation with the **Inbound webhook** trigger, copy its URL, and POST the raw payload to it from an HTTP Request node. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates (`{{.email}}`, `{{.answers.company}}`), tags the contact and enrols it in a campaign, and the same flow can notify Slack or open a task. See [Lead intake](/guides/automations/#lead-intake). This route needs no API key: the URL is the credential. + +## Lead forms and other lead sources + +n8n's *Facebook Lead Ads Trigger* node fires on a new Facebook or Instagram submission. LinkedIn Lead Gen Forms and TikTok Lead Generation have no dedicated node; point their lead notification webhooks at an n8n *Webhook* node instead. Connect either to one of the routes above and the ad's answers become a tagged, campaign-enrolled contact that your mailbox follows up with. The follow-up still runs under each mailbox's daily cap and spacing, so a burst of leads queues rather than going out at once. + +## Good to know + +- **Sending respects your limits.** Sends and replies made through the API honor each mailbox's daily cap and spacing. +- **Rate limits** apply per API key; a large backfill should go through the [import endpoints](/api/reference/contacts/#commit-an-import) rather than one request per row. +- For the lowest latency without hosting a URL, subscribe to the [developer websocket](/api/realtime/) instead of a webhook. diff --git a/docs/content/docs/guides/segments.mdx b/docs/content/docs/guides/segments.mdx index a79b614f..d34e1fd4 100644 --- a/docs/content/docs/guides/segments.mdx +++ b/docs/content/docs/guides/segments.mdx @@ -51,7 +51,9 @@ Sequences can pin too: the **Add to segment** and **Remove from segment** action - **Browse**: a segment page lists its current members with the same table, filters, detail drawer and bulk actions as the contacts page. - **Add to campaign**: enrols every current member as a lead of the campaign you pick, from the segment page or with **From segment** on a campaign's Leads tab. Contacts already in that campaign are skipped, and a running campaign wakes up to schedule the new leads. This is a snapshot: contacts who join the segment later are not added until you run it again. To keep a campaign fed automatically, [link the segment](#linking-a-segment-to-a-campaign) instead. - **Link to campaign**: attaches the segment to a campaign as a live audience, so contacts who join the segment later become leads on their own. See [below](#linking-a-segment-to-a-campaign). +- **One-time email**: send one message to the segment without building a sequence. **Campaigns** > **New campaign** > **One-time email** picks the segments, shows how many contacts they resolve to and how long the mailbox pool needs, and sends now or on a date. See [one-time emails](/guides/campaigns/#one-time-emails). - **Filter**: the contact list's filter bar has a **Segment** pill, and the same scope carries into an export. +- **Export**: **Export** on a segment page downloads its members as CSV, XLSX or JSON, either every member, the ones matching the page's filters, or the selected rows. - **Duplicate**: the segment menu copies a definition to start a variation from. - **Search and export**: the contact search and export accept `segment_ids`, so anything that takes a contact filter can be scoped to a segment. @@ -59,14 +61,16 @@ Sequences can pin too: the **Add to segment** and **Remove from segment** action Where **Add to campaign** copies today's members once, a linked segment is a live audience: the campaign keeps enrolling whoever the segment says belongs. Manage the links from the **Segments** button on a campaign's Leads tab; a campaign can link up to 20 segments. -Linking enrols every current member as a lead immediately. From then on, any contact who enters the segment, a new contact, an import, a condition edit, a manual pin-in, or simply drifting into a date or engagement condition, is enrolled automatically; a background check picks up drift about every 2 minutes. +Linking enrols every current member as a lead immediately and turns on the campaign's **Keep running for new leads** setting, so the campaign waits for members instead of finishing when it runs out (turn it off in the campaign's preferences if you want it to finish). From then on, any contact who enters the segment, a new contact, an import, a condition edit, a manual pin-in, or simply drifting into a date or engagement condition, is enrolled automatically; a background check picks up drift about every 2 minutes. + +The Leads tab shows every linked segment in a **Linked segments** strip: the segment's name, how many of its members are leads (`120/120`), a **no contacts** mark when it currently matches nobody, and a **held out** count when members were removed from the campaign by hand. Clicking a chip filters the list to that segment's leads; **Add back** re-enrols held-out members. Enrolment is additive. A contact who later falls out of the segment keeps their lead row and their progress; the link only ever adds. Detaching a segment stops future enrolment but leaves the leads it already added in place. Removing a lead from the campaign by hand is respected too: automatic enrolment will not re-add that contact, even while they still match a linked segment, until you add them back yourself. The campaign reacts to growth by status: -- **Active**: woken up to schedule the new leads straight away. -- **Finished**: restarted automatically, through the same launch checks as pressing play. +- **Active**, including one **waiting for leads**: woken up to schedule the new leads straight away. +- **Finished**: restarted automatically, through the same launch checks as pressing play. A refused restart is noted in the campaign's activity log. - **Paused** (or draft): the leads accumulate and send when the campaign runs. A segment linked to a campaign cannot be deleted; detach it from the campaign first. diff --git a/docs/content/docs/guides/sequences.mdx b/docs/content/docs/guides/sequences.mdx index 88c11d07..a34576f7 100644 --- a/docs/content/docs/guides/sequences.mdx +++ b/docs/content/docs/guides/sequences.mdx @@ -20,12 +20,16 @@ Each card has two source dots: the **bottom dot** is a plain "go there next" con A step has an internal **name** (never seen by recipients), a **subject** with a variable menu for merge fields like `{{.FirstName}}`, and a rich-text **body**. -The Preview tab renders through the real send engine against a sample contact, so merge fields, conditionals, and spintax resolve exactly as they will at send time. Malformed templates (an `{{if}}` with no `{{end}}`) are flagged before you start the campaign. See [Personalization](/guides/expressions/) for everything you can put in copy. +The Preview tab renders through the real send engine, so merge fields, conditionals, and spintax resolve exactly as they will at send time. **Preview as** picks who it renders for: a built-in sample contact, one of the campaign's leads, or any contact you search for, so a custom field your list does not actually have shows up as an unresolved token instead of looking fine. The mailbox picker next to it adds that sender's signature and shows the From name recipients will see. When the campaign is known the preview also appends the opt-out footer and lists the files attached to the campaign. Malformed templates (an `{{if}}` with no `{{end}}`) are flagged before you start the campaign. See [Personalization](/guides/expressions/) for everything you can put in copy. + +**Send test** in the preview mails the saved step to an address of your choice through the chosen mailbox, rendered for the chosen contact and carrying the attachments, signature and opt-out footer. Save the step first: the test sends what is stored, not unsaved edits. Opens and clicks on a test are not tracked, and its opt-out link names nobody, so clicking it suppresses no one. Apply a saved template to a step, or save a step as a reusable one from the composer. Templates carry their subject and body and live in your shared library. +Upload attachments for a step below its composer by dragging and dropping files or clicking the upload area. A file uploaded there is sent with every email from that step and from no other step, and can be removed in the same place. Files attached to the campaign itself rather than to a step, which can only be added through the API, ride every step: they are listed under **Sent with every step** so each step shows everything it carries. Attachments count against your workspace storage limit, shown under **Settings > Billing**; an upload that would pass it is refused with `storage_limit_reached`, and two uploads racing for the last of the quota cannot both get in. Removing a step deletes the files scoped to it. + ### Action steps A step can perform an action instead of sending: add or remove a tag, label email, create a task, create a deal or move its stage, unsubscribe the contact, notify (fires your webhooks and integrations), run an automation, or switch. They connect like email steps and work best at the end of a reply branch, for example creating a deal and notifying your team on a positive reply. diff --git a/docs/content/docs/guides/unsubscribe.mdx b/docs/content/docs/guides/unsubscribe.mdx new file mode 100644 index 00000000..6afc440c --- /dev/null +++ b/docs/content/docs/guides/unsubscribe.mdx @@ -0,0 +1,70 @@ +--- +title: Unsubscribe and suppression +description: How recipients opt out of campaign email, what Warmbly does when they do, and how the suppression list works. +--- + +Every campaign email gives its recipient a way to stop hearing from you, and every opt-out is honoured automatically across the whole workspace. This page covers the three mechanisms, the suppression list behind them, and the settings that shape what a recipient sees. + +## What a recipient sees + +Warmbly appends an opt-out to every campaign email, after the signature. There are three modes, set for the workspace under **Settings > Sending > Unsubscribe** and overridable per campaign under **Preferences**: + +| Mode | What is appended | Default | +| --- | --- | --- | +| Reply to opt out | A plain sentence such as "If this isn't relevant, just reply and let me know and I won't email you again." | Yes | +| Unsubscribe link | A sentence with a real link, for example "Not the right person, or not interested? Unsubscribe" | | +| Nothing | No opt-out in the body | | + +The wording of the sentence and of the link text is yours to change. + +**Reply to opt out** is the default because it reads as a personal email, which is what cold outreach is. A formal unsubscribe link and footer are the strongest signal a mailbox provider has that a message is bulk marketing, and several deliverability teams report worse placement for cold email that carries one. A reply that asks to stop is detected and honoured automatically (see below), so the plain sentence is a real mechanism, not a courtesy. CAN-SPAM, CASL and the Australian Spam Act all accept a reply as the opt-out method. + +**Unsubscribe link** is the right choice when your list skews toward consumers, when your legal team asks for a link, or when your volume is high enough that provider bulk-sender rules apply. The link is unique to the recipient and campaign, signed so it cannot be guessed or altered, and valid for a year after the send. Clicking it opens a plain confirmation page with one button. Nothing happens until the button is pressed, because link scanners and preview fetchers follow every link in an email. The page then offers a way back for anyone who unsubscribed by mistake. + +You can also place the link inside your own copy instead of the footer: insert the **Unsubscribe link** variable from the variable menu, or type `{{.UnsubscribeLink}}`. Click tracking never rewrites it, so an opt-out is never counted as a click. + +## The List-Unsubscribe header + +Independently of the body, each campaign can attach the `List-Unsubscribe` and `List-Unsubscribe-Post` headers (RFC 8058). Mail clients that recognise them show their own **Unsubscribe** control next to the sender's name, and pressing it opts the recipient out with no page at all. The toggle is **Preferences > Unsubscribe header**, on by default. + +Two things to know. Gmail only shows its button for mail it already classes as bulk, so a one-to-one style cold email from a Workspace mailbox usually does not get one. And Google and Yahoo require the header only above roughly five thousand messages a day to their consumer inboxes, which no mailbox at Warmbly's default limits approaches. The header costs nothing and is worth leaving on; it is not a substitute for the opt-out line. + +## Replies that ask to stop + +Warmbly reads every reply to a campaign email. One that asks to stop, in wording such as "unsubscribe", "remove me", "stop emailing me", "opt out" or "do not contact", puts the sender on the suppression list immediately. Only the new text of the reply is read: the quoted history below it carries your own opt-out wording and is ignored. Phrases match on whole words, so "stop by our booth" does not opt anyone out. + +A reply that says "not interested" is classed as negative and can stop the sequence, but it does not suppress the contact. Someone who says no today may be a fit next year; someone who says stop is asking not to be contacted at all. + +Turn this off with **Settings > Sending > Honour replies that ask to stop** if you would rather handle such replies by hand. Leaving it off with the reply-to-opt-out line in place means promising an opt-out you then honour manually. + +## The suppression list + +**Contacts > Suppression list** is every address and domain that no campaign in the workspace will email, however it got there: + +| Source | How it got there | +| --- | --- | +| Unsubscribed | Clicked the link, pressed the mail client's button, or replied asking to stop | +| Spam complaint | The recipient's provider sent a complaint report for a message you sent | +| Bounced | A permanent delivery failure for the address | +| Added by hand | Someone on the team added it | +| Imported | Pasted in as part of a list | + +The list is workspace-wide. An entry stops the address in every campaign, in new campaigns created later, and in the unibox composer, which refuses to send to it. Import a list containing a suppressed address and the contact is created but never mailed. The [launch check](/guides/campaigns/#the-launch-check-on-your-list) counts suppressed leads out of a campaign's deliverable total, so what the campaign says it will send is what it sends. + +**Adding entries.** Press **Add** and paste addresses or domains, one per line or as a column from a spreadsheet. A bare domain (`acme.com` or `@acme.com`) suppresses every address at it, which is the usual way to keep customers, partners and your own company out of outreach. An optional reason is kept with each entry. + +**Removing entries.** Each entry has a **Remove** action, and a suppressed contact's drawer shows the same. Removing an entry the recipient made themselves, by unsubscribing, complaining or bouncing, is confirmed with a stronger warning and recorded in the audit log with who lifted it and why the address was listed. Removing an address also restores the contact's **Subscribed** flag, so the two never disagree. + +**Contacts and the Subscribed flag.** The contact's own **Subscribed** toggle is a second gate: a contact switched off is skipped by every campaign even with no suppression entry. An opt-out sets both, so a contact who unsubscribed reads as unsubscribed everywhere. + +Suppression entries travel with a [workspace export](/guides/workspace-export-import/), so a workspace moved to another instance does not re-mail people who already opted out. + +## Webhooks and the API + +An opt-out fires the `campaign.unsubscribed` [webhook](/guides/webhooks/) with a `source` of `one_click` (the mail client's button), `link` (the link in the email), `reply` (a reply asking to stop) or `action` (a sequence's **Unsubscribe** step). + +The suppression list is available over the API as `GET /suppressions`, `POST /suppressions` and `DELETE /suppressions/:id`. See [deliverability and ops](/api/reference/deliverability-ops/#list-the-suppression-list). + +## Self-hosting + +Unsubscribe links are served by the API process on the origin in `API_PUBLIC_URL`, so that variable must be set to the address recipients can reach. Without it Warmbly cannot mint links: the header is left off, and the link mode falls back to the reply-to-opt-out sentence. Links are signed under `AUTH_SECRET`; rotating it invalidates links in emails already sent. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index a408b0e9..42d4c03c 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -5,6 +5,10 @@ description: "Move a whole workspace between Warmbly instances, including mailbo A workspace archive is a single file holding everything one workspace owns. It exists so you can move between instances: a self-hosted install to the cloud, the cloud back to self-hosted, or one self-host to another. + +This page moves **one workspace** between two running instances, re-sealing its secrets for the destination's keys. To move an entire self-hosted install (every workspace, its users, its platform admins) use `warmblyctl backup` and `warmblyctl restore`, which carry the database, the blob root and the encryption keys as one bundle. See [data control](/development/data-control/#backups). The two are not interchangeable: a bundle cannot be applied to a single workspace, and this archive cannot restore an instance. + + Everything here lives under **Settings > Data**, and is limited to the workspace owner. An export with credentials contains every mailbox password in the workspace, so it sits at the same level as deleting the workspace. ## What an archive contains @@ -15,7 +19,7 @@ The data is split into groups. Every export includes **Workspace**; the rest are |-------|----------| | Workspace | The organization, members, roles, teams, mailboxes, API keys, webhooks, and settings, including the website tracking site key. Always included | | Contacts | Contacts, categories, segments with their manual overrides, forms with their images, submissions, personalized link tickets and funnel events, notes, activities, and the suppression list | -| Campaigns | Campaigns, sequences, senders, linked segments, attachments, and per-campaign settings | +| Campaigns | Campaigns, sequences, senders, linked segments, attachments, per-campaign settings, and each lead's step progress with its per-link clicks and per-event opens | | CRM | Pipelines, deals, tasks, and meeting bookings | | Automations | Automations, connected integrations, and lead sync sources | | Assistant | Assistant sessions and messages, skills, MCP servers, and AI settings | diff --git a/docs/content/docs/guides/zapier.mdx b/docs/content/docs/guides/zapier.mdx index cfeb2e2e..6d8f3d25 100644 --- a/docs/content/docs/guides/zapier.mdx +++ b/docs/content/docs/guides/zapier.mdx @@ -51,6 +51,17 @@ Find Contact (by exact email), Find Deal, Find CRM Task, Find Campaign, Find Mai Pair a search with an action for idempotent flows: Find Contact, then Create or Update Contact. +## Lead forms and other lead sources + +Facebook and Instagram Lead Ads, LinkedIn Lead Gen Forms and TikTok Lead Generation all deliver new form submissions to Zapier in real time, which makes Zapier the shortest route from an ad to a follow-up sent from your own mailbox. + +1. **Trigger:** *Facebook Lead Ads: New Lead* (or the LinkedIn or TikTok equivalent), picking the Page and form. +2. **Action:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the categories. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. + +Prefer to keep the mapping inside Warmbly? Use *Webhooks by Zapier: POST* to an [inbound webhook automation](/guides/automations/#lead-intake) instead, sending the lead as JSON. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates, and the same flow can tag, notify Slack and open a task. + +The follow-up still runs through the campaign's mailboxes, so a burst of leads queues under each mailbox's daily cap and spacing rather than going out at once. Cold email tools do not ship a native Meta connector; this route is what their customers use too. + ## Good to know - **Sending respects your limits.** Send Email and Reply in Inbox honor each mailbox's daily cap and spacing, and support smart and scheduled send modes. diff --git a/docs/content/docs/learn/cold-email-rules.mdx b/docs/content/docs/learn/cold-email-rules.mdx index 9da3bada..5ece0871 100644 --- a/docs/content/docs/learn/cold-email-rules.mdx +++ b/docs/content/docs/learn/cold-email-rules.mdx @@ -73,8 +73,9 @@ Requires consent (express, inferred or implied), accurate sender identification ## How Warmbly helps - Sender profiles include physical address fields that render in every send. -- Opt-out detection on incoming replies (STOP, unsubscribe, "please remove me"). -- Workspace-wide suppression list shared across [sequences](/guides/sequences/). +- Every campaign email carries an opt-out: a reply-to-opt-out line by default, or an unsubscribe link, plus optional RFC 8058 headers. See [unsubscribe and suppression](/guides/unsubscribe/). +- Opt-out detection on incoming replies (unsubscribe, "remove me", "stop emailing me"). +- Workspace-wide suppression list shared across [sequences](/guides/sequences/), with addresses and whole domains added by hand. - Contact source captured at [import](/guides/contacts-crm/); preserved through merges. - Region-aware sending windows so you don't accidentally blast at 3am local time. diff --git a/docs/public/openapi.json b/docs/public/openapi.json index 1e9409fa..e457030d 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -4286,7 +4286,7 @@ "step_id": { "type": "string", "format": "uuid", - "description": "Scope the attachment to one sequence step." + "description": "Scope the attachment to one sequence step of this campaign, which is then the only step that sends it. Omit to attach the file to every step. A step of another campaign is 404." } } } @@ -6793,7 +6793,7 @@ ], "operationId": "contacts_timeline_list", "summary": "List a contact's timeline", - "description": "Merged activity feed: sends, opens, clicks, replies, bounces, deliverability/suppression events, notes, and meeting bookings. Requires a selected organization. Paginate via the `before` timestamp. Scope `READ_CONTACTS`.", + "description": "Merged activity feed: sends, opens, clicks, replies, bounces, deliverability/suppression events, notes, meeting bookings, lifecycle events, and website page views. Requires a selected organization. Paginate with the opaque `cursor` from `pagination.next_cursor`; the cursor carries the exact position of the last event (time, source, row), so events that share a timestamp are never skipped or repeated. Scope `READ_CONTACTS`.", "security": [ { "bearerAuth": [] @@ -6821,11 +6821,21 @@ "default": 50 } }, + { + "name": "cursor", + "in": "query", + "required": false, + "description": "Opaque pagination cursor from `pagination.next_cursor`. A malformed cursor is a 400.", + "schema": { + "type": "string" + } + }, { "name": "before", "in": "query", "required": false, - "description": "The `at` timestamp of the oldest event from the previous page (RFC 3339 nano).", + "deprecated": true, + "description": "Deprecated: use `cursor`. Returns the events strictly older than this timestamp (RFC 3339 nano), which can skip events that share an instant with the page boundary. Ignored when `cursor` is set.", "schema": { "type": "string", "format": "date-time" @@ -6834,7 +6844,7 @@ ], "responses": { "200": { - "description": "Timeline events with a has_more flag.", + "description": "A page of timeline events with the pagination envelope.", "content": { "application/json": { "schema": { @@ -6844,7 +6854,7 @@ } }, "400": { - "description": "Invalid contact ID, or no organization selected.", + "description": "Invalid contact ID, limit, cursor or before timestamp, or no organization selected.", "content": { "application/json": { "schema": { @@ -18151,6 +18161,258 @@ } } } + }, + "/suppressions": { + "get": { + "tags": [ + "deliverability-ops" + ], + "operationId": "deliverability-ops_list_suppressions", + "summary": "List the suppression list", + "description": "Pages the workspace suppression list newest first: every address and domain no campaign will email. Scope `READ_CONTACTS`, org permission `view_contacts`.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "q", + "in": "query", + "required": false, + "description": "Substring filter on the address or domain.", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "description": "Page size, 1 to 200. Default 50.", + "schema": { + "type": "integer" + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "description": "Opaque pagination cursor from the previous page's pagination.next_cursor.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A page of suppression entries.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuppressionListResult" + } + } + } + }, + "400": { + "description": "Invalid limit or cursor.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "API key lacks READ_CONTACTS or the member lacks view_contacts.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + }, + "post": { + "tags": [ + "deliverability-ops" + ], + "operationId": "deliverability-ops_add_suppressions", + "summary": "Add to the suppression list", + "description": "Adds addresses and domains. Unparseable values are reported in `skipped`; existing entries are updated in place, so the call is safe to repeat. Scope `WRITE_CONTACTS`, org permission `manage_contacts`.", + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddSuppressionsRequest" + } + } + } + }, + "responses": { + "200": { + "description": "What the request did.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddSuppressionsResult" + } + } + } + }, + "400": { + "description": "No entries, or more than 5000.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "API key lacks WRITE_CONTACTS or the member lacks manage_contacts.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/suppressions/{id}": { + "delete": { + "tags": [ + "deliverability-ops" + ], + "operationId": "deliverability-ops_remove_suppression", + "summary": "Remove from the suppression list", + "description": "Lifts one entry so campaigns can email the address (or every address at the domain) again. Recorded in the audit log. Scope `WRITE_CONTACTS`, org permission `manage_contacts`.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "Entry removed." + }, + "400": { + "description": "Invalid suppression ID.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "API key lacks WRITE_CONTACTS or the member lacks manage_contacts.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "No such entry in this organization.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } } }, "components": { @@ -20478,6 +20740,7 @@ "required": [ "id", "campaign_id", + "step_id", "filename", "created_at" ], @@ -20495,7 +20758,8 @@ "string", "null" ], - "format": "uuid" + "format": "uuid", + "description": "The sequence step that sends this file. Null means every step of the campaign sends it." }, "filename": { "type": "string" @@ -20781,6 +21045,32 @@ "type": "boolean" } } + }, + "unsubscribe": { + "type": "object", + "description": "Workspace default for the opt-out appended after the signature of every campaign email; a campaign's unsubscribe_mode overrides it.", + "properties": { + "mode": { + "type": "string", + "enum": [ + "text", + "link", + "off" + ] + }, + "text": { + "type": "string", + "maxLength": 300 + }, + "link_intro": { + "type": "string", + "maxLength": 300 + }, + "link_text": { + "type": "string", + "maxLength": 300 + } + } } } }, @@ -20938,6 +21228,26 @@ "body_plain": { "type": "string" }, + "contact_id": { + "type": "string", + "format": "uuid", + "description": "A contact of the organization to render for. Omitted renders the built-in sample. Requires READ_CONTACTS." + }, + "campaign_id": { + "type": "string", + "format": "uuid", + "description": "Campaign whose opt-out footer, plain-text setting and attachments apply." + }, + "account_id": { + "type": "string", + "format": "uuid", + "description": "Mailbox whose signature applies and which is reported as the sender." + }, + "step_id": { + "type": "string", + "format": "uuid", + "description": "The step being previewed, so the attachment list is the one that step sends. Omitted lists the campaign-wide files only." + }, "contact": { "type": "object", "description": "Override fields on the built-in sample contact.", @@ -20993,6 +21303,42 @@ "type": "string" }, "description": "Literal {{...}} tokens left after render. Omitted when empty." + }, + "from": { + "type": "object", + "description": "The sender as recipients see it. Present when account_id was given.", + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + } + } + }, + "attachments": { + "type": "array", + "description": "The files this send carries: the campaign-wide attachments plus step_id's own. Present when campaign_id was given and there are any.", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "filename": { + "type": "string" + }, + "size": { + "type": "integer", + "description": "Size in bytes." + }, + "mime_type": { + "type": "string" + } + } + } } } }, @@ -22027,7 +22373,14 @@ "note", "meeting_booked", "meeting_rescheduled", - "meeting_canceled" + "meeting_canceled", + "contact_created", + "campaign_added", + "campaign_removed", + "category_added", + "category_removed", + "form_submitted", + "page_hit" ] }, "at": { @@ -22104,15 +22457,100 @@ "type": "string", "format": "uuid", "description": "Note author." + }, + "machine": { + "type": [ + "boolean", + "null" + ], + "description": "email_opened / email_clicked: true when an automated fetcher (a mail privacy proxy, a security gateway following links) did it rather than the person." + }, + "machine_reason": { + "type": [ + "string", + "null" + ], + "description": "Which rule caught a machine open or click: prefetch, instant or burst. Absent on an open summarised from the lead alone." + }, + "link": { + "oneOf": [ + { + "$ref": "#/components/schemas/ContactLinkClick" + }, + { + "type": "null" + } + ], + "description": "email_clicked: the exact link clicked. Absent on clicks recorded before per-link attribution." + }, + "origin": { + "oneOf": [ + { + "$ref": "#/components/schemas/EngagementOrigin" + }, + { + "type": "null" + } + ], + "description": "email_opened / email_clicked: where the event came from, when it was logged per event." + }, + "category_id": { + "type": [ + "string", + "null" + ], + "format": "uuid", + "description": "category_added / category_removed: the category, as it was at the time." + }, + "category_title": { + "type": [ + "string", + "null" + ], + "description": "category_added / category_removed: the category's title at the time." + }, + "source_detail": { + "type": [ + "string", + "null" + ], + "description": "contact_created: the file, campaign, sheet or API key name behind `source`." + }, + "form_id": { + "type": [ + "string", + "null" + ], + "format": "uuid", + "description": "form_submitted: the hosted form." + }, + "form_name": { + "type": [ + "string", + "null" + ], + "description": "form_submitted: the form's name." + }, + "page_hit": { + "oneOf": [ + { + "$ref": "#/components/schemas/WebsitePageHit" + }, + { + "type": "null" + } + ], + "description": "page_hit: the page view from the website tracking snippet. `subject` carries the page title, or its path when untitled." } } }, "ContactTimelineResult": { "type": "object", - "description": "A page of timeline events. Paginate via has_more and the `before` query param; this list does not use a cursor envelope.", + "description": "A page of timeline events. Paginate with pagination.next_cursor as the `cursor` query param. has_more mirrors pagination.has_more and is kept for older clients.", "required": [ "data", - "has_more" + "has_more", + "pagination" ], "properties": { "data": { @@ -22123,6 +22561,9 @@ }, "has_more": { "type": "boolean" + }, + "pagination": { + "$ref": "#/components/schemas/Pagination" } } }, @@ -28230,6 +28671,325 @@ } } } + }, + "SuppressedRecipient": { + "type": "object", + "description": "One entry on the workspace suppression list. A domain entry keeps the bare host in `email` and matches every address at it.", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "organization_id": { + "type": "string", + "format": "uuid" + }, + "email": { + "type": "string", + "description": "The address, or the bare domain when kind is domain." + }, + "kind": { + "type": "string", + "enum": [ + "email", + "domain" + ] + }, + "reason": { + "type": "string" + }, + "source": { + "type": "string", + "enum": [ + "bounce", + "complaint", + "unsubscribe", + "manual", + "import" + ] + }, + "campaign_id": { + "type": "string", + "format": "uuid", + "nullable": true + }, + "expires_at": { + "type": "string", + "format": "date-time", + "nullable": true + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + }, + "SuppressionListResult": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SuppressedRecipient" + } + }, + "pagination": { + "type": "object", + "properties": { + "next_cursor": { + "type": "string", + "nullable": true + }, + "has_more": { + "type": "boolean" + } + } + } + } + }, + "AddSuppressionsRequest": { + "type": "object", + "required": [ + "entries" + ], + "properties": { + "entries": { + "type": "array", + "maxItems": 5000, + "items": { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string", + "description": "An address, or a bare domain (with or without a leading @)." + }, + "reason": { + "type": "string" + } + } + } + }, + "reason": { + "type": "string", + "description": "Applied to every entry without its own reason." + } + } + }, + "AddSuppressionsResult": { + "type": "object", + "properties": { + "added": { + "type": "integer" + }, + "skipped": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Values that were neither a valid address nor a valid domain." + } + } + }, + "ContactLinkClick": { + "type": "object", + "description": "The link behind an email_clicked event: where it went, the anchor text it was minted from, and the UTM parameters the destination carried.", + "required": [ + "id", + "url" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "url": { + "type": "string" + }, + "label": { + "type": "string", + "description": "Anchor text the link was minted from." + }, + "utm_source": { + "type": "string" + }, + "utm_medium": { + "type": "string" + }, + "utm_campaign": { + "type": "string" + }, + "utm_term": { + "type": "string" + }, + "utm_content": { + "type": "string" + }, + "user_agent": { + "type": "string" + } + } + }, + "EngagementOrigin": { + "type": "object", + "description": "Where an open or click came from. Every field is omitted when unknown; the source address itself is never stored.", + "properties": { + "client": { + "type": "string", + "description": "Mail client or image proxy named by the user agent (Gmail, Apple Mail, Outlook)." + }, + "device_type": { + "type": "string" + }, + "os": { + "type": "string" + }, + "browser": { + "type": "string" + }, + "browser_version": { + "type": "string" + }, + "country_code": { + "type": "string" + }, + "region": { + "type": "string" + }, + "city": { + "type": "string" + } + } + }, + "WebsitePageHit": { + "type": "object", + "description": "A page view reported by the website tracking snippet and tied to the contact through an email-link ticket.", + "required": [ + "id", + "visitor_id", + "session_key", + "occurred_at", + "url", + "path", + "title", + "referrer", + "referrer_domain", + "landing", + "utm_source", + "utm_medium", + "utm_campaign", + "utm_term", + "utm_content", + "device_type", + "os", + "browser", + "browser_version", + "device_brand", + "language", + "timezone", + "screen_width", + "screen_height", + "country_code", + "region", + "city" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "visitor_id": { + "type": "string", + "format": "uuid" + }, + "session_key": { + "type": "string" + }, + "occurred_at": { + "type": "string", + "format": "date-time" + }, + "url": { + "type": "string" + }, + "path": { + "type": "string" + }, + "title": { + "type": "string" + }, + "referrer": { + "type": "string" + }, + "referrer_domain": { + "type": "string" + }, + "landing": { + "type": "boolean", + "description": "True for the first view of a session." + }, + "utm_source": { + "type": "string" + }, + "utm_medium": { + "type": "string" + }, + "utm_campaign": { + "type": "string" + }, + "utm_term": { + "type": "string" + }, + "utm_content": { + "type": "string" + }, + "device_type": { + "type": "string" + }, + "os": { + "type": "string" + }, + "browser": { + "type": "string" + }, + "browser_version": { + "type": "string" + }, + "device_brand": { + "type": "string" + }, + "language": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "screen_width": { + "type": "integer" + }, + "screen_height": { + "type": "integer" + }, + "country_code": { + "type": "string" + }, + "region": { + "type": "string" + }, + "city": { + "type": "string" + } + } } } } diff --git a/go.mod b/go.mod index 26b10d7c..df64c994 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.19.1 github.com/golangci/golangci-lint v1.64.8 github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.0 github.com/invopop/jsonschema v0.13.0 github.com/jackc/pgx/v5 v5.9.0 github.com/meszmate/apple-go v0.0.0-20250828163208-7fea48c91b32 @@ -36,23 +37,26 @@ require ( github.com/oschwald/geoip2-golang/v2 v2.0.0 github.com/redis/go-redis/v9 v9.11.0 github.com/rs/zerolog v1.34.0 + github.com/spf13/cobra v1.9.1 github.com/stripe/stripe-go/v76 v76.25.0 github.com/xuri/excelize/v2 v2.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.55.0 golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 - google.golang.org/api v0.260.0 - google.golang.org/grpc v1.82.1 + golang.org/x/term v0.45.0 + google.golang.org/api v0.264.0 + google.golang.org/grpc v1.83.1 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 google.golang.org/protobuf v1.36.11 + gopkg.in/yaml.v3 v3.0.1 ) require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect cloud.google.com/go v0.121.6 // indirect - cloud.google.com/go/auth v0.18.0 // indirect + cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.3 // indirect @@ -159,7 +163,7 @@ require ( github.com/google/go-tpm v0.9.8 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect - github.com/googleapis/gax-go/v2 v2.16.0 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect @@ -261,7 +265,6 @@ require ( github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spf13/viper v1.12.0 // indirect @@ -305,9 +308,9 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect @@ -321,12 +324,11 @@ require ( golang.org/x/tools v0.49.0 // indirect golang.org/x/tools/go/expect v0.1.1-deprecated // indirect golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect - google.golang.org/genproto v0.0.0-20260114163908-3f89685c29c3 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.6.1 // indirect mvdan.cc/gofumpt v0.7.0 // indirect mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect diff --git a/go.sum b/go.sum index 992d0bfa..4538dc55 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= -cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= -cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/cloudtasks v1.13.7 h1:H2v8GEolNtMFfYzUpZBaZbydqU7drpyo99GtAgA+m4I= @@ -15,8 +15,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= -cloud.google.com/go/kms v1.23.2 h1:4IYDQL5hG4L+HzJBhzejUySoUOheh3Lk5YT4PCyyW6k= -cloud.google.com/go/kms v1.23.2/go.mod h1:rZ5kK0I7Kn9W4erhYVoIRPtpizjunlrfU4fUkumUp8g= +cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ= +cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM= @@ -456,8 +456,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= -github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= -github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s= github.com/gordonklaus/ineffassign v0.1.0/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= @@ -983,8 +983,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0. go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= @@ -997,14 +997,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqhe go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= @@ -1178,26 +1178,26 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.260.0 h1:XbNi5E6bOVEj/uLXQRlt6TKuEzMD7zvW/6tNwltE4P4= -google.golang.org/api v0.260.0/go.mod h1:Shj1j0Phr/9sloYrKomICzdYgsSDImpTxME8rGLaZ/o= +google.golang.org/api v0.264.0 h1:+Fo3DQXBK8gLdf8rFZ3uLu39JpOnhvzJrLMQSoSYZJM= +google.golang.org/api v0.264.0/go.mod h1:fAU1xtNNisHgOF5JooAs8rRaTkl2rT3uaoNGo9NS3R8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20260114163908-3f89685c29c3 h1:rUamZFBwsWVWg4Yb7iTbwYp81XVHUvOXNdrFCoYRRNE= -google.golang.org/genproto v0.0.0-20260114163908-3f89685c29c3/go.mod h1:wE6SUYr3iNtF/D0GxVAjT+0CbDFktQNssYs9PVptCt4= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 h1:/WILD1UcXj/ujCxgoL/DvRgt2CP3txG8+FwkUbb9110= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1/go.mod h1:YNKnb2OAApgYn2oYY47Rn7alMr1zWjb2U8Q0aoGWiNc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/internal/api/handler/access.go b/internal/api/handler/access.go new file mode 100644 index 00000000..34c80577 --- /dev/null +++ b/internal/api/handler/access.go @@ -0,0 +1,51 @@ +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// hasAccess is middleware.RequireAccess as an inline check, for a permission a +// handler needs only when an optional request field is present. +func (h *Handler) hasAccess(c *gin.Context, orgPerm models.OrganizationPermission, apiPerm uint64) *errx.Error { + switch middleware.GetAuthType(c) { + case middleware.AuthTypeAPIKey, middleware.AuthTypeOAuth: + if !models.HasAPIPermission(middleware.GetAPIKeyPermissions(c), apiPerm) { + return errx.New(errx.Forbidden, "insufficient API key permissions") + } + return nil + default: + if h.OrganizationService == nil { + return nil + } + userID, err := middleware.GetUserUUID(c) + if err != nil { + return errx.ErrUnauthorized + } + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + return errx.ErrNoOrganization + } + has, xerr := h.OrganizationService.HasPermission(c.Request.Context(), *orgID, userID, orgPerm) + if xerr != nil { + return xerr + } + if !has { + return errx.ErrForbidden + } + return nil + } +} + +// mailboxAllowed enforces an API key's mailbox allow-list on an account id +// taken from a request body, the way RequireAPIKeyEmailAccountParam does for +// a route parameter. +func mailboxAllowed(c *gin.Context, accountID uuid.UUID) *errx.Error { + if !middleware.APIKeyAllowsEmailAccount(c, accountID) { + return errx.New(errx.Forbidden, "email account is not allowed for this API key") + } + return nil +} diff --git a/internal/api/handler/admin_instance.go b/internal/api/handler/admin_instance.go index 6f658619..c10ab46a 100644 --- a/internal/api/handler/admin_instance.go +++ b/internal/api/handler/admin_instance.go @@ -56,12 +56,21 @@ func (h *Handler) AdminInstanceHealth(c *gin.Context) { } // AdminGetInstanceSettings returns the database-backed settings document. +// +// Notification channel targets and secrets never leave the process in full: a +// chat webhook URL is a bearer credential, so it reads back as a recognisable +// preview and the panel sends the preview (or nothing) to mean "unchanged". func (h *Handler) AdminGetInstanceSettings(c *gin.Context) { if h.InstanceSettings == nil { - c.JSON(http.StatusOK, instancesettings.Defaults()) + c.JSON(http.StatusOK, redactSettings(instancesettings.Defaults())) return } - c.JSON(http.StatusOK, h.InstanceSettings.Get(c.Request.Context())) + c.JSON(http.StatusOK, redactSettings(h.InstanceSettings.Get(c.Request.Context()))) +} + +func redactSettings(doc instancesettings.Document) instancesettings.Document { + doc.Notifications.Channels = doc.Notifications.RedactedChannels() + return doc } // AdminPutInstanceSettings validates, clamps and stores the settings document. @@ -103,7 +112,7 @@ func (h *Handler) AdminPutInstanceSettings(c *gin.Context) { ) } - c.JSON(http.StatusOK, doc) + c.JSON(http.StatusOK, redactSettings(doc)) } func instanceSettingsAuditDetails(doc instancesettings.Document) map[string]any { @@ -115,8 +124,12 @@ func instanceSettingsAuditDetails(doc instancesettings.Document) map[string]any "sync_backfill_messages": doc.Sync.BackfillMessages, "sync_daily_messages_mailbox": doc.Sync.DailyMessagesPerMailbox, "sync_daily_messages_org": doc.Sync.DailyMessagesPerOrg, + "retention_engagement_event_days": doc.Retention.EngagementEventDays, + "retention_form_event_days": doc.Retention.FormEventDays, + "retention_audit_log_days": doc.Retention.AuditLogDays, "deliverability_enforce_domain_auth": doc.Deliverability.EnforceDomainAuth, "deliverability_auth_grace_hours": doc.Deliverability.AuthGraceHours, + "notification_channels": len(doc.Notifications.Channels), } return details } diff --git a/internal/api/handler/admin_notifications.go b/internal/api/handler/admin_notifications.go new file mode 100644 index 00000000..c3bfbb83 --- /dev/null +++ b/internal/api/handler/admin_notifications.go @@ -0,0 +1,166 @@ +package handler + +import ( + "errors" + "net/http" + "net/url" + "strings" + + "github.com/gin-gonic/gin" + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/app/instancesettings" + "github.com/warmbly/warmbly/internal/app/opsnotify" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" +) + +// Operator notification channels. The channels themselves live in the instance +// settings document and are written through PUT /admin/instance/settings; this +// file adds the two things that document cannot carry: the catalog of +// subscribable events, and a way to prove a channel actually works. + +// AdminNotificationEvents returns the subscribable event catalog. +// +// A self-hosted deployment gets the subset that means something without +// billing, so the panel never offers to alert on a commercial event that can +// never fire there. +func (h *Handler) AdminNotificationEvents(c *gin.Context) { + selfHosted := config.SelfHosted() + out := make([]opsnotify.EventDef, 0, len(opsnotify.Catalog)) + for _, def := range opsnotify.Catalog { + if selfHosted && !def.SelfHostRelevant { + continue + } + out = append(out, def) + } + c.JSON(http.StatusOK, gin.H{"events": out, "self_hosted": selfHosted}) +} + +// AdminTestNotificationChannel delivers a test alert. +// +// It accepts either a saved channel id or a full channel body, so an operator +// can prove a webhook URL before committing it to the document. Unlike every +// other delivery this one is synchronous and reports the transport error, and +// it is the only place a channel is exercised on demand. +func (h *Handler) AdminTestNotificationChannel(c *gin.Context) { + adminID := middleware.GetAdminUserID(c) + if adminID == nil { + errx.JSON(c, errx.ErrUnauthorized) + return + } + if h.OpsNotifier == nil { + errx.JSON(c, errx.New(errx.BadRequest, "Operator notifications are not available on this deployment.")) + return + } + + var req struct { + // ID selects a saved channel; when set the body's other fields are ignored. + ID string `json:"id"` + // The unsaved-channel form. + Type string `json:"type"` + Name string `json:"name"` + Target string `json:"target"` + Secret string `json:"secret"` + } + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + + var ch instancesettings.NotifyChannel + if req.ID != "" { + if h.InstanceSettings == nil { + errx.JSON(c, errx.New(errx.BadRequest, "Instance settings are not available on this deployment.")) + return + } + saved, ok := h.InstanceSettings.Get(c.Request.Context()).Notifications.Find(req.ID) + if !ok { + errx.JSON(c, errx.New(errx.NotFound, "no such notification channel")) + return + } + ch = saved + } else { + ch = instancesettings.NotifyChannel{ + ID: "test", + Name: req.Name, + Type: req.Type, + Target: req.Target, + Secret: req.Secret, + Enabled: true, + } + // A masked target means the client sent back a redacted read without + // retyping it, which cannot be delivered to. Ask for the id instead. + if ch.Target == instancesettings.Masked { + errx.JSON(c, errx.New(errx.BadRequest, "Save the channel first, then send a test to it by id.")) + return + } + if ch.IsWebhookTransport() { + if err := instancesettings.ValidateChannelURL(ch.Target); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, err.Error())) + return + } + } + } + + // A test is always delivered, whatever the channel is subscribed to: + // Enabled and Events gate real alerts, not a deliberate probe. + ch.Enabled = true + ch.Events = nil + + event := opsnotify.NewEvent( + opsnotify.EventTest, + "Test alert from Warmbly", + "If you can read this, this channel is wired up correctly.", + opsnotify.Field{Label: "Channel", Value: firstNonBlank(ch.Name, ch.Type)}, + ) + + if err := h.OpsNotifier.Deliver(c.Request.Context(), ch, event); err != nil { + // Never echo the raw error: a *url.Error embeds the request URL, and + // the whole point of redacting targets is that this endpoint does not + // hand a webhook URL back out. + errx.JSON(c, errx.New(errx.BadRequest, "Delivery failed: "+sanitizeDeliveryError(err, ch.Target))) + return + } + + if h.AdminService != nil { + h.AdminService.LogAdminAction( + c.Request.Context(), *adminID, + "test_notification_channel", "instance", nil, + map[string]any{"channel_type": ch.Type, "channel_name": ch.Name}, + c.ClientIP(), c.Request.UserAgent(), + ) + } + + c.JSON(http.StatusOK, gin.H{"delivered": true}) +} + +// sanitizeDeliveryError reduces a transport error to something safe to show: +// the status or cause, with any occurrence of the target stripped out. +func sanitizeDeliveryError(err error, target string) string { + msg := err.Error() + var uerr *url.Error + if errors.As(err, &uerr) { + // Keep the cause, drop the URL the stdlib prefixes onto it. + msg = uerr.Err.Error() + } + if target != "" { + msg = strings.ReplaceAll(msg, target, "the configured target") + } + // Belt and braces: never let a bare URL through whatever the shape. + if i := strings.Index(msg, "http"); i >= 0 { + msg = strings.TrimSpace(msg[:i]) + " (endpoint redacted)" + } + if strings.TrimSpace(msg) == "" { + return "the endpoint could not be reached" + } + return msg +} + +func firstNonBlank(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "channel" +} diff --git a/internal/api/handler/admin_updates.go b/internal/api/handler/admin_updates.go new file mode 100644 index 00000000..2daba611 --- /dev/null +++ b/internal/api/handler/admin_updates.go @@ -0,0 +1,80 @@ +package handler + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/updates" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/version" +) + +// The update surface behind the admin panel's top bar: what is running, what +// is newest, and the one button that applies it through the host-side updater. +// +// GET /admin/instance/update state; ?log=1 keeps the job log +// POST /admin/instance/update/check refresh GitHub and the updater now +// POST /admin/instance/update/apply start an update job + +// AdminUpdateState returns the cached release check plus a live read of the +// updater. Cheap enough for the top bar to poll. +func (h *Handler) AdminUpdateState(c *gin.Context) { + if h.UpdatesService == nil { + c.JSON(http.StatusOK, updates.State{Running: version.Current(), Updater: updates.UpdaterView{Status: "off"}}) + return + } + withLog := c.Query("log") == "1" + c.JSON(http.StatusOK, h.UpdatesService.State(c.Request.Context(), withLog)) +} + +// AdminUpdateCheck re-runs both checks immediately. +func (h *Handler) AdminUpdateCheck(c *gin.Context) { + if h.UpdatesService == nil { + errx.JSON(c, errx.New(errx.BadRequest, "Update checks are not available on this deployment.")) + return + } + h.audit(c, models.AuditActionCheckReleases, models.AuditEntityInstance, nil, nil) + c.JSON(http.StatusOK, h.UpdatesService.Check(c.Request.Context())) +} + +type applyUpdateBody struct { + // Target is "latest" (default) or a release tag. + Target string `json:"target"` +} + +// AdminUpdateApply starts an update job on the updater and returns it. The +// backend restarts as part of the job, so the caller polls the state endpoint +// until it answers again with a new version. +func (h *Handler) AdminUpdateApply(c *gin.Context) { + if h.UpdatesService == nil { + errx.JSON(c, errx.New(errx.BadRequest, "Updates are not available on this deployment.")) + return + } + var body applyUpdateBody + if c.Request.ContentLength > 0 { + if err := c.ShouldBindJSON(&body); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + } + job, err := h.UpdatesService.Apply(c.Request.Context(), body.Target) + if err != nil { + switch { + case errors.Is(err, updates.ErrUpdaterNotConfigured), errors.Is(err, updates.ErrNothingToApply): + errx.JSON(c, errx.New(errx.BadRequest, err.Error())) + default: + errx.JSON(c, errx.New(errx.Conflict, err.Error())) + } + return + } + jobID, _ := uuid.Parse(job.ID) + h.audit(c, models.AuditActionUpgrade, models.AuditEntityInstance, &jobID, map[string]string{ + "target": job.Target, + "from_commit": job.FromCommit, + "running": version.String(), + }) + c.JSON(http.StatusAccepted, job) +} diff --git a/internal/api/handler/api_key.go b/internal/api/handler/api_key.go index db627213..e7d5ed79 100644 --- a/internal/api/handler/api_key.go +++ b/internal/api/handler/api_key.go @@ -171,6 +171,37 @@ func (h *Handler) RevokeAPIKey(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "revoked"}) } +// RevokeOwnAPIKey revokes the key the request was made with. +// +// Deliberately outside the API_KEYS scope gate: a credential must always be +// able to end itself. Requiring a privilege to sign out means a read-only key +// on a laptop someone is handing back stays live, which is the opposite of +// what a `warmbly auth logout` promises. +func (h *Handler) RevokeOwnAPIKey(c *gin.Context) { + keyID := middleware.GetAPIKeyID(c) + if keyID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "this endpoint revokes the API key it is called with, and this request did not use one")) + return + } + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + + reason := c.Query("reason") + if reason == "" { + reason = "Revoked by the credential itself" + } + if xerr := h.APIKeyService.Revoke(c.Request.Context(), *orgID, *keyID, reason); xerr != nil { + errx.JSON(c, xerr) + return + } + + h.auditOrg(c, models.AuditActionRevoke, models.AuditEntityAPIKey, keyID, nil, map[string]string{"self": "true"}) + c.JSON(http.StatusOK, gin.H{"status": "revoked"}) +} + // ListAPIPermissions lists all available API permissions // GET /api-keys/permissions func (h *Handler) ListAPIPermissions(c *gin.Context) { diff --git a/internal/api/handler/attachment.go b/internal/api/handler/attachment.go index 859554ca..20152d86 100644 --- a/internal/api/handler/attachment.go +++ b/internal/api/handler/attachment.go @@ -7,6 +7,7 @@ package handler import ( "bytes" + "context" "fmt" "io" "net/http" @@ -14,6 +15,7 @@ import ( "strings" "time" + "github.com/getsentry/sentry-go" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -52,16 +54,41 @@ func sanitizeFilename(name string) string { func mb(b int64) int64 { return b / (1024 * 1024) } -// UploadCampaignAttachment — POST /campaigns/:id/attachments (multipart "file") -func (h *Handler) UploadCampaignAttachment(c *gin.Context) { +// attachmentCampaign resolves the campaign these attachment routes address and +// proves it belongs to the caller's organization. The route id is a raw path +// parameter, so without this an attachment could be listed, uploaded or deleted +// on another workspace's campaign. +func (h *Handler) attachmentCampaign(c *gin.Context) (campaignID, orgID uuid.UUID, xerr *errx.Error) { campaignID, err := uuid.Parse(c.Param("id")) if err != nil { - errx.JSON(c, errx.ErrUuid) - return + return uuid.Nil, uuid.Nil, errx.ErrUuid } - orgID := middleware.GetOrganizationID(c) - if orgID == nil { - errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + org := middleware.GetOrganizationID(c) + if org == nil { + return uuid.Nil, uuid.Nil, errx.New(errx.BadRequest, "no organization selected") + } + if _, xerr := h.CampaignService.Get(c.Request.Context(), org.String(), campaignID.String()); xerr != nil { + return uuid.Nil, uuid.Nil, xerr + } + return campaignID, *org, nil +} + +// deleteObjectDetached removes an object whose row was never written, on a +// bounded context that is not cancelled with the request: a client that gives +// up mid-upload must not leave bytes in storage that no quota counts. +func (h *Handler) deleteObjectDetached(ctx context.Context, key string) { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if err := h.Storage.Delete(cleanup, key); err != nil { + sentry.CaptureException(fmt.Errorf("attachment %s: cleanup after refused reservation: %w", key, err)) + } +} + +// UploadCampaignAttachment — POST /campaigns/:id/attachments (multipart "file") +func (h *Handler) UploadCampaignAttachment(c *gin.Context) { + campaignID, orgID, xerr := h.attachmentCampaign(c) + if xerr != nil { + errx.JSON(c, xerr) return } userID, err := uuid.Parse(middleware.GetUserID(c)) @@ -74,16 +101,32 @@ func (h *Handler) UploadCampaignAttachment(c *gin.Context) { return } - // Optional sequence_id form field scopes the attachment to one step. + // Cap the body before anything reads it, so a huge upload can't pin a + // worker: the first form field read parses the whole multipart body. + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, attachmentMaxBytes+(1<<20)) + + // Optional step_id scopes the attachment to one step; without it the file + // rides every step of the campaign. A malformed or foreign step is refused + // rather than silently widened to the whole campaign. var seqID *uuid.UUID - if s := strings.TrimSpace(c.PostForm("step_id")); s != "" { - if id, perr := uuid.Parse(s); perr == nil { - seqID = &id + if raw := strings.TrimSpace(c.PostForm("step_id")); raw != "" { + id, perr := uuid.Parse(raw) + if perr != nil { + errx.JSON(c, errx.New(errx.BadRequest, "step_id must be a uuid")) + return } + belongs, berr := h.AttachmentRepo.StepBelongsToCampaign(c.Request.Context(), campaignID, id) + if berr != nil { + errx.JSON(c, errx.InternalError()) + return + } + if !belongs { + errx.JSON(c, errx.New(errx.NotFound, "step not found in this campaign")) + return + } + seqID = &id } - // Cap the body before parsing so a huge upload can't pin a worker. - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, attachmentMaxBytes+(1<<20)) fh, err := c.FormFile("file") if err != nil { errx.JSON(c, errx.New(errx.BadRequest, "file is required")) @@ -99,20 +142,21 @@ func (h *Handler) UploadCampaignAttachment(c *gin.Context) { return } - // Plan-based overall storage quota (org-wide). - limit, xerr := h.FeatureGateService.GetStorageLimitBytes(c.Request.Context(), *orgID) + // Plan-based overall storage quota (org-wide). This read is only a fast + // refusal before the bytes are copied to storage; the check that counts + // is CreateWithinQuota below, which runs under the org's quota lock. + limit, xerr := h.FeatureGateService.GetStorageLimitBytes(c.Request.Context(), orgID) if xerr != nil { errx.JSON(c, xerr) return } - used, err := h.AttachmentRepo.SumStorageUsedByOrg(c.Request.Context(), *orgID) + used, err := h.AttachmentRepo.SumStorageUsedByOrg(c.Request.Context(), orgID) if err != nil { errx.JSON(c, errx.InternalError()) return } if used+fh.Size > limit { - errx.JSON(c, errx.New(errx.BadRequest, fmt.Sprintf( - "storage limit reached (%d MB of %d MB used) — remove attachments or upgrade your plan", mb(used), mb(limit)))) + errx.JSON(c, errx.StorageLimitReached(used, limit, fh.Size)) return } @@ -148,11 +192,30 @@ func (h *Handler) UploadCampaignAttachment(c *gin.Context) { MimeType: mimeType, S3Key: key, } - if err := h.AttachmentRepo.Create(c.Request.Context(), att); err != nil { - _ = h.Storage.Delete(c.Request.Context(), key) // best-effort cleanup + // The row is the reservation: it is written only if the total still fits + // once this upload is counted, so concurrent uploads cannot interleave + // past the limit. The limit is re-read under the lock so a plan change + // that lands between the pre-check and the insert is honored. A refused + // file is removed from storage again, on a context that outlives the + // request so a cancelled upload cannot strand the object. + limitFn := func(ctx context.Context) (int64, error) { + l, xerr := h.FeatureGateService.GetStorageLimitBytes(ctx, orgID) + if xerr != nil { + return 0, xerr + } + return l, nil + } + created, used, limit, err := h.AttachmentRepo.CreateWithinQuota(c.Request.Context(), att, orgID, limitFn) + if err != nil { + h.deleteObjectDetached(c.Request.Context(), key) errx.JSON(c, errx.InternalError()) return } + if !created { + h.deleteObjectDetached(c.Request.Context(), key) + errx.JSON(c, errx.StorageLimitReached(used, limit, fh.Size)) + return + } h.auditOrg(c, models.AuditActionCreate, models.AuditEntityCampaign, &att.ID, nil, map[string]string{ "scope": "attachment", "campaign_id": campaignID.String(), "filename": filename, @@ -163,9 +226,9 @@ func (h *Handler) UploadCampaignAttachment(c *gin.Context) { // ListCampaignAttachments — GET /campaigns/:id/attachments func (h *Handler) ListCampaignAttachments(c *gin.Context) { - campaignID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.ErrUuid) + campaignID, _, xerr := h.attachmentCampaign(c) + if xerr != nil { + errx.JSON(c, xerr) return } atts, err := h.AttachmentRepo.ListByCampaign(c.Request.Context(), campaignID) @@ -182,9 +245,9 @@ func (h *Handler) ListCampaignAttachments(c *gin.Context) { // DeleteCampaignAttachment — DELETE /campaigns/:id/attachments/:attachmentId func (h *Handler) DeleteCampaignAttachment(c *gin.Context) { - campaignID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.ErrUuid) + campaignID, _, xerr := h.attachmentCampaign(c) + if xerr != nil { + errx.JSON(c, xerr) return } attID, err := uuid.Parse(c.Param("attachmentId")) diff --git a/internal/api/handler/auth_config.go b/internal/api/handler/auth_config.go index 05a1e292..888afa53 100644 --- a/internal/api/handler/auth_config.go +++ b/internal/api/handler/auth_config.go @@ -72,6 +72,18 @@ type DeploymentAuthConfig struct { // DocsURL is where to send someone whose signup was refused by deployment // policy rather than by anything they did wrong. DocsURL string `json:"docs_url"` + + // WebsocketURL is the realtime gateway. Served here because a developer + // client (the CLI's event stream, an SDK) has no other way to find the + // socket on a self-hosted instance. Empty when the instance runs no + // realtime service. + WebsocketURL string `json:"websocket_url,omitempty"` + + // AppURL is the dashboard origin, the same one every emailed link is built + // from. A client that wants to send someone to a page (the CLI's `browse`, + // a chat integration) cannot derive it: on a self-hosted instance the host + // layout is whatever the operator chose. + AppURL string `json:"app_url,omitempty"` } // accountsDocsURL is the page every registration refusal points at. @@ -104,5 +116,7 @@ func (h *Handler) AuthConfig(c *gin.Context) { SetupRequired: h.BootstrapService != nil && h.BootstrapService.Required(c.Request.Context()), InvitesRequired: registration == config.RegistrationInviteOnly, DocsURL: accountsDocsURL, + WebsocketURL: config.WebsocketURL(), + AppURL: config.AppBaseURL(), }) } diff --git a/internal/api/handler/auth_instance.go b/internal/api/handler/auth_instance.go new file mode 100644 index 00000000..3909cdb4 --- /dev/null +++ b/internal/api/handler/auth_instance.go @@ -0,0 +1,54 @@ +package handler + +import ( + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/version" +) + +// InstanceVersion answers GET /v1/auth/instance for any signed-in member of a +// self-hosted instance: which Warmbly this is and whether a newer one exists. +// It carries no updater detail and no log; applying the update lives in the +// admin panel. Hosted deployments answer self_hosted=false and nothing else, +// so the dashboard has nothing to show there. +type instanceVersionResponse struct { + SelfHosted bool `json:"self_hosted"` + Version string `json:"version,omitempty"` + Commit string `json:"commit,omitempty"` + UpdateAvailable bool `json:"update_available"` + Latest *instanceLatestRelease `json:"latest,omitempty"` + CheckedAt *time.Time `json:"checked_at,omitempty"` +} + +type instanceLatestRelease struct { + Tag string `json:"tag"` + HTMLURL string `json:"html_url,omitempty"` + PublishedAt time.Time `json:"published_at,omitempty"` +} + +func (h *Handler) InstanceVersion(c *gin.Context) { + if !config.SelfHosted() { + c.JSON(http.StatusOK, instanceVersionResponse{SelfHosted: false}) + return + } + resp := instanceVersionResponse{ + SelfHosted: true, + Version: version.String(), + Commit: version.ShortCommit(), + } + if h.UpdatesService != nil { + st := h.UpdatesService.State(c.Request.Context(), false) + resp.UpdateAvailable = st.UpdateAvailable + if st.Latest != nil { + resp.Latest = &instanceLatestRelease{Tag: st.Latest.Tag, HTMLURL: st.Latest.HTMLURL, PublishedAt: st.Latest.PublishedAt} + } + if !st.CheckedAt.IsZero() { + t := st.CheckedAt + resp.CheckedAt = &t + } + } + c.JSON(http.StatusOK, resp) +} diff --git a/internal/api/handler/avatar.go b/internal/api/handler/avatar.go index a2231afd..50e59d5e 100644 --- a/internal/api/handler/avatar.go +++ b/internal/api/handler/avatar.go @@ -8,11 +8,14 @@ // Constants: // // - max size: 2 MiB. Anything larger gets a 400. -// - accepted MIME: image/png, image/jpeg, image/webp, image/gif. -// - object key: avatars/{kind}/{id}-{epoch}.{ext} +// - accepted MIME: image/png, image/jpeg (see allowedAvatarMIME). +// - object key: avatars/{kind}/{id}-{epoch_ms}-{nonce}.{ext} // // The epoch suffix forces cache busting on replacement so the -// browser doesn't keep showing the old avatar at the same URL. +// browser doesn't keep showing the old avatar at the same URL (the +// objects are served immutable, so a reused key would never refresh). +// The previous object is deleted best-effort once the row points at +// the new one, so replacing or removing an avatar doesn't leak blobs. package handler @@ -42,6 +45,9 @@ import ( const ( avatarMaxBytes int64 = 2 * 1024 * 1024 avatarMaxDimension = 1024 // px — reject anything bigger so a phone-camera dump doesn't sneak through + + userAvatarKeyPrefix = "avatars/users/" + orgAvatarKeyPrefix = "avatars/organizations/" ) // Intentionally narrow allowlist: only PNG and JPEG. WebP, GIF and @@ -77,17 +83,23 @@ func (h *Handler) UploadUserAvatar(c *gin.Context) { return } - key := fmt.Sprintf("avatars/users/%s-%d%s", userID.String(), time.Now().Unix(), ext) - url, xerr := putPublicObject(c.Request.Context(), h.Storage, key, bytesRead, mime) + ctx := c.Request.Context() + previous := h.currentUserAvatarURL(ctx, userID) + + key := avatarObjectKey(userAvatarKeyPrefix, userID, ext) + url, xerr := putPublicObject(ctx, h.Storage, key, bytesRead, mime) if xerr != nil { errx.Handle(c, xerr) return } - if err := h.UserRepo.UpdateAvatar(c.Request.Context(), userID, &url); err != nil { - errx.Handle(c, errx.InternalError()) + // Through the service, not the repo: /auth/me is served from a cached + // copy, and only the service drops it. + if xerr := h.UserService.UpdateAvatar(ctx, userID, &url); xerr != nil { + errx.Handle(c, xerr) return } + h.deleteAvatarObject(ctx, previous, userAvatarKeyPrefix, key) h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityUser, &userID, nil, map[string]string{"field": "avatar_url"}) @@ -103,10 +115,14 @@ func (h *Handler) DeleteUserAvatar(c *gin.Context) { return } - if err := h.UserRepo.UpdateAvatar(c.Request.Context(), userID, nil); err != nil { - errx.Handle(c, errx.InternalError()) + ctx := c.Request.Context() + previous := h.currentUserAvatarURL(ctx, userID) + + if xerr := h.UserService.UpdateAvatar(ctx, userID, nil); xerr != nil { + errx.Handle(c, xerr) return } + h.deleteAvatarObject(ctx, previous, userAvatarKeyPrefix, "") h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityUser, &userID, nil, map[string]string{"field": "avatar_url", "value": "cleared"}) @@ -140,17 +156,21 @@ func (h *Handler) UploadOrganizationAvatar(c *gin.Context) { return } - key := fmt.Sprintf("avatars/organizations/%s-%d%s", orgID.String(), time.Now().Unix(), ext) - url, xerr := putPublicObject(c.Request.Context(), h.Storage, key, bytesRead, mime) + ctx := c.Request.Context() + previous := h.currentOrgAvatarURL(ctx, *orgID) + + key := avatarObjectKey(orgAvatarKeyPrefix, *orgID, ext) + url, xerr := putPublicObject(ctx, h.Storage, key, bytesRead, mime) if xerr != nil { errx.Handle(c, xerr) return } - if err := h.OrgRepo.UpdateAvatar(c.Request.Context(), *orgID, &url); err != nil { + if err := h.OrgRepo.UpdateAvatar(ctx, *orgID, &url); err != nil { errx.Handle(c, errx.InternalError()) return } + h.deleteAvatarObject(ctx, previous, orgAvatarKeyPrefix, key) h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityOrganization, orgID, nil, map[string]string{"field": "avatar_url"}) @@ -175,14 +195,82 @@ func (h *Handler) DeleteOrganizationAvatar(c *gin.Context) { return } - if err := h.OrgRepo.UpdateAvatar(c.Request.Context(), *orgID, nil); err != nil { + ctx := c.Request.Context() + previous := h.currentOrgAvatarURL(ctx, *orgID) + + if err := h.OrgRepo.UpdateAvatar(ctx, *orgID, nil); err != nil { errx.Handle(c, errx.InternalError()) return } + h.deleteAvatarObject(ctx, previous, orgAvatarKeyPrefix, "") + + // Audited like the upload so teammates' org switcher refreshes live. + h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityOrganization, orgID, nil, map[string]string{"field": "avatar_url", "value": "cleared"}) c.Status(http.StatusNoContent) } +// avatarObjectKey builds a key that is unique per upload: the epoch keeps keys +// sortable, the random nonce keeps two uploads in the same millisecond from +// sharing an immutably cached URL. +func avatarObjectKey(prefix string, id uuid.UUID, ext string) string { + nonce := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + return fmt.Sprintf("%s%s-%d-%s%s", prefix, id.String(), time.Now().UnixMilli(), nonce, ext) +} + +// currentUserAvatarURL returns the avatar URL stored on the user row, or "" +// when there is none or the lookup fails (cleanup is best-effort). +func (h *Handler) currentUserAvatarURL(ctx context.Context, userID uuid.UUID) string { + u, xerr := h.UserService.GetUser(ctx, userID) + if xerr != nil || u == nil || u.AvatarURL == nil { + return "" + } + return *u.AvatarURL +} + +// currentOrgAvatarURL is the organization counterpart of currentUserAvatarURL. +func (h *Handler) currentOrgAvatarURL(ctx context.Context, orgID uuid.UUID) string { + org, xerr := h.OrganizationService.Get(ctx, orgID) + if xerr != nil || org == nil || org.AvatarURL == nil { + return "" + } + return *org.AvatarURL +} + +// deleteAvatarObject removes the object behind a previous avatar URL once the +// row no longer points at it. Only keys under our own prefix are touched, so +// an external URL (an OAuth profile picture, say) is left alone, and keepKey +// guards the freshly written object. Failures are ignored: an orphaned blob +// is harmless, a failed request after a successful update is not. +func (h *Handler) deleteAvatarObject(ctx context.Context, previousURL, prefix, keepKey string) { + if h.Storage == nil || previousURL == "" { + return + } + key := avatarKeyFromURL(previousURL, prefix) + if key == "" || key == keepKey { + return + } + _ = h.Storage.Delete(ctx, key) +} + +// avatarKeyFromURL recovers the object key from a public avatar URL. Both +// storage backends build the URL differently, but the key always starts with +// the known prefix, so that is what is looked for. +func avatarKeyFromURL(url, prefix string) string { + idx := strings.Index(url, prefix) + if idx < 0 { + return "" + } + key := url[idx:] + if q := strings.IndexAny(key, "?#"); q >= 0 { + key = key[:q] + } + if key == prefix || strings.Contains(key, "..") { + return "" + } + return key +} + func (h *Handler) requireOrgOwner(c *gin.Context, orgID, userID uuid.UUID) *errx.Error { m, err := h.OrgRepo.GetMember(c.Request.Context(), orgID, userID) if err != nil || m == nil { diff --git a/internal/api/handler/campaign.go b/internal/api/handler/campaign.go index b5a2974f..b0232b54 100644 --- a/internal/api/handler/campaign.go +++ b/internal/api/handler/campaign.go @@ -1,6 +1,7 @@ package handler import ( + "context" "errors" "io" "net/http" @@ -16,7 +17,9 @@ import ( ) // templatePreviewRequest is the composer's preview/validate payload: the raw -// templates plus an optional sample contact to render against. +// templates plus, optionally, the context of the real send: a contact of the +// organization to render for (or ad-hoc sample overrides), the campaign whose +// opt-out footer and attachments apply, and the mailbox whose signature does. type templatePreviewRequest struct { Subject string `json:"subject"` BodyHTML string `json:"body_html"` @@ -29,6 +32,28 @@ type templatePreviewRequest struct { Phone string `json:"phone"` CustomFields map[string]string `json:"custom_fields"` } `json:"contact"` + ContactID *uuid.UUID `json:"contact_id"` + CampaignID *uuid.UUID `json:"campaign_id"` + AccountID *uuid.UUID `json:"account_id"` + // The step being previewed, so the attachment list is the one that step + // sends. Omitted lists the campaign-wide files only. + StepID *uuid.UUID `json:"step_id"` +} + +// orgContact resolves a contact that must belong to orgID; any other id, or an +// unknown one, is not found. +func (h *Handler) orgContact(ctx context.Context, orgID, contactID uuid.UUID) (*models.Contact, *errx.Error) { + if h.ContactRepo == nil { + return nil, errx.ErrNotFound + } + found, xerr := h.ContactRepo.GetByIDsAndOrganization(ctx, orgID, []uuid.UUID{contactID}) + if xerr != nil { + return nil, xerr + } + if len(found) == 0 { + return nil, errx.New(errx.NotFound, "contact not found") + } + return &found[0], nil } func sampleContact() models.Contact { @@ -42,17 +67,67 @@ func sampleContact() models.Contact { } } -// PreviewCampaignTemplate renders subject/body against a sample (or supplied) -// contact EXACTLY as the send path would, returning the output plus any parse -// errors and unresolved tokens. No side effects — powers the composer's live +// PreviewCampaignTemplate renders subject/body for a contact EXACTLY as the +// send path would, returning the output plus any parse errors and unresolved +// tokens. With campaign_id and account_id it also applies the mailbox +// signature, the opt-out footer and the plain-text rule, and lists the +// attachments the send carries (the campaign-wide files, plus step_id's own). No side effects — powers the composer's live // preview + inline validation. func (h *Handler) PreviewCampaignTemplate(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.ErrNoOrganization) + return + } var req templatePreviewRequest if err := c.ShouldBindJSON(&req); err != nil { errx.JSON(c, errx.ErrInvalid) return } + + in := tasks.EmailPreviewInput{Subject: req.Subject, BodyHTML: req.BodyHTML, BodyPlain: req.BodyPlain} + if req.CampaignID != nil { + campaign, xerr := h.CampaignService.Get(c.Request.Context(), orgID.String(), req.CampaignID.String()) + if xerr != nil { + errx.JSON(c, xerr) + return + } + in.Campaign = campaign + if req.StepID != nil { + in.SequenceID = *req.StepID + } + } + if req.AccountID != nil { + if xerr := mailboxAllowed(c, *req.AccountID); xerr != nil { + errx.JSON(c, xerr) + return + } + account, xerr := h.EmailService.Get(c.Request.Context(), orgID.String(), req.AccountID.String()) + if xerr != nil { + errx.JSON(c, xerr) + return + } + in.Account = account + } + contact := sampleContact() + if req.ContactID != nil { + // Rendering a real contact reads its fields back, so it takes the + // contacts read permission on top of the route's campaigns one. + if xerr := h.hasAccess(c, models.PermViewContacts, models.APIPermReadContacts); xerr != nil { + errx.JSON(c, xerr) + return + } + found, xerr := h.orgContact(c.Request.Context(), *orgID, *req.ContactID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + contact = *found + if contact.CustomFields == nil { + contact.CustomFields = map[string]string{} + } + } if rc := req.Contact; rc != nil { if rc.FirstName != "" { contact.FirstName = rc.FirstName @@ -73,7 +148,8 @@ func (h *Handler) PreviewCampaignTemplate(c *gin.Context) { contact.CustomFields[k] = v } } - c.JSON(http.StatusOK, tasks.PreviewTemplates(req.Subject, req.BodyHTML, req.BodyPlain, contact)) + in.Contact = contact + c.JSON(http.StatusOK, h.TasksService.PreviewEmail(c.Request.Context(), *orgID, in)) } func (h *Handler) CreateCampaign(c *gin.Context) { @@ -134,9 +210,36 @@ func (h *Handler) SearchCampaigns(c *gin.Context) { cursor := c.Query("cursor") folder := c.Query("folder") status := c.Query("status") + kind := c.Query("kind") limit := c.Query("limit") - resp, err := h.CampaignService.Search(c.Request.Context(), orgID.String(), query, cursor, folder, status, limit) + resp, err := h.CampaignService.Search(c.Request.Context(), orgID.String(), query, cursor, folder, status, kind, limit) + if err != nil { + errx.JSON(c, err) + return + } + + c.JSON(http.StatusOK, resp) +} + +// EstimateCampaign projects an audience against a sender pool before a +// campaign exists: recipients, daily capacity and the day the last send +// lands. Read-only. +// POST /campaigns-estimate +func (h *Handler) EstimateCampaign(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.ErrNoOrganization) + return + } + + var data models.CampaignEstimate + if err := c.ShouldBindJSON(&data); err != nil { + errx.JSON(c, errx.ErrInvalid) + return + } + + resp, err := h.CampaignService.Estimate(c.Request.Context(), *orgID, &data) if err != nil { errx.JSON(c, err) return diff --git a/internal/api/handler/cli_auth.go b/internal/api/handler/cli_auth.go new file mode 100644 index 00000000..3cf2c2ae --- /dev/null +++ b/internal/api/handler/cli_auth.go @@ -0,0 +1,126 @@ +package handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// Device-code sign-in for the `warmbly` CLI. The public half (start, poll) is +// what the CLI calls; the session half is the browser approval screen. + +func (h *Handler) cliAuthReady(c *gin.Context) bool { + if h.CLIAuthService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "CLI sign-in is not enabled on this instance")) + return false + } + return true +} + +// CLIAuthStart opens a handshake for a CLI that holds no key yet. +func (h *Handler) CLIAuthStart(c *gin.Context) { + if !h.cliAuthReady(c) { + return + } + var req models.CLIAuthStartRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + res, xerr := h.CLIAuthService.StartCode(c.Request.Context(), req) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusCreated, res) +} + +// CLIAuthPoll is polled by the CLI until a member approves the code. The key is +// handed out exactly once, on the poll that follows approval. +func (h *Handler) CLIAuthPoll(c *gin.Context) { + if !h.cliAuthReady(c) { + return + } + var req struct { + DeviceCode string `json:"device_code"` + } + if err := c.ShouldBindJSON(&req); err != nil || req.DeviceCode == "" { + errx.JSON(c, errx.New(errx.BadRequest, "device_code is required")) + return + } + res, xerr := h.CLIAuthService.PollCode(c.Request.Context(), req.DeviceCode) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, res) +} + +// CLIAuthDescribeCode shows the approving member what they are authorizing. +func (h *Handler) CLIAuthDescribeCode(c *gin.Context) { + if !h.cliAuthReady(c) { + return + } + code, xerr := h.CLIAuthService.DescribeCode(c.Request.Context(), c.Param("code")) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, code) +} + +// CLIAuthApproveCode mints the key into the workspace named in the body, not +// the session's, because a member with several workspaces picks on the screen. +func (h *Handler) CLIAuthApproveCode(c *gin.Context) { + if !h.cliAuthReady(c) { + return + } + userID, err := middleware.GetUserUUID(c) + if err != nil { + errx.JSON(c, errx.ErrUnauthorized) + return + } + var req models.CLIAuthApproveRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + orgID, perr := uuid.Parse(req.OrganizationID) + if perr != nil { + if sessionOrg := middleware.GetOrganizationID(c); sessionOrg != nil { + orgID = *sessionOrg + } else { + errx.JSON(c, errx.ErrNoOrganization) + return + } + } + + code, xerr := h.CLIAuthService.ApproveCode(c.Request.Context(), c.Param("code"), orgID, userID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + // Logged against the org that was picked on screen, which is not always the + // session's, so this cannot go through auditOrg. + h.AuditService.LogAction(c.Request.Context(), orgID, userID, models.AuditActionCreate, models.AuditEntityAPIKey, code.APIKeyID, + c.ClientIP(), c.Request.UserAgent(), nil, map[string]string{"source": "cli", "client": code.ClientName, "hostname": code.Hostname}) + c.JSON(http.StatusOK, code) +} + +// CLIAuthDenyCode declines the request. Deliberately not audited: nothing was +// created, and a denial is not a change to the workspace. +func (h *Handler) CLIAuthDenyCode(c *gin.Context) { + if !h.cliAuthReady(c) { + return + } + if xerr := h.CLIAuthService.DenyCode(c.Request.Context(), c.Param("code")); xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"ok": true}) +} diff --git a/internal/api/handler/contact_detail.go b/internal/api/handler/contact_detail.go index 97517c36..73beb7da 100644 --- a/internal/api/handler/contact_detail.go +++ b/internal/api/handler/contact_detail.go @@ -11,6 +11,7 @@ import ( "github.com/warmbly/warmbly/internal/api/middleware" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" ) // GetContact returns the hydrated contact 360 payload. Used by the @@ -160,18 +161,40 @@ func (h *Handler) ListContactTimeline(c *gin.Context) { } limit := 50 - if l, err := strconv.Atoi(c.Query("limit")); err == nil && l > 0 && l <= 200 { + if raw := c.Query("limit"); raw != "" { + l, err := strconv.Atoi(raw) + if err != nil || l < 1 || l > 200 { + errx.Handle(c, errx.New(errx.BadRequest, "limit must be between 1 and 200")) + return + } limit = l } - var before *time.Time - if v := c.Query("before"); v != "" { - if t, perr := time.Parse(time.RFC3339Nano, v); perr == nil { - before = &t + // The page resumes after an opaque (at, source, id) position. The older + // `before` timestamp is still accepted and maps onto the same keyset at + // rank zero, which admits exactly the events strictly older than it. + var cursor *models.ContactTimelineKey + if raw := c.Query("cursor"); raw != "" { + at, source, id, xerr := paging.DecodeMergedCursor(raw) + if xerr != nil { + errx.Handle(c, xerr) + return } + if !models.ContactTimelineSource(source).Valid() { + errx.Handle(c, errx.New(errx.BadRequest, "invalid cursor")) + return + } + cursor = &models.ContactTimelineKey{At: at, Source: models.ContactTimelineSource(source), ID: id} + } else if raw := c.Query("before"); raw != "" { + t, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + errx.Handle(c, errx.New(errx.BadRequest, "before must be an RFC 3339 timestamp")) + return + } + cursor = &models.ContactTimelineKey{At: t} } - res, xerr := h.ContactService.ListTimeline(c.Request.Context(), userID, orgID, contactID, limit, before) + res, xerr := h.ContactService.ListTimeline(c.Request.Context(), userID, orgID, contactID, limit, cursor) if xerr != nil { errx.Handle(c, xerr) return diff --git a/internal/api/handler/email_onboarding.go b/internal/api/handler/email_onboarding.go index e4a03688..719ac303 100644 --- a/internal/api/handler/email_onboarding.go +++ b/internal/api/handler/email_onboarding.go @@ -1,11 +1,13 @@ package handler import ( + "fmt" "net/http" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" ) @@ -183,3 +185,72 @@ func (h *Handler) ConnectEmailSMTPIMAP(c *gin.Context) { c.JSON(http.StatusCreated, acc) } + +// OnboardingSMTPIMAPBulkRequest carries up to config.MailboxBulkBatchMax rows. +type OnboardingSMTPIMAPBulkRequest struct { + Accounts []OnboardingSMTPIMAPRequest `json:"accounts"` +} + +// ConnectEmailSMTPIMAPBulk is POST /emails/onboarding/smtp-imap/bulk: the +// dashboard's CSV import streams a file through it in batches. The answer is +// always 200 with a per-row status, so one bad row never hides the others. +// Re-sending a batch is safe: a mailbox that is already connected is skipped. +func (h *Handler) ConnectEmailSMTPIMAPBulk(c *gin.Context) { + userIDStr := middleware.GetUserID(c) + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.Handle(c, errx.ErrNoOrganization) + return + } + + var req OnboardingSMTPIMAPBulkRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.Handle(c, errx.ErrInvalid) + return + } + if len(req.Accounts) == 0 { + errx.Handle(c, errx.New(errx.BadRequest, "accounts must carry at least one row")) + return + } + if len(req.Accounts) > config.MailboxBulkBatchMax { + errx.Handle(c, errx.New(errx.BadRequest, fmt.Sprintf("accounts may carry at most %d rows per request", config.MailboxBulkBatchMax))) + return + } + + rows := make([]models.NewSMTPIMAPAccount, len(req.Accounts)) + for i, a := range req.Accounts { + rows[i] = models.NewSMTPIMAPAccount{Email: a.Email, Name: a.Name, SMTP: a.SMTP, IMAP: a.IMAP} + } + + result := h.EmailService.OnboardSMTPIMAPBulk(c.Request.Context(), userIDStr, orgID, rows) + for _, r := range result.Data { + if r.Status != models.MailboxBulkConnected || r.ID == nil { + continue + } + h.auditOrg(c, models.AuditActionConnect, models.AuditEntityEmailAccount, r.ID, nil, map[string]string{ + "provider": "smtp_imap", + "email": r.Email, + "bulk": "true", + }) + } + + c.JSON(http.StatusOK, result) +} + +// GetMailboxAllowance is GET /emails/allowance: how many mailboxes the +// workspace holds, how many it may hold and why, and any open request for +// more. The dashboard reads it before a connect so the answer is never a +// surprise after the credentials were typed. +func (h *Handler) GetMailboxAllowance(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.Handle(c, errx.ErrNoOrganization) + return + } + a, xerr := h.OrganizationService.MailboxAllowance(c.Request.Context(), *orgID) + if xerr != nil { + errx.Handle(c, xerr) + return + } + c.JSON(http.StatusOK, a) +} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 8a856e3f..39c14277 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -14,6 +14,7 @@ import ( "github.com/warmbly/warmbly/internal/app/behavior" "github.com/warmbly/warmbly/internal/app/bootstrap" "github.com/warmbly/warmbly/internal/app/campaign" + "github.com/warmbly/warmbly/internal/app/cliauth" "github.com/warmbly/warmbly/internal/app/cloudlink" "github.com/warmbly/warmbly/internal/app/compose" "github.com/warmbly/warmbly/internal/app/contact" @@ -35,6 +36,7 @@ import ( "github.com/warmbly/warmbly/internal/app/mcp" "github.com/warmbly/warmbly/internal/app/notification" "github.com/warmbly/warmbly/internal/app/oauth" + "github.com/warmbly/warmbly/internal/app/opsnotify" "github.com/warmbly/warmbly/internal/app/organization" "github.com/warmbly/warmbly/internal/app/orgrisk" "github.com/warmbly/warmbly/internal/app/orgtransfer" @@ -59,6 +61,8 @@ import ( "github.com/warmbly/warmbly/internal/app/twofa" "github.com/warmbly/warmbly/internal/app/tz" "github.com/warmbly/warmbly/internal/app/unibox" + "github.com/warmbly/warmbly/internal/app/unsublink" + "github.com/warmbly/warmbly/internal/app/updates" "github.com/warmbly/warmbly/internal/app/user" "github.com/warmbly/warmbly/internal/app/warmup" "github.com/warmbly/warmbly/internal/app/warmupcontent" @@ -160,12 +164,17 @@ type Handler struct { WorkerRepo repository.WorkerRepository CredentialsRepo repository.CredentialsRepository ReleasesService *releases.Service + // UpdatesService backs the admin panel's update indicator and button. + UpdatesService *updates.Service // Notifications EmailNotificationService notify.EmailNotificationService // Advanced outreach controls AdvancedService advanced.Service + // UnsubscribeLinks verifies the signed tokens on recipient unsubscribe + // links. Nil when the instance has no public API URL to mint them on. + UnsubscribeLinks *unsublink.Signer // Website tracking snippet: settings and the page-view ingest path. WebsiteTrackingService websitetracking.Service @@ -304,6 +313,9 @@ type Handler struct { PoolLinkService poollink.Service CloudLinkService cloudlink.Service + // Device-code sign-in for the `warmbly` CLI. Nil-safe: routes answer 501. + CLIAuthService cliauth.Service + // Infrastructure liveness probes for the admin System Status page. // Wired in cmd/backend/main.go where the concrete clients live. SystemChecker *sysstatus.Checker @@ -321,4 +333,7 @@ type Handler struct { // InstanceSettings is the database-backed settings tier. It holds only // keys no environment variable owns. InstanceSettings instancesettings.Service + // OpsNotifier delivers instance-wide operator alerts. Nil disables the + // notification admin surface. + OpsNotifier opsnotify.Notifier } diff --git a/internal/api/handler/organization.go b/internal/api/handler/organization.go index 7fa113d8..e02359e9 100644 --- a/internal/api/handler/organization.go +++ b/internal/api/handler/organization.go @@ -493,7 +493,10 @@ func (h *Handler) GetMyPendingInvitations(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"data": invitations}) } -// GetOrganizationLimits returns the organization's limits and current usage +// GetOrganizationLimits returns the limits the server actually enforces for +// the workspace (plan, then any approved override) beside the live counts, +// plus the mailbox allowance and attachment storage, which have no plan +// column of their own. A nil limit is unmetered. func (h *Handler) GetOrganizationLimits(c *gin.Context) { orgID := middleware.GetOrganizationID(c) if orgID == nil { @@ -501,7 +504,7 @@ func (h *Handler) GetOrganizationLimits(c *gin.Context) { return } - limits, xerr := h.OrganizationService.GetOrganizationLimits(c.Request.Context(), *orgID) + limits, xerr := h.OrganizationService.GetEffectiveLimits(c.Request.Context(), *orgID) if xerr != nil { errx.JSON(c, xerr) return @@ -513,8 +516,33 @@ func (h *Handler) GetOrganizationLimits(c *gin.Context) { return } + mailboxes, xerr := h.OrganizationService.MailboxAllowance(c.Request.Context(), *orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + + // Storage is reported here because nothing else does: a workspace that + // dropped to a smaller plan is over its quota with no upload refused yet. + storage := gin.H{"used_bytes": int64(0), "limit_bytes": int64(0)} + if h.FeatureGateService != nil && h.AttachmentRepo != nil { + limit, xerr := h.FeatureGateService.GetStorageLimitBytes(c.Request.Context(), *orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + used, err := h.AttachmentRepo.SumStorageUsedByOrg(c.Request.Context(), *orgID) + if err != nil { + errx.JSON(c, errx.InternalError()) + return + } + storage = gin.H{"used_bytes": used, "limit_bytes": limit, "over_quota": used > limit} + } + c.JSON(http.StatusOK, gin.H{ - "limits": limits, - "counts": counts, + "limits": limits, + "counts": counts, + "mailboxes": mailboxes, + "storage": storage, }) } diff --git a/internal/api/handler/suppressions.go b/internal/api/handler/suppressions.go new file mode 100644 index 00000000..8804c978 --- /dev/null +++ b/internal/api/handler/suppressions.go @@ -0,0 +1,137 @@ +package handler + +import ( + "encoding/base64" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// The workspace suppression list: every address and domain campaign mail +// must not go to, whatever put it there. + +func (h *Handler) ListSuppressions(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + limit := 50 + if raw := c.Query("limit"); raw != "" { + l, err := strconv.Atoi(raw) + if err != nil || l < 1 || l > 200 { + errx.JSON(c, errx.New(errx.BadRequest, "limit must be between 1 and 200")) + return + } + limit = l + } + var beforeAt *time.Time + var beforeID *uuid.UUID + if raw := c.Query("cursor"); raw != "" { + at, id, ok := decodeSuppressionCursor(raw) + if !ok { + errx.JSON(c, errx.New(errx.BadRequest, "invalid cursor")) + return + } + beforeAt, beforeID = &at, &id + } + + rows, xerr := h.AdvancedService.ListSuppressions(c.Request.Context(), *orgID, c.Query("q"), beforeAt, beforeID, limit+1) + if xerr != nil { + errx.JSON(c, xerr) + return + } + res := models.SuppressionListResult{Data: rows} + if len(rows) > limit { + res.Data = rows[:limit] + last := res.Data[limit-1] + next := encodeSuppressionCursor(last.CreatedAt, last.ID) + res.Pagination = models.CPagination{NextCursor: &next, HasMore: true} + } + c.JSON(http.StatusOK, res) +} + +func (h *Handler) AddSuppressions(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + actorID, _ := middleware.GetUserUUID(c) + var req models.AddSuppressionsRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.ErrInvalid) + return + } + res, xerr := h.AdvancedService.AddSuppressions(c.Request.Context(), *orgID, actorID, &req) + if xerr != nil { + errx.JSON(c, xerr) + return + } + h.auditOrg(c, models.AuditActionCreate, models.AuditEntitySuppression, nil, nil, map[string]string{ + "added": strconv.Itoa(res.Added), + "skipped": strconv.Itoa(len(res.Skipped)), + }) + c.JSON(http.StatusOK, res) +} + +func (h *Handler) RemoveSuppression(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid suppression id")) + return + } + entry, xerr := h.AdvancedService.RemoveSuppression(c.Request.Context(), *orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + // Lifting a recipient's own opt-out is the audited action: the entry it + // removed is recorded so the trail shows who was re-enabled and why they + // had been on the list. + h.auditOrg(c, models.AuditActionDelete, models.AuditEntitySuppression, &id, nil, map[string]string{ + "value": entry.Email, + "kind": string(entry.Kind), + "source": string(entry.Source), + }) + c.Status(http.StatusNoContent) +} + +// The cursor is the keyset (created_at, id) of the last row, base64 so +// clients treat it as opaque. +func encodeSuppressionCursor(at time.Time, id uuid.UUID) string { + return base64.RawURLEncoding.EncodeToString([]byte(at.UTC().Format(time.RFC3339Nano) + "|" + id.String())) +} + +func decodeSuppressionCursor(raw string) (time.Time, uuid.UUID, bool) { + b, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return time.Time{}, uuid.Nil, false + } + parts := strings.SplitN(string(b), "|", 2) + if len(parts) != 2 { + return time.Time{}, uuid.Nil, false + } + at, err := time.Parse(time.RFC3339Nano, parts[0]) + if err != nil { + return time.Time{}, uuid.Nil, false + } + id, err := uuid.Parse(parts[1]) + if err != nil { + return time.Time{}, uuid.Nil, false + } + return at, id, true +} diff --git a/internal/api/handler/test_email.go b/internal/api/handler/test_email.go index a4ca2c4d..b886add4 100644 --- a/internal/api/handler/test_email.go +++ b/internal/api/handler/test_email.go @@ -14,6 +14,9 @@ type sendTestEmailRequest struct { SequenceID *uuid.UUID `json:"step_id"` AccountID uuid.UUID `json:"account_id" binding:"required"` Recipient string `json:"recipient" binding:"required,email"` + // ContactID renders the copy for a real contact of the organization + // instead of the placeholder one. + ContactID *uuid.UUID `json:"contact_id"` } // SendTestEmail sends a preview/test email for a campaign sequence @@ -24,7 +27,6 @@ func (h *Handler) SendTestEmail(c *gin.Context) { errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) return } - userID := middleware.GetUserID(c) campaignID, err := uuid.Parse(c.Param("id")) if err != nil { @@ -39,7 +41,7 @@ func (h *Handler) SendTestEmail(c *gin.Context) { } // Load campaign - campaign, xerr := h.CampaignService.Get(c.Request.Context(), userID, campaignID.String()) + campaign, xerr := h.CampaignService.Get(c.Request.Context(), orgID.String(), campaignID.String()) if xerr != nil { errx.JSON(c, xerr) return @@ -73,17 +75,43 @@ func (h *Handler) SendTestEmail(c *gin.Context) { sequence = &sequences[0] } + if xerr := mailboxAllowed(c, req.AccountID); xerr != nil { + errx.JSON(c, xerr) + return + } + + var contact *models.Contact + if req.ContactID != nil { + // Rendering a real contact reads its fields back, so it takes the + // contacts read permission on top of the route's campaigns one. + if xerr := h.hasAccess(c, models.PermViewContacts, models.APIPermReadContacts); xerr != nil { + errx.JSON(c, xerr) + return + } + found, cxerr := h.orgContact(c.Request.Context(), *orgID, *req.ContactID) + if cxerr != nil { + errx.JSON(c, cxerr) + return + } + contact = found + } + // Send the test email - xerr = h.TasksService.SendTestEmail(c.Request.Context(), userID, req.AccountID, req.Recipient, campaign, sequence) + xerr = h.TasksService.SendTestEmail(c.Request.Context(), *orgID, req.AccountID, req.Recipient, campaign, sequence, contact) if xerr != nil { errx.JSON(c, xerr) return } - c.JSON(http.StatusOK, gin.H{ + resp := gin.H{ "message": "test email sent", "recipient": req.Recipient, "subject": sequence.Subject, "account_id": req.AccountID.String(), - }) + "step_id": sequence.ID.String(), + } + if contact != nil { + resp["contact_id"] = contact.ID.String() + } + c.JSON(http.StatusOK, resp) } diff --git a/internal/api/handler/unsubscribe.go b/internal/api/handler/unsubscribe.go index 5389c359..52767fc4 100644 --- a/internal/api/handler/unsubscribe.go +++ b/internal/api/handler/unsubscribe.go @@ -1,39 +1,96 @@ package handler import ( + "html/template" "net/http" + "strings" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/unsublink" "github.com/warmbly/warmbly/internal/errx" ) -// Unsubscribe handles the List-Unsubscribe link and the RFC 8058 one-click POST. -// It is PUBLIC + unauthenticated — mailbox providers and recipients hit it -// directly. Both GET (a recipient clicking the link) and POST (the provider's -// one-click, body "List-Unsubscribe=One-Click") suppress the recipient org-wide. -// The link shape is /unsubscribe?cid=&rid=. -func (h *Handler) Unsubscribe(c *gin.Context) { - isPost := c.Request.Method == http.MethodPost +// The recipient-facing unsubscribe endpoints. PUBLIC and unauthenticated by +// design: the only credential is the signed token in the path, minted per +// recipient when the email was sent. +// +// GET /unsubscribe/:token a click on the link: a confirm page +// POST /unsubscribe/:token the confirm button, or the mail +// client's RFC 8058 one-click POST +// (body List-Unsubscribe=One-Click), +// which suppresses with no page +// POST /unsubscribe/:token/resubscribe the "unsubscribed by mistake" button +// +// A GET never changes anything, because link scanners and preview fetchers +// follow every link in an email; only a POST suppresses. - cid, err1 := uuid.Parse(c.Query("cid")) - rid, err2 := uuid.Parse(c.Query("rid")) - if err1 != nil || err2 != nil { - if isPost { - c.Status(http.StatusBadRequest) +func (h *Handler) UnsubscribePage(c *gin.Context) { + claims, ok := h.unsubscribeClaims(c) + if !ok { + return + } + if claims.ContactID == uuid.Nil { + renderUnsubPage(c, http.StatusOK, unsubView{Title: "This was a test email", Body: "Test sends carry a link that is not tied to anyone, so there is nothing to unsubscribe."}) + return + } + renderUnsubPage(c, http.StatusOK, unsubView{ + Title: "Unsubscribe from these emails?", + Body: "Confirm and you will not receive further emails from this sender.", + Confirm: c.Request.URL.Path, + }) +} + +// unsubscribeBodyLimit caps the public POST bodies. The engine-wide limit is +// registered after these routes, so it does not cover them; a one-click or +// confirm body is a few bytes. +const unsubscribeBodyLimit = 16 << 10 + +func (h *Handler) UnsubscribeSubmit(c *gin.Context) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, unsubscribeBodyLimit) + oneClick := strings.EqualFold(strings.TrimSpace(c.PostForm("List-Unsubscribe")), "One-Click") + confirmed := c.PostForm("confirm") == "1" + + claims, err := h.verifyUnsubscribeToken(c.Param("token")) + if err != nil { + if oneClick { + // RFC 8058: a bad or expired link is terminal, so 200 stops the + // provider retrying; only a genuine server failure gets a 5xx. + c.Status(http.StatusOK) return } - c.Data(http.StatusBadRequest, "text/html; charset=utf-8", unsubPage("This unsubscribe link is invalid.")) + renderUnsubPage(c, http.StatusBadRequest, unsubInvalid(err)) + return + } + if claims.ContactID == uuid.Nil { + if oneClick { + c.Status(http.StatusOK) + return + } + renderUnsubPage(c, http.StatusOK, unsubView{Title: "This was a test email", Body: "Test sends carry a link that is not tied to anyone, so there is nothing to unsubscribe."}) return } - xerr := h.AdvancedService.Unsubscribe(c.Request.Context(), cid, rid) + // A browser POST without the confirm field is not the button: show the + // confirm page again rather than act on it. + if !oneClick && !confirmed { + renderUnsubPage(c, http.StatusOK, unsubView{ + Title: "Unsubscribe from these emails?", + Body: "Confirm and you will not receive further emails from this sender.", + Confirm: c.Request.URL.Path, + }) + return + } - if isPost { - // RFC 8058: acknowledge one-click. Return 5xx only on a genuine - // server-side failure so the provider can retry; a bad/expired link - // (BadRequest) is terminal, so 200 to stop pointless retries. + via := "link" + if oneClick { + via = "one_click" + } + xerr := h.AdvancedService.UnsubscribeFromLink(c.Request.Context(), claims.OrgID, claims.CampaignID, claims.ContactID, via) + + if oneClick { if xerr != nil && xerr.Code != errx.BadRequest { c.Status(http.StatusBadGateway) return @@ -41,18 +98,81 @@ func (h *Handler) Unsubscribe(c *gin.Context) { c.Status(http.StatusOK) return } - - msg := "You've been unsubscribed." if xerr != nil { - msg = "We couldn't process that unsubscribe link." + renderUnsubPage(c, http.StatusOK, unsubView{Title: "We couldn't process that link", Body: "The link is no longer valid. Reply to the email instead and the sender will stop."}) + return } - c.Data(http.StatusOK, "text/html; charset=utf-8", unsubPage(msg)) + renderUnsubPage(c, http.StatusOK, unsubView{ + Title: "You've been unsubscribed", + Body: "You will not receive further emails from this sender.", + Resubscribe: c.Request.URL.Path + "/resubscribe", + }) } -func unsubPage(msg string) []byte { - return []byte(`` + - `Unsubscribe` + - `` + - `

` + msg + `

` + - `

You will no longer receive emails from this sender.

`) +func (h *Handler) UnsubscribeUndo(c *gin.Context) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, unsubscribeBodyLimit) + claims, ok := h.unsubscribeClaims(c) + if !ok { + return + } + if claims.ContactID == uuid.Nil { + renderUnsubPage(c, http.StatusBadRequest, unsubInvalid(unsublink.ErrInvalid)) + return + } + if xerr := h.AdvancedService.Resubscribe(c.Request.Context(), claims.OrgID, claims.ContactID); xerr != nil { + renderUnsubPage(c, http.StatusOK, unsubView{Title: "We couldn't process that link", Body: "The link is no longer valid. Reply to the email and the sender can add you back."}) + return + } + renderUnsubPage(c, http.StatusOK, unsubView{Title: "You're subscribed again", Body: "The sender can email you as before. You can unsubscribe from any later email."}) +} + +func (h *Handler) verifyUnsubscribeToken(token string) (unsublink.Claims, error) { + if h.UnsubscribeLinks == nil { + return unsublink.Claims{}, unsublink.ErrInvalid + } + return h.UnsubscribeLinks.Verify(token, time.Now()) +} + +func (h *Handler) unsubscribeClaims(c *gin.Context) (unsublink.Claims, bool) { + claims, err := h.verifyUnsubscribeToken(c.Param("token")) + if err != nil { + renderUnsubPage(c, http.StatusBadRequest, unsubInvalid(err)) + return claims, false + } + return claims, true +} + +func unsubInvalid(err error) unsubView { + if err == unsublink.ErrExpired { + return unsubView{Title: "This link has expired", Body: "Reply to the email instead and the sender will stop."} + } + return unsubView{Title: "This unsubscribe link is invalid", Body: "Reply to the email instead and the sender will stop."} +} + +type unsubView struct { + Title string + Body string + Confirm string // POST target of the confirm button, when shown + Resubscribe string // POST target of the resubscribe button, when shown +} + +// A neutral page: the email came from the customer's mailbox, so the page +// names no brand and carries no scripts or external assets. +var unsubTemplate = template.Must(template.New("unsubscribe").Parse(` +{{.Title}} + +

{{.Title}}

{{.Body}}

+{{if .Confirm}}
{{end}} +{{if .Resubscribe}}
{{end}} +`)) + +func renderUnsubPage(c *gin.Context, status int, v unsubView) { + c.Header("Cache-Control", "no-store") + c.Header("X-Robots-Tag", "noindex") + c.Status(status) + c.Header("Content-Type", "text/html; charset=utf-8") + _ = unsubTemplate.Execute(c.Writer, v) } diff --git a/internal/api/middleware/apikey.go b/internal/api/middleware/apikey.go index 255e3c23..8069c6b2 100644 --- a/internal/api/middleware/apikey.go +++ b/internal/api/middleware/apikey.go @@ -336,8 +336,7 @@ func RequireAPIKeyEmailAccountParam(param string) gin.HandlerFunc { return } - allowed := GetAPIKeyAllowedEmailAccounts(c) - if len(allowed) == 0 { + if len(GetAPIKeyAllowedEmailAccounts(c)) == 0 { c.Next() return } @@ -349,11 +348,9 @@ func RequireAPIKeyEmailAccountParam(param string) gin.HandlerFunc { return } - for _, id := range allowed { - if id == accountID { - c.Next() - return - } + if APIKeyAllowsEmailAccount(c, accountID) { + c.Next() + return } errx.Handle(c, errx.New(errx.Forbidden, "email account is not allowed for this API key")) @@ -361,6 +358,24 @@ func RequireAPIKeyEmailAccountParam(param string) gin.HandlerFunc { } } +// APIKeyAllowsEmailAccount reports whether the authenticating API key may act +// on accountID. JWT callers and unrestricted keys always may. +func APIKeyAllowsEmailAccount(c *gin.Context, accountID uuid.UUID) bool { + if c.GetString(AuthTypeKey) != AuthTypeAPIKey { + return true + } + allowed := GetAPIKeyAllowedEmailAccounts(c) + if len(allowed) == 0 { + return true + } + for _, id := range allowed { + if id == accountID { + return true + } + } + return false +} + // GetAuthType returns "jwt" or "api_key" (empty if unauthenticated). func GetAuthType(c *gin.Context) string { return c.GetString(AuthTypeKey) diff --git a/internal/api/middleware/ratelimit_ip.go b/internal/api/middleware/ratelimit_ip.go index f2ecea79..579009f9 100644 --- a/internal/api/middleware/ratelimit_ip.go +++ b/internal/api/middleware/ratelimit_ip.go @@ -16,8 +16,27 @@ const ( // password, and far below what credential stuffing needs. authIPWindow = 15 * time.Minute authIPDefaultLimit = 60 + + // The CLI sign-in handshake gets its own budget, on its own key. + // + // It cannot share the auth one: `warmbly auth login` polls every + // CLIAuthPollIntervalSeconds for up to CLIAuthCodeTTLMinutes, which is + // around 200 requests for a single sign-in. On the shared budget that + // exhausts the allowance in three minutes, and then blocks the person's + // actual login from the same address for the rest of the window. The + // allowance below covers two concurrent sign-ins from one NAT with slack. + cliAuthIPWindow = 15 * time.Minute + cliAuthIPDefaultLimit = 500 ) +// CLIAuthIPRateLimitMiddleware throttles the public CLI sign-in handshake per +// source IP, on a key of its own so a long poll cannot lock the same address +// out of signing in through the browser. +func (h *Handler) CLIAuthIPRateLimitMiddleware() gin.HandlerFunc { + return h.ipRateLimiter("cli_auth_ip:", cliAuthIPDefaultLimit, "CLI_AUTH_IP_RATE_LIMIT", cliAuthIPWindow, + "Too many CLI sign-in requests from this address. Try again later.") +} + // AuthIPRateLimitMiddleware throttles the public /auth group per source IP. // // This is the only limiter those routes have. RateLimitMiddleware keys on the @@ -29,8 +48,15 @@ const ( // Fails open on a cache error, deliberately: a Redis blip must not lock every // user out of their own instance. func (h *Handler) AuthIPRateLimitMiddleware() gin.HandlerFunc { - limit := authIPDefaultLimit - if v := os.Getenv("AUTH_IP_RATE_LIMIT"); v != "" { + return h.ipRateLimiter("auth_ip:", authIPDefaultLimit, "AUTH_IP_RATE_LIMIT", authIPWindow, + "Too many authentication attempts from this address. Try again later.") +} + +// ipRateLimiter is the shared fixed-window limiter behind both. Each caller +// brings its own Redis key prefix, so budgets never bleed into each other. +func (h *Handler) ipRateLimiter(prefix string, defaultLimit int, env string, window time.Duration, message string) gin.HandlerFunc { + limit := defaultLimit + if v := os.Getenv(env); v != "" { if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 { limit = parsed } @@ -53,21 +79,38 @@ func (h *Handler) AuthIPRateLimitMiddleware() gin.HandlerFunc { return } - key := "auth_ip:" + ip + key := prefix + ip n, err := h.Cache.Incr(c.Request.Context(), key).Result() if err != nil { c.Next() return } + // A counter with no TTL never resets, so the address it belongs to + // stays blocked forever once it passes the limit. That is a worse + // outcome than not counting at all, so a failed EXPIRE drops the key + // and lets the request through, matching how the rest of this + // middleware handles a cache it cannot trust. if n == 1 { - _ = h.Cache.Expire(c.Request.Context(), key, authIPWindow).Err() + if err := h.Cache.Expire(c.Request.Context(), key, window).Err(); err != nil { + _ = h.Cache.Del(c.Request.Context(), key).Err() + c.Next() + return + } + } else if n > int64(limit) { + // Repair a key that lost its expiry some other way (an older + // build, a restore, an eviction between the INCR and the EXPIRE + // above). Only on the reject path, which is rare, so it costs a + // round trip nobody feels. + if ttl, terr := h.Cache.TTL(c.Request.Context(), key).Result(); terr == nil && ttl < 0 { + _ = h.Cache.Expire(c.Request.Context(), key, window).Err() + } } if n > int64(limit) { - c.Header("Retry-After", fmt.Sprintf("%d", int(authIPWindow.Seconds()))) + c.Header("Retry-After", fmt.Sprintf("%d", int(window.Seconds()))) c.JSON(http.StatusTooManyRequests, gin.H{ "error": "rate_limit_exceeded", - "message": "Too many authentication attempts from this address. Try again later.", + "message": message, "code": "rate_limit_exceeded", }) c.Abort() diff --git a/internal/api/routes.go b/internal/api/routes.go index 4b33b5b8..d4aa56ec 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -97,11 +97,12 @@ func Run( // postMessages code+state to the SPA opener, which calls oauth/finish. r.GET("/integrations/oauth/callback", h.IntegrationOAuthCallback) - // Public List-Unsubscribe endpoint (RFC 8058). GET = recipient clicks the - // link; POST = mailbox provider's one-click (body List-Unsubscribe=One-Click). - // Both suppress the recipient org-wide. Unauthenticated by design. - r.GET("/unsubscribe", h.Unsubscribe) - r.POST("/unsubscribe", h.Unsubscribe) + // Public recipient unsubscribe (RFC 8058 one-click and the link in the + // email). The path token is signed per recipient; GET only shows a + // confirm page, POST suppresses. Unauthenticated by design. + r.GET("/unsubscribe/:token", h.UnsubscribePage) + r.POST("/unsubscribe/:token", h.UnsubscribeSubmit) + r.POST("/unsubscribe/:token/resubscribe", h.UnsubscribeUndo) // Public invitation preview for the /invite landing page. Unauthenticated: // the secret token in the query is the capability. @@ -239,6 +240,18 @@ func Run( poolLinkPublic.POST("/poll", h.PoolLinkPoll) } + // `warmbly auth login`. Unauthenticated by nature (the CLI has no key yet), + // so it is throttled per source IP, but on its OWN budget: one sign-in + // polls around 200 times, which would exhaust the auth allowance and then + // lock the same address out of the browser login for the rest of the + // window. + cliAuthPublic := v1.Group("/auth/cli") + cliAuthPublic.Use(m.CLIAuthIPRateLimitMiddleware()) + { + cliAuthPublic.POST("/code", h.CLIAuthStart) + cliAuthPublic.POST("/poll", h.CLIAuthPoll) + } + auth := v1.Group("/auth") // Every unauthenticated auth route shares one per-IP budget. Nothing // throttled these before: RateLimitMiddleware is keyed on the user id and @@ -315,6 +328,10 @@ func Run( protectedAuth.DELETE("/sessions", h.SessionRevokeOthers) protectedAuth.DELETE("/sessions/:id", h.SessionRevoke) + // The instance's version and whether an update exists, for the + // dashboard's version pill. Any member may read it; applying an update + // stays behind the admin permissions on /admin/instance/update. + protectedAuth.GET("/instance", h.InstanceVersion) protectedAuth.GET("/me", h.GetUser) protectedAuth.PATCH("/me", h.UpdateUserProfile) protectedAuth.PATCH("/me/onboarding", h.CompleteOnboarding) @@ -394,6 +411,8 @@ func Run( // Bulk tag add/remove across many mailboxes (set semantics, // naturally idempotent). Static path beside /:id like /verify. emails.PATCH("/tags", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), h.BulkTagEmails) + // How many mailboxes the workspace holds and may hold, and why. + emails.GET("/allowance", m.RequireOrganization(), m.RequireAccess(models.PermManageEmails, models.APIPermReadEmails), h.GetMailboxAllowance) emails.GET("/:id/track", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailTrackingDomain) emails.PATCH("/:id/track", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailTrackingDomain) // Write-scoped like the auth-check refresh: persisting the @@ -432,6 +451,9 @@ func Run( onboardingEmails.POST("/oauth/start", h.StartEmailOAuth) onboardingEmails.POST("/oauth/finish", h.FinishEmailOAuth) onboardingEmails.POST("/smtp-imap", h.ConnectEmailSMTPIMAP) + // The CSV import: up to MailboxBulkBatchMax rows per call, answered + // per row. Same bar as a single connect. + onboardingEmails.POST("/smtp-imap/bulk", h.ConnectEmailSMTPIMAPBulk) // Reconnect flows for an existing mailbox whose credential the // provider invalidated (issue #274). They mutate an existing // org asset, so unlike first connect they sit behind the same @@ -460,6 +482,10 @@ func Run( // lives one level up). protected.GET("/campaigns-overview", m.RequireOrganization(), m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), h.GetCampaignsOverview) + // Audience-versus-pool projection for the campaign wizard (no + // campaign id yet). Read-level: it writes nothing. + protected.POST("/campaigns-estimate", m.RateLimitMiddleware(models.RateLimitRead), m.RequireOrganization(), m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), h.EstimateCampaign) + campaigns := protected.Group("/campaigns") campaigns.Use(m.RateLimitMiddleware(models.RateLimitWrite)) { @@ -740,6 +766,11 @@ func Run( // API key management. JWT users need PermManageAPIKeys; API keys // need the APIPermAPIKeys self-service bit. This lets an integration // rotate its own keys without going through the dashboard. + // Self-revocation, outside the API_KEYS gate below on purpose: any + // valid key may end itself, which is what makes signing a machine + // out actually end its access. + protected.DELETE("/api-keys/self", m.RequireOrganization(), m.RateLimitMiddleware(models.RateLimitWrite), h.RevokeOwnAPIKey) + apiKeys := protected.Group("/api-keys") apiKeys.Use(m.RequireOrganization(), m.RequireAccess(models.PermManageAPIKeys, models.APIPermAPIKeys)) apiKeys.Use(m.RateLimitMiddleware(models.RateLimitWrite)) @@ -794,6 +825,17 @@ func Run( outreach.PATCH("/settings", h.UpdateOutreachSettings) } + // The workspace suppression list (org-scoped). Reading it is a + // contacts read; adding or lifting an entry changes who gets mail, + // so it needs the contacts write permission. + suppressions := protected.Group("/suppressions") + suppressions.Use(m.RequireOrganization()) + { + suppressions.GET("", m.RateLimitMiddleware(models.RateLimitRead), m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.ListSuppressions) + suppressions.POST("", m.RateLimitMiddleware(models.RateLimitWrite), m.RequireAccess(models.PermManageContacts, models.APIPermWriteContacts), h.AddSuppressions) + suppressions.DELETE("/:id", m.RateLimitMiddleware(models.RateLimitWrite), m.RequireAccess(models.PermManageContacts, models.APIPermWriteContacts), h.RemoveSuppression) + } + // Deliverability event ingestion (org-scoped). API-key callable so // downstream pipelines (e.g. SES bounce processors) can post events // without a human in the loop. @@ -1198,6 +1240,18 @@ func Run( poolLink.GET("/instances", m.RequireOrganization(), m.RequirePermission(models.PermManageSettings), h.PoolLinkListInstances) poolLink.DELETE("/instances/:id", m.RequireOrganization(), m.RequirePermission(models.PermManageSettings), h.PoolLinkRevokeInstance) } + + // Browser half of `warmbly auth login`: a member reviews the code + // and approves it into one of their workspaces. Session-only, like + // the pool link approval, because approving mints a credential and + // an API key must not be able to mint another CLI's key. + cliAuth := jwtOnly.Group("/auth/cli") + cliAuth.Use(m.RateLimitMiddleware(models.RateLimitWrite)) + { + cliAuth.GET("/codes/:code", h.CLIAuthDescribeCode) + cliAuth.POST("/codes/:code/approve", h.CLIAuthApproveCode) + cliAuth.POST("/codes/:code/deny", h.CLIAuthDenyCode) + } // The linked instance's own surface, authenticated by its token. poolLinkInstance := base.Group("/pool-link/instance") poolLinkInstance.Use(m.PoolLinkAuthMiddleware()) @@ -1437,6 +1491,16 @@ func Run( adminRoutes.GET("/instance/limits", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminInstanceLimits) adminRoutes.GET("/instance/settings", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminGetInstanceSettings) adminRoutes.PUT("/instance/settings", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminPutInstanceSettings) + // Operator notification channels: the channels themselves are part of + // the settings document above; these two are the event catalog the + // panel renders and the on-demand delivery probe. + adminRoutes.GET("/instance/notifications/events", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminNotificationEvents) + adminRoutes.POST("/instance/notifications/test", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminTestNotificationChannel) + // Updates: the top-bar indicator polls the state; applying one goes + // through the host-side updater and restarts this process. + adminRoutes.GET("/instance/update", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminUpdateState) + adminRoutes.POST("/instance/update/check", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminUpdateCheck) + adminRoutes.POST("/instance/update/apply", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminUpdateApply) // Analytics Dashboard adminRoutes.GET("/analytics/overview", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetPlatformOverview) diff --git a/internal/app/advanced/content_score_test.go b/internal/app/advanced/content_score_test.go index 843ce8f6..27c9cef9 100644 --- a/internal/app/advanced/content_score_test.go +++ b/internal/app/advanced/content_score_test.go @@ -4,6 +4,8 @@ import ( "strings" "testing" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" ) @@ -29,7 +31,7 @@ func TestWorstStepContentScoreSkipsNonEmailSteps(t *testing.T) { emailStep(3, "Following up on my note", good), } - worst, _, _, scored := worstStepContentScore(seqs, 0) + worst, _, _, scored := worstStepContentScore(seqs, nil) if scored != 2 { t.Errorf("scored %d steps, want the 2 email steps", scored) } @@ -44,7 +46,7 @@ func TestWorstStepContentScoreReportsNothingToScore(t *testing.T) { _, _, _, scored := worstStepContentScore([]models.Sequence{ {Kind: "wait", Position: 0}, {Kind: "action", Position: 1}, - }, 0) + }, nil) if scored != 0 { t.Errorf("scored %d steps, want 0", scored) } @@ -60,7 +62,7 @@ func TestWorstStepContentScoreReportsThePositionOfTheWorstStep(t *testing.T) { emailStep(2, "FREE CASH PRIZE GUARANTEED!!!", "Act now, click here, 100% free, risk free."), } - worst, step, issue, scored := worstStepContentScore(seqs, 0) + worst, step, issue, scored := worstStepContentScore(seqs, nil) if scored != 2 { t.Fatalf("scored %d steps, want 2", scored) } @@ -78,20 +80,46 @@ func TestWorstStepContentScoreReportsThePositionOfTheWorstStep(t *testing.T) { // An empty step list falls out with nothing scored: the caller reports that // rather than treating it as passing content. func TestWorstStepContentScoreOnEmptyCampaign(t *testing.T) { - if _, _, _, scored := worstStepContentScore(nil, 0); scored != 0 { + if _, _, _, scored := worstStepContentScore(nil, nil); scored != 0 { t.Errorf("scored %d steps on an empty campaign, want 0", scored) } } -// Attachments are campaign-wide, so preflight weighs them the way the send path -// does instead of reporting a score the activity feed later contradicts. +// Preflight weighs attachments the way the send path does instead of reporting +// a score the activity feed later contradicts. func TestWorstStepContentScoreCountsAttachments(t *testing.T) { good := strings.Repeat("A real sentence about the recipient's work. ", 5) seqs := []models.Sequence{emailStep(0, "Quick question about hiring", good)} - clean, _, _, _ := worstStepContentScore(seqs, 0) - withAtt, _, _, _ := worstStepContentScore(seqs, 2) + clean, _, _, _ := worstStepContentScore(seqs, nil) + withAtt, _, _, _ := worstStepContentScore(seqs, func(models.Sequence) int { return 2 }) if withAtt >= clean { t.Errorf("attachment score %d not below the clean %d", withAtt, clean) } } + +// A file scoped to one step is only carried by that step, so it may only drag +// that step's score down: counting it against every step made preflight report +// a step the send path would never warn about. +func TestWorstStepContentScoreCountsAttachmentsPerStep(t *testing.T) { + good := strings.Repeat("A real sentence about the recipient's work. ", 5) + first := emailStep(0, "Quick question about hiring", good) + first.ID = uuid.New() + second := emailStep(1, "Following up on my note", good) + second.ID = uuid.New() + + perStep := map[uuid.UUID]int{first.ID: 3} + worst, step, _, scored := worstStepContentScore([]models.Sequence{first, second}, func(seq models.Sequence) int { + return perStep[seq.ID] + }) + if scored != 2 { + t.Fatalf("scored %d steps, want 2", scored) + } + if step != 1 { + t.Errorf("worst step reported as %d, want 1 (the step holding the files)", step) + } + clean, _, _, _ := worstStepContentScore([]models.Sequence{second}, nil) + if worst >= clean { + t.Errorf("step with attachments scored %d, want below the clean %d", worst, clean) + } +} diff --git a/internal/app/advanced/service.go b/internal/app/advanced/service.go index ca879ddd..476917d1 100644 --- a/internal/app/advanced/service.go +++ b/internal/app/advanced/service.go @@ -4,13 +4,17 @@ import ( "context" "encoding/json" "fmt" - "github.com/warmbly/warmbly/internal/pkg/emailverify" "hash/fnv" "math/rand" "net/mail" + "regexp" "strings" "time" + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/pkg/emailverify" + "github.com/warmbly/warmbly/internal/utils/validate" + "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/listgate" "github.com/warmbly/warmbly/internal/app/replyclassify" @@ -56,6 +60,19 @@ type Service interface { // (one-click POST or the manual link). Always suppresses — it's an explicit // recipient request, independent of the auto-suppress settings. Unsubscribe(ctx context.Context, campaignID, contactID uuid.UUID) *errx.Error + // UnsubscribeFromLink is Unsubscribe for a verified link token: the + // organization in the token must own the campaign, and via names the + // mechanism ("one_click" for the RFC 8058 POST, "link" for a click). + UnsubscribeFromLink(ctx context.Context, organizationID, campaignID, contactID uuid.UUID, via string) *errx.Error + // Resubscribe undoes a recipient's own unsubscribe from the hosted page. + // Only an entry the recipient made (source "unsubscribe") is removed; a + // bounce, complaint or manual entry stays. + Resubscribe(ctx context.Context, organizationID, contactID uuid.UUID) *errx.Error + + // The workspace suppression list. + ListSuppressions(ctx context.Context, organizationID uuid.UUID, q string, beforeAt *time.Time, beforeID *uuid.UUID, limit int) ([]models.SuppressedRecipient, *errx.Error) + AddSuppressions(ctx context.Context, organizationID, actorID uuid.UUID, req *models.AddSuppressionsRequest) (*models.AddSuppressionsResult, *errx.Error) + RemoveSuppression(ctx context.Context, organizationID, id uuid.UUID) (*models.SuppressedRecipient, *errx.Error) SelectVariant(ctx context.Context, organizationID, campaignID, contactID, sequenceID uuid.UUID, subject, bodyHTML, bodyPlain string) (*models.VariantSelection, *errx.Error) OptimizeSendTime(ctx context.Context, organizationID uuid.UUID, contact *models.Contact, base time.Time) (time.Time, *errx.Error) @@ -513,34 +530,185 @@ func (s *service) ListPipelines(ctx context.Context, orgID uuid.UUID) ([]models. } func (s *service) Unsubscribe(ctx context.Context, campaignID, contactID uuid.UUID) *errx.Error { + return s.unsubscribe(ctx, nil, campaignID, contactID, "action") +} + +func (s *service) UnsubscribeFromLink(ctx context.Context, organizationID, campaignID, contactID uuid.UUID, via string) *errx.Error { + if via != "one_click" { + via = "link" + } + return s.unsubscribe(ctx, &organizationID, campaignID, contactID, via) +} + +// unsubscribe records an explicit opt-out: the address goes on the workspace +// suppression list and the contact's own subscription flag is cleared, so the +// CRM and the send gate tell the same story. +func (s *service) unsubscribe(ctx context.Context, expectOrg *uuid.UUID, campaignID, contactID uuid.UUID, via string) *errx.Error { campaign, err := s.campaignRepo.GetByID(ctx, campaignID) if err != nil || campaign == nil || campaign.OrganizationID == nil { return errx.New(errx.BadRequest, "invalid unsubscribe link") } + if expectOrg != nil && *expectOrg != *campaign.OrganizationID { + return errx.New(errx.BadRequest, "invalid unsubscribe link") + } contact, cerr := s.contactRepo.GetByID(ctx, contactID) if cerr != nil || contact == nil || contact.Email == "" { return errx.New(errx.BadRequest, "invalid unsubscribe link") } + reason := map[string]string{ + "one_click": "one-click unsubscribe (mail client)", + "link": "clicked the unsubscribe link", + "action": "unsubscribed by a sequence action", + }[via] if err := s.repo.UpsertSuppressedRecipient(ctx, &models.SuppressedRecipient{ OrganizationID: *campaign.OrganizationID, Email: contact.Email, - Reason: "one-click unsubscribe", + Kind: models.SuppressionKindEmail, + Reason: reason, Source: models.DeliverabilityEventUnsubscribe, CampaignID: &campaignID, + Metadata: map[string]interface{}{"via": via}, }); err != nil { return toErrx(err) } + if err := s.contactRepo.SetSubscribedByEmail(ctx, *campaign.OrganizationID, contact.Email, false); err != nil { + log.Warn().Err(err).Str("contact_id", contactID.String()).Msg("unsubscribe: could not clear the contact's subscription flag") + } s.emit(ctx, *campaign.OrganizationID, models.WebhookEventCampaignUnsubscribed, map[string]any{ "campaign_id": campaignID.String(), "contact_id": contactID.String(), "contact_email": contact.Email, - "source": "one_click", + "source": via, }) return nil } +func (s *service) Resubscribe(ctx context.Context, organizationID, contactID uuid.UUID) *errx.Error { + contact, cerr := s.contactRepo.GetByID(ctx, contactID) + if cerr != nil || contact == nil || contact.Email == "" { + return errx.New(errx.BadRequest, "invalid unsubscribe link") + } + // The contact must belong to the organization in the token. + if owned, oerr := s.contactRepo.GetByEmailAndOrganization(ctx, organizationID, contact.Email); oerr != nil || owned == nil { + return errx.New(errx.BadRequest, "invalid unsubscribe link") + } + if _, err := s.repo.DeleteSuppressionByEmail(ctx, organizationID, contact.Email, models.DeliverabilityEventUnsubscribe); err != nil { + return toErrx(err) + } + if err := s.contactRepo.SetSubscribedByEmail(ctx, organizationID, contact.Email, true); err != nil { + return toErrx(err) + } + return nil +} + +func (s *service) ListSuppressions(ctx context.Context, organizationID uuid.UUID, q string, beforeAt *time.Time, beforeID *uuid.UUID, limit int) ([]models.SuppressedRecipient, *errx.Error) { + out, err := s.repo.ListSuppressedRecipients(ctx, organizationID, q, beforeAt, beforeID, limit) + if err != nil { + return nil, toErrx(err) + } + return out, nil +} + +// suppressionDomain matches a bare host ("acme.com", "mail.acme.co.uk"). +var suppressionDomain = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$`) + +// AddSuppressions adds each value as an address, or as a domain when it has +// no local part ("acme.com" or "@acme.com"). Unparseable values are reported +// back rather than failing the whole batch, because the batch is usually a +// pasted list with a stray header or blank line in it. +func (s *service) AddSuppressions(ctx context.Context, organizationID, actorID uuid.UUID, req *models.AddSuppressionsRequest) (*models.AddSuppressionsResult, *errx.Error) { + if req == nil || len(req.Entries) == 0 { + return nil, errx.New(errx.BadRequest, "entries are required") + } + if len(req.Entries) > 5000 { + return nil, errx.New(errx.BadRequest, "at most 5000 entries per request") + } + source := models.SuppressionSourceManual + if len(req.Entries) > 1 { + source = models.SuppressionSourceImport + } + res := &models.AddSuppressionsResult{Skipped: []string{}} + seen := map[string]bool{} + entries := make([]models.SuppressedRecipient, 0, len(req.Entries)) + for _, e := range req.Entries { + raw := strings.TrimSpace(e.Value) + value := strings.ToLower(strings.TrimPrefix(raw, "@")) + if value == "" || seen[value] { + continue + } + kind := models.SuppressionKindEmail + if strings.Contains(value, "@") { + if !validate.Email(value) { + res.Skipped = append(res.Skipped, raw) + continue + } + } else if suppressionDomain.MatchString(value) { + kind = models.SuppressionKindDomain + } else { + res.Skipped = append(res.Skipped, raw) + continue + } + seen[value] = true + reason := strings.TrimSpace(e.Reason) + if reason == "" { + reason = strings.TrimSpace(req.Reason) + } + if reason == "" { + reason = "added to the suppression list" + } + if r := []rune(reason); len(r) > models.UnsubscribeCopyMaxLen { + reason = string(r[:models.UnsubscribeCopyMaxLen]) + } + entries = append(entries, models.SuppressedRecipient{ + OrganizationID: organizationID, + Email: value, + Kind: kind, + Reason: reason, + Source: source, + Metadata: map[string]interface{}{"added_by": actorID.String()}, + }) + } + // One transaction: a pasted list lands whole or not at all, so a failure + // part-way never leaves the caller guessing which half got in. + if err := s.repo.UpsertSuppressedRecipients(ctx, entries); err != nil { + return nil, toErrx(err) + } + for _, e := range entries { + if e.Kind != models.SuppressionKindEmail { + continue + } + if err := s.contactRepo.SetSubscribedByEmail(ctx, organizationID, e.Email, false); err != nil { + log.Warn().Err(err).Msg("suppression: could not clear the contact's subscription flag") + } + } + res.Added = len(entries) + return res, nil +} + +func (s *service) RemoveSuppression(ctx context.Context, organizationID, id uuid.UUID) (*models.SuppressedRecipient, *errx.Error) { + entry, err := s.repo.GetSuppressedRecipient(ctx, organizationID, id) + if err != nil { + return nil, toErrx(err) + } + if entry == nil { + return nil, errx.New(errx.NotFound, "suppression entry not found") + } + if _, err := s.repo.DeleteSuppressedRecipient(ctx, organizationID, id); err != nil { + return nil, toErrx(err) + } + // Lifting an address's suppression restores the contact too; otherwise + // the send gate still refuses it on the subscription flag and the list + // says one thing while the contact says another. + if entry.Kind == models.SuppressionKindEmail { + if err := s.contactRepo.SetSubscribedByEmail(ctx, organizationID, entry.Email, true); err != nil { + log.Warn().Err(err).Msg("suppression: could not restore the contact's subscription flag") + } + } + return entry, nil +} + // pickVariantWeightedRandom does a weighted random draw over active variants. func pickVariantWeightedRandom(variants []models.CampaignABVariant) *models.CampaignABVariant { total := 0 @@ -742,6 +910,22 @@ func buildReplyHeaders(msg *models.EmailMessageStoreData) map[string][]string { return h } +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func uuidString(id *uuid.UUID) string { + if id == nil { + return "" + } + return id.String() +} + func containsAnyKeyword(text string, keywords []string) bool { if text == "" { return false @@ -984,18 +1168,31 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. actionTaken = "paused_campaign" } + // Reply-based opt-out is what makes the plain "just reply and I'll stop" + // line a real mechanism. The check ignores the quoted history (which + // carries our own opt-out wording) and matches whole phrases only. if settings.ReplyIntent.AutoSuppressOnUnsubWord && - containsAnyKeyword(text, []string{"unsubscribe", "remove me", "stop"}) { + replyclassify.IsOptOut(msg.Subject, firstNonEmpty(msg.BodyText, msg.Snippet)) { _ = s.repo.UpsertSuppressedRecipient(ctx, &models.SuppressedRecipient{ OrganizationID: *account.OrganizationID, Email: sender, - Reason: "reply intent unsubscribe detected", + Kind: models.SuppressionKindEmail, + Reason: "asked to stop in a reply", Source: models.DeliverabilityEventUnsubscribe, CampaignID: campaignID, Metadata: map[string]interface{}{ - "via": "reply_intent", + "via": "reply", }, }) + if err := s.contactRepo.SetSubscribedByEmail(ctx, *account.OrganizationID, sender, false); err != nil { + log.Warn().Err(err).Msg("reply opt-out: could not clear the contact's subscription flag") + } + s.emit(ctx, *account.OrganizationID, models.WebhookEventCampaignUnsubscribed, map[string]any{ + "campaign_id": uuidString(campaignID), + "contact_id": uuidString(contactID), + "contact_email": sender, + "source": "reply", + }) if actionTaken == "" { actionTaken = "suppressed_recipient" } else { @@ -1535,17 +1732,24 @@ func (s *service) RunPreflight(ctx context.Context, organizationID, campaignID u } if settings.Preflight.CheckUnsubscribeHeader { - pass := campaign.UnsubscribeHeader + optOut := settings.Unsubscribe.Effective(campaign.UnsubscribeMode) + bodyOptOut := optOut.Mode != models.UnsubscribeModeOff + pass := campaign.UnsubscribeHeader || bodyOptOut check := models.PreflightCheckResult{ Key: "unsubscribe_header", Passed: pass, Severity: "warning", - Message: "Unsubscribe header is enabled.", + Message: "Recipients have a way to opt out.", } - if !pass { - check.Message = "Unsubscribe header is disabled." - check.Remediation = "Enable unsubscribe_header for compliance and deliverability." - recommendations = append(recommendations, "Enable unsubscribe header.") + switch { + case !pass: + check.Message = "Recipients have no way to opt out: the unsubscribe header and the opt-out line are both off." + check.Remediation = "Turn the opt-out line back on in Settings > Sending or on the campaign, or enable the unsubscribe header." + recommendations = append(recommendations, "Give recipients a way to opt out.") + case !campaign.UnsubscribeHeader: + check.Message = "Opt-out line is on; the List-Unsubscribe header is off." + case !bodyOptOut: + check.Message = "List-Unsubscribe header is on; no opt-out line in the body." } checks = append(checks, check) } @@ -1790,13 +1994,19 @@ func (s *service) ProcessRetryableDeadLetters(ctx context.Context) (int, *errx.E // worstStepContentScore returns the lowest-scoring email step's score, number, // leading issue, and how many steps were scored. Only email steps carry copy: a // wait or action node would otherwise score as the campaign's worst content. -func worstStepContentScore(seqs []models.Sequence, attachments int) (worst, worstStep int, issue string, scored int) { +// attachmentsFor gives the file count of one step's send, which differs per +// step now that a file can be scoped to one. +func worstStepContentScore(seqs []models.Sequence, attachmentsFor func(models.Sequence) int) (worst, worstStep int, issue string, scored int) { worst = 101 for _, seq := range seqs { if seq.Kind != "" && seq.Kind != "email" { continue } scored++ + attachments := 0 + if attachmentsFor != nil { + attachments = attachmentsFor(seq) + } r := warmlint.ScoreWithAttachments(seq.Subject, seq.BodyHTML, seq.BodyPlain, attachments) if r.Score >= worst { continue @@ -1843,9 +2053,11 @@ func (s *service) contentScoreCheck(ctx context.Context, campaignID uuid.UUID, f } } - // Attachments are campaign-wide and the send path scores them, so preflight - // weighs them too rather than reporting a score the feed later contradicts. - attachments := 0 + // The send path scores the files each step actually carries, so preflight + // counts them the same way (campaign-wide plus that step's own) rather than + // reporting a score the feed later contradicts. + campaignWide := 0 + perStep := map[uuid.UUID]int{} if s.attachmentRepo != nil { atts, aerr := s.attachmentRepo.ListByCampaign(ctx, campaignID) if aerr != nil { @@ -1859,10 +2071,18 @@ func (s *service) contentScoreCheck(ctx context.Context, campaignID uuid.UUID, f Remediation: "Re-run preflight.", } } - attachments = len(atts) + for _, a := range atts { + if a.SequenceID == nil { + campaignWide++ + continue + } + perStep[*a.SequenceID]++ + } } - worst, worstStep, issue, scored := worstStepContentScore(seqs, attachments) + worst, worstStep, issue, scored := worstStepContentScore(seqs, func(seq models.Sequence) int { + return campaignWide + perStep[seq.ID] + }) if scored == 0 { return models.PreflightCheckResult{ Key: "content_score", diff --git a/internal/app/aitools/tools_campaigns.go b/internal/app/aitools/tools_campaigns.go index e27f0f41..b33e83a5 100644 --- a/internal/app/aitools/tools_campaigns.go +++ b/internal/app/aitools/tools_campaigns.go @@ -69,6 +69,7 @@ func (d Deps) registerCampaignTools(r *Registry) { InputSchema: objectSchema(map[string]any{ "name": strProp("Campaign name (required)."), "description": strProp("Optional description."), + "kind": strProp("Optional: 'sequence' (default, follow-ups allowed) or 'one_time' (a single message, at most one step)."), "steps": arrProp("Optional email steps to seed the sequence.", objectSchema(map[string]any{ "subject": strProp("Email subject (may contain {{merge}} vars)."), "body": strProp("Email body text (may contain {{merge}} vars)."), @@ -104,8 +105,13 @@ func (d Deps) registerCampaignTools(r *Registry) { "stop_on_reply": boolProp("Stop sequencing a lead once they reply."), "open_tracking": boolProp("Track opens."), "link_tracking": boolProp("Track link clicks."), + "utm_tracking": boolProp("Tag every link with UTM parameters automatically."), + "utm_source": strProp("utm_source override (empty means the default, warmbly)."), + "utm_medium": strProp("utm_medium override (empty means the default, email)."), + "utm_campaign": strProp("utm_campaign override (empty means the campaign name as a slug)."), "text_only": boolProp("Send plain text only."), "unsubscribe_header": boolProp("Send the List-Unsubscribe header (one-click unsubscribe)."), + "unsubscribe_mode": strProp("In-body opt-out: inherit (workspace default), text (reply-to-opt-out line), link (unsubscribe link), off."), "ramp_enabled": boolProp("Gradually ramp daily volume."), "ramp_start": intProp("Ramp starting volume."), "ramp_increment": intProp("Ramp daily increment."), @@ -211,8 +217,13 @@ func (d Deps) updateCampaign(ctx context.Context, inv Invocation, args json.RawM StopOnReply *bool `json:"stop_on_reply"` OpenTracking *bool `json:"open_tracking"` LinkTracking *bool `json:"link_tracking"` + UTMTracking *bool `json:"utm_tracking"` + UTMSource *string `json:"utm_source"` + UTMMedium *string `json:"utm_medium"` + UTMCampaign *string `json:"utm_campaign"` TextOnly *bool `json:"text_only"` UnsubscribeHeader *bool `json:"unsubscribe_header"` + UnsubscribeMode *string `json:"unsubscribe_mode"` RampEnabled *bool `json:"ramp_enabled"` RampStart *int `json:"ramp_start"` RampIncrement *int `json:"ramp_increment"` @@ -232,8 +243,13 @@ func (d Deps) updateCampaign(ctx context.Context, inv Invocation, args json.RawM StopOnReply: in.StopOnReply, OpenTracking: in.OpenTracking, LinkTracking: in.LinkTracking, + UTMTracking: in.UTMTracking, + UTMSource: in.UTMSource, + UTMMedium: in.UTMMedium, + UTMCampaign: in.UTMCampaign, TextOnly: in.TextOnly, UnsubscribeHeader: in.UnsubscribeHeader, + UnsubscribeMode: in.UnsubscribeMode, RampEnabled: in.RampEnabled, RampStart: in.RampStart, RampIncrement: in.RampIncrement, @@ -368,7 +384,7 @@ func (d Deps) listCampaigns(ctx context.Context, inv Invocation, args json.RawMe if limit <= 0 || limit > 50 { limit = 20 } - res, xerr := d.Campaigns.Search(ctx, inv.OrgID.String(), in.Query, "", "", in.Status, fmt.Sprintf("%d", limit)) + res, xerr := d.Campaigns.Search(ctx, inv.OrgID.String(), in.Query, "", "", in.Status, "", fmt.Sprintf("%d", limit)) if xerr != nil { return "", fromErrx(xerr) } @@ -413,8 +429,9 @@ func (d Deps) getCampaignStats(ctx context.Context, inv Invocation, args json.Ra func (d Deps) createCampaignDraft(ctx context.Context, inv Invocation, args json.RawMessage) (string, error) { in, err := decodeArgs[struct { - Name string `json:"name"` - Description string `json:"description"` + Name string `json:"name"` + Description string `json:"description"` + Kind *string `json:"kind"` Steps []struct { Subject string `json:"subject"` Body string `json:"body"` @@ -427,6 +444,9 @@ func (d Deps) createCampaignDraft(ctx context.Context, inv Invocation, args json if in.Name == "" { return "", ErrInvalidArgs } + if in.Kind != nil && *in.Kind != "" && !models.ValidCampaignKind(*in.Kind) { + return "", ErrInvalidArgs + } seqs := make([]models.CreateSequenceInput, 0, len(in.Steps)) for i, st := range in.Steps { @@ -442,6 +462,7 @@ func (d Deps) createCampaignDraft(ctx context.Context, inv Invocation, args json camp, xerr := d.Campaigns.Create(ctx, inv.UserID.String(), &orgID, &models.CreateCampaign{ Name: in.Name, Description: in.Description, + Kind: in.Kind, Sequences: seqs, }) if xerr != nil { diff --git a/internal/app/analytics/service.go b/internal/app/analytics/service.go index 41fa49b4..14aa44e2 100644 --- a/internal/app/analytics/service.go +++ b/internal/app/analytics/service.go @@ -132,12 +132,19 @@ func (s *analyticsService) GetCampaignAnalytics(ctx context.Context, userID, cam return nil, xerr } + // Where and on what people engaged; best-effort, the totals stand alone. + engagement, xerr := s.analyticsRepo.GetCampaignEngagementBreakdown(ctx, campaignID, 8) + if xerr != nil { + engagement = nil + } + return &models.CampaignAnalytics{ CampaignID: campaignID, Name: campaign.Name, Status: campaign.Status, Summary: *summary, Sequences: sequences, + Engagement: engagement, }, nil } diff --git a/internal/app/auth/provision.go b/internal/app/auth/provision.go index e1e1b828..a4561f47 100644 --- a/internal/app/auth/provision.go +++ b/internal/app/auth/provision.go @@ -59,6 +59,9 @@ func (s *authService) createAccount(ctx context.Context, address, passwordHash, // would have accepted the signup anyway. if invite != "" && s.organizationService != nil { if _, err := s.organizationService.AcceptInvitation(ctx, invite, u.ID, u.Email); err == nil { + // An invited account finished signing up just as much as a + // self-serve one; it simply joined an existing workspace. + s.notifyOperatorSignup(u, "") return u, nil } if inviteRequired { @@ -105,9 +108,33 @@ func (s *authService) createAccount(ctx context.Context, address, passwordHash, } } + workspace := "" + if org != nil { + workspace = org.Name + } + s.notifyOperatorSignup(u, workspace) + return u, nil } +// notifyOperatorSignup raises the operator alert for a finished signup. Both +// the invited and the self-serve path go through it so neither can be missed. +func (s *authService) notifyOperatorSignup(u *models.User, workspace string) { + if s.opsNotify == nil || u == nil { + return + } + s.opsNotify.NotifyOperator( + "user.registered", + "New signup: "+u.Email, + "A new account finished signing up.", + map[string]string{ + "Email": u.Email, + "Name": strings.TrimSpace(u.FirstName + " " + u.LastName), + "Workspace": workspace, + }, + ) +} + // signupAllowed enforces DISABLE_REGISTRATION. // // The first-launch exemption is what makes invite_only safe as a default: an diff --git a/internal/app/auth/service.go b/internal/app/auth/service.go index 1bc098ec..c0a1e813 100644 --- a/internal/app/auth/service.go +++ b/internal/app/auth/service.go @@ -54,6 +54,10 @@ type AuthService interface { // referral attribution at signup). WireReferral(r ReferralAttributor) + // WireOperatorNotifier attaches the instance-wide operator alert channel + // (post-construction; nil = no alerts). + WireOperatorNotifier(n OperatorNotifier) + // WireInstanceSettings attaches the database-backed instance settings // (post-construction; nil keeps the permissive defaults). WireInstanceSettings(s InstanceSettings) @@ -104,6 +108,12 @@ type AuthService interface { SSOExchange(ctx context.Context, code, binding string) (*models.LoginResult, *errx.Error) } +// OperatorNotifier is the instance-wide operator alert surface, injected +// post-construction so this package needs no import of it. Nil disables it. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + type authService struct { // orgRisk files signup findings onto the new workspace's posture. // Optional/nil-safe: without it signups are scored but not fused. @@ -120,12 +130,14 @@ type authService struct { trialService trial.TrialService organizationService organization.OrganizationService emailNotificationService notify.EmailNotificationService - cache *cache.Cache - captcha *captcha.Turnstile - appleIDTokens IDTokenVerifier - googleIDTokens IDTokenVerifier - twofa TwoFAChallenger - referral ReferralAttributor + // opsNotify raises instance-wide operator alerts. Nil is the default. + opsNotify OperatorNotifier + cache *cache.Cache + captcha *captcha.Turnstile + appleIDTokens IDTokenVerifier + googleIDTokens IDTokenVerifier + twofa TwoFAChallenger + referral ReferralAttributor // settings is the operator-editable settings document, wired after // construction because it needs the database pool. settings InstanceSettings @@ -159,6 +171,9 @@ func (s *authService) WireIdentities(r repository.IdentityRepository) { s.identi func (s *authService) WireReferral(r ReferralAttributor) { s.referral = r } +// WireOperatorNotifier attaches the operator alert channel. +func (s *authService) WireOperatorNotifier(n OperatorNotifier) { s.opsNotify = n } + // WireInstanceSettings attaches the instance settings document, so the signup // knobs on the admin page reach the registration paths. func (s *authService) WireInstanceSettings(set InstanceSettings) { s.settings = set } diff --git a/internal/app/campaign/handlers.go b/internal/app/campaign/handlers.go index e738aff4..240235b4 100644 --- a/internal/app/campaign/handlers.go +++ b/internal/app/campaign/handlers.go @@ -15,6 +15,8 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/dailythrottle" "github.com/warmbly/warmbly/internal/app/listgate" + "github.com/warmbly/warmbly/internal/app/tz" + "github.com/warmbly/warmbly/internal/bitmask" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" @@ -86,7 +88,7 @@ func (s *campaignService) Get(ctx context.Context, orgID, id string) (*models.Ca return resp, nil } -func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, folder, status, limit string) (*models.CampaignsResult, *errx.Error) { +func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, folder, status, kind, limit string) (*models.CampaignsResult, *errx.Error) { cursorId, err := paging.DecodeCursor(cursor) if err != nil { return nil, err @@ -104,8 +106,11 @@ func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, fold default: return nil, errx.New(errx.BadRequest, "invalid status filter: must be draft, active, paused, or completed") } + if kind != "" && !models.ValidCampaignKind(kind) { + return nil, errx.New(errx.BadRequest, "invalid kind filter: must be sequence or one_time") + } - resp, xerr := s.campaignRepository.Search(ctx, orgID, query, cursorId, folderId, status, limitN) + resp, xerr := s.campaignRepository.Search(ctx, orgID, query, cursorId, folderId, status, kind, limitN) if xerr != nil { return nil, errx.InternalError() } @@ -218,23 +223,37 @@ func (s *campaignService) Duplicate(ctx context.Context, orgID, userID uuid.UUID } newID := uuid.New() - copied, cleanup, xerr := s.copyAttachments(ctx, orgID, cID, newID) + copied, storageLimit, cleanup, xerr := s.copyAttachments(ctx, orgID, cID, newID) if xerr != nil { return nil, xerr } campaign, err := s.campaignRepository.Duplicate(ctx, repository.DuplicateCampaignInput{ - SourceID: cID, - NewID: newID, - UserID: userID, - Name: name, - Attachments: copied, + SourceID: cID, + NewID: newID, + UserID: userID, + Name: name, + Attachments: copied, + OrganizationID: orgID, + StorageLimit: storageLimit, }) if err != nil { cleanup() if errors.Is(err, errx.ErrResourceNotFound) { return nil, errx.ErrNotFound } + if errors.Is(err, repository.ErrStorageQuotaExceeded) { + var adding int64 + for _, att := range copied { + adding += att.Size + } + used, _ := s.attachmentRepo.SumStorageUsedByOrg(ctx, orgID) + var limit int64 + if storageLimit != nil { + limit, _ = storageLimit(ctx) + } + return nil, errx.StorageLimitReached(used, limit, adding) + } return nil, errx.InternalError() } @@ -264,41 +283,49 @@ func (s *campaignService) Duplicate(ctx context.Context, orgID, userID uuid.UUID } // copyAttachments writes a copy of every attachment object of src under dst -// and returns the rows to insert plus a best-effort undo for when the copy -// transaction fails. The copies count against the organization's storage -// quota exactly like an upload would. An attachment whose bytes cannot be -// read is reported and skipped rather than failing the whole duplicate. -func (s *campaignService) copyAttachments(ctx context.Context, orgID, src, dst uuid.UUID) ([]models.CampaignAttachment, func(), *errx.Error) { +// and returns the rows to insert, the storage limit the insert must respect, +// and a best-effort undo for when the copy transaction fails. The copies +// count against the organization's storage quota exactly like an upload +// would: the read here only refuses a hopeless copy before any bytes move, +// and the insert re-checks under the quota lock. An attachment whose bytes +// cannot be read is reported and skipped rather than failing the whole +// duplicate. +func (s *campaignService) copyAttachments(ctx context.Context, orgID, src, dst uuid.UUID) ([]models.CampaignAttachment, repository.StorageLimitFunc, func(), *errx.Error) { noop := func() {} if s.attachmentRepo == nil || s.storage == nil { - return nil, noop, nil + return nil, nil, noop, nil } sources, err := s.attachmentRepo.ListByCampaign(ctx, src) if err != nil { - return nil, noop, errx.InternalError() + return nil, nil, noop, errx.InternalError() } if len(sources) == 0 { - return nil, noop, nil + return nil, nil, noop, nil } + var limit repository.StorageLimitFunc if s.featureGate != nil { - limit, xerr := s.featureGate.GetStorageLimitBytes(ctx, orgID) + l, xerr := s.featureGate.GetStorageLimitBytes(ctx, orgID) if xerr != nil { - return nil, noop, xerr + return nil, nil, noop, xerr + } + limit = func(ctx context.Context) (int64, error) { + v, xerr := s.featureGate.GetStorageLimitBytes(ctx, orgID) + if xerr != nil { + return 0, xerr + } + return v, nil } used, err := s.attachmentRepo.SumStorageUsedByOrg(ctx, orgID) if err != nil { - return nil, noop, errx.InternalError() + return nil, nil, noop, errx.InternalError() } var adding int64 for _, att := range sources { adding += att.Size } - if used+adding > limit { - const mb = 1024 * 1024 - return nil, noop, errx.New(errx.BadRequest, fmt.Sprintf( - "duplicating would exceed your storage limit (%d MB of %d MB used, %d MB of attachments to copy): remove attachments or upgrade your plan", - used/mb, limit/mb, adding/mb)) + if used+adding > l { + return nil, nil, noop, errx.StorageLimitReached(used, l, adding) } } @@ -319,9 +346,13 @@ func (s *campaignService) copyAttachments(ctx context.Context, orgID, src, dst u att.S3Key = key copied = append(copied, att) } - return copied, func() { + return copied, limit, func() { + // The undo must outlive a cancelled request, or the copies are left in + // storage with no row counting them. + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() for _, att := range copied { - if err := s.storage.Delete(ctx, att.S3Key); err != nil { + if err := s.storage.Delete(cleanup, att.S3Key); err != nil { sentry.CaptureException(fmt.Errorf("campaign %s duplicate undo: object %s: %w", src, att.S3Key, err)) } } @@ -535,12 +566,22 @@ func (s *campaignService) WakeCampaigns(ctx context.Context, orgID uuid.UUID, ca seen[id] = true campaign, err := s.campaignRepository.GetByID(ctx, id) - if err != nil || campaign == nil || campaign.Status != "active" { + if err != nil || campaign == nil { continue } if campaign.OrganizationID == nil || *campaign.OrganizationID != orgID { continue } + // Finished only means it ran out of leads, and a lead just arrived by + // whichever path (a linked segment, the API, an automation): restart + // it through the full launch checks, never a raw status flip. + if campaign.Status == "completed" { + s.restartForNewLeads(ctx, orgID, campaign) + continue + } + if campaign.Status != "active" { + continue + } pending, perr2 := s.campaignRepository.GetPendingCampaignTasks(ctx, id) if perr2 != nil { @@ -576,6 +617,63 @@ func (s *campaignService) WakeCampaigns(ctx context.Context, orgID uuid.UUID, ca } } +// restartForNewLeads starts a finished campaign again because leads were added +// to it. A refusal is written to the campaign's activity log (once an hour per +// reason) so the owner can see why the new leads are waiting, instead of a +// finished campaign quietly ignoring them. +func (s *campaignService) restartForNewLeads(ctx context.Context, orgID uuid.UUID, campaign *models.Campaign) { + xerr := s.StartCampaign(ctx, orgID, campaign.ID.String(), models.StartCampaignOptions{Automatic: true}) + if xerr == nil { + return + } + log.Info().Str("campaign_id", campaign.ID.String()).Str("reason", xerr.Message).Msg("finished campaign not restarted for new leads") + if s.campaignLogRepo == nil { + return + } + entry := &repository.CampaignLogEntry{ + CampaignID: campaign.ID, + EventType: "restart_refused", + Message: "New leads arrived but the campaign could not restart: " + xerr.Message + " Fix the cause, then press play.", + Metadata: map[string]interface{}{"reason": xerr.Message}, + } + written, lerr := s.campaignLogRepo.CreateLogOnce(ctx, entry, "reason", xerr.Message, time.Now().Add(-time.Hour)) + if lerr != nil || !written || s.streamingPublisher == nil { + return + } + // An update with no status refreshes the activity feed without moving + // the campaign's badge. + s.streamingPublisher.PublishCampaignEvent(ctx, &pubsub.CampaignEvent{ + BaseEvent: pubsub.BaseEvent{EventType: pubsub.EventCampaignUpdated, UserID: campaign.UserID}, + OrgID: modelOrgID(campaign.OrganizationID), + CampaignID: campaign.ID.String(), + }) +} + +// idleContinuousCampaign keeps a continuous campaign active with nothing to +// send: it waits for leads. Logged and broadcast on the transition only. +func (s *campaignService) idleContinuousCampaign(ctx context.Context, campaign *models.Campaign) { + transitioned, err := s.campaignRepository.MarkIdle(ctx, campaign.ID) + if err != nil || !transitioned { + return + } + if s.campaignLogRepo != nil { + s.campaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ + CampaignID: campaign.ID, + EventType: tasks.CampaignIdleEventType, + Message: tasks.CampaignIdleMessage, + }) + } + if s.streamingPublisher != nil { + s.streamingPublisher.PublishCampaignEvent(ctx, &pubsub.CampaignEvent{ + BaseEvent: pubsub.BaseEvent{EventType: pubsub.EventCampaignIdle, UserID: campaign.UserID}, + OrgID: modelOrgID(campaign.OrganizationID), + CampaignID: campaign.ID.String(), + Name: campaign.Name, + Status: "active", + }) + } +} + func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID uuid.UUID) *errx.Error { if s.scheduler == nil || s.tasksClient == nil || s.taskRepo == nil { return nil @@ -599,8 +697,8 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID case errors.Is(err, scheduler.ErrNoEligibleMailbox): _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_no_accounts") return errx.New(errx.BadRequest, - "this campaign's mailboxes are all outside their sending window or over their daily limit right now; "+ - "check each mailbox's timezone, sending behaviour and daily cap") + "no mailbox on this campaign can send under its current sending settings; "+ + "check each mailbox's sending behaviour profile (working days) and timezone") case errors.Is(err, scheduler.ErrNoEmailAccounts): _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_no_accounts") return errx.New(errx.BadRequest, "no active email accounts found for campaign's email tags") @@ -612,6 +710,11 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID fmt.Sprintf("%d remaining lead(s) were refused by address verification; re-verify them or mark them deliverable to continue", n)) } } + // A continuous campaign starts with nothing to send and waits. + if c, gerr := s.campaignRepository.GetByID(ctx, campaignID); gerr == nil && c != nil && c.Continuous { + s.idleContinuousCampaign(ctx, c) + return nil + } _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "completed") return errx.New(errx.BadRequest, "campaign has no remaining contacts to send") case errors.Is(err, scheduler.ErrCampaignEnded): @@ -644,6 +747,7 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID if !created { return nil } + _ = s.campaignRepository.ClearIdle(ctx, campaignID) cloudTaskName, err := s.tasksClient.CreateTask(ctx, &proto.ProcessTask{TaskId: taskID.String()}, nextTime) if err != nil { @@ -857,3 +961,130 @@ func modelOrgID(orgID *uuid.UUID) string { } return orgID.String() } + +// estimateHorizonDays bounds the finish-date walk; an audience that needs +// longer than this reports no finish date rather than a meaningless one. +const estimateHorizonDays = 2 * 366 + +// Estimate projects an audience against a sender pool. It applies the same +// cap rule as the scheduler (the smaller of the mailbox cap and the campaign +// limit, per mailbox, per day) but none of its pacing, so the result is the +// earliest the last send can land, not a promise. +func (s *campaignService) Estimate(ctx context.Context, orgID uuid.UUID, in *models.CampaignEstimate) (*models.CampaignEstimateResult, *errx.Error) { + out := &models.CampaignEstimateResult{} + + if len(in.SegmentIDs) > models.CampaignSegmentsMax { + return nil, errx.New(errx.BadRequest, fmt.Sprintf("a campaign can link at most %d segments", models.CampaignSegmentsMax)) + } + seen := map[string]bool{} + segmentIDs := make([]string, 0, len(in.SegmentIDs)) + for _, raw := range in.SegmentIDs { + if _, err := uuid.Parse(raw); err != nil { + return nil, errx.New(errx.BadRequest, "invalid segment id") + } + if seen[raw] { + continue + } + seen[raw] = true + segmentIDs = append(segmentIDs, raw) + } + // One preview over "in any of these segments" counts each contact once + // however many of the segments they belong to. + if len(segmentIDs) > 0 && s.segments != nil { + n, xerr := s.segments.Preview(ctx, orgID, &models.SegmentPreview{ + Match: models.SegmentMatchAny, + Conditions: []models.SegmentCondition{{Field: "segment", Operator: models.SegOpIn, Values: segmentIDs}}, + }) + if xerr != nil { + return nil, xerr + } + out.Recipients = n + } + + dailyLimit := config.CampaignLimitDefault + if in.DailyLimit != nil { + if xerr := validate.CampaignDailyLimit(*in.DailyLimit); xerr != nil { + return nil, xerr + } + dailyLimit = *in.DailyLimit + } + + // Same pool resolution as the scheduler: tags when given, otherwise + // every active mailbox in the workspace. + scope := repository.NewAccountScope(&orgID) + var accounts []models.Email + var xerr *errx.Error + if len(in.EmailTagIDs) > 0 { + accounts, xerr = s.emailRepo.GetByTags(ctx, scope, in.EmailTagIDs) + } else { + accounts, xerr = s.emailRepo.GetAllActiveInScope(ctx, scope) + } + if xerr != nil { + return nil, xerr + } + out.Mailboxes = len(accounts) + for _, acct := range accounts { + lim := min(acct.CampaignLimit, dailyLimit) + if lim < 0 { + lim = 0 + } + out.DailyCapacity += lim + sent, err := s.taskRepo.CountCampaignEmailsSentToday(ctx, acct.ID) + if err != nil { + // A counter blip must not blank the whole estimate, but it must + // not flatter it either: a mailbox whose sends today are unknown + // contributes nothing to today and only counts from tomorrow. + continue + } + out.RemainingToday += max(0, lim-sent) + } + if out.Recipients == 0 || out.DailyCapacity == 0 { + return out, nil + } + + // Walk calendar days from the start, spending each sending day's + // capacity, until the audience is covered. Today only has what the pool + // has not already sent. + days := bitmask.DefaultDays() + if in.Days != nil && *in.Days != 0 { + days = *in.Days + } + loc := time.UTC + if in.Timezone != nil && tz.Valid(*in.Timezone) { + if l, err := time.LoadLocation(*in.Timezone); err == nil { + loc = l + } + } + now := time.Now().In(loc) + start := now + if in.StartDate != nil && in.StartDate.After(now) { + start = in.StartDate.In(loc) + } + startDay := time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, loc) + startsToday := startDay.Equal(time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc)) + remaining := out.Recipients + sendingDays := 0 + for i := 0; i < estimateHorizonDays; i++ { + d := startDay.AddDate(0, 0, i) + // Mask bit 0 is Monday; time.Weekday starts at Sunday. + if days&(1<<((int(d.Weekday())+6)%7)) == 0 { + continue + } + capacity := out.DailyCapacity + if i == 0 && startsToday { + capacity = out.RemainingToday + } + if capacity <= 0 { + continue + } + sendingDays++ + remaining -= capacity + if remaining <= 0 { + finish := d + out.SendingDays = &sendingDays + out.EstimatedFinishAt = &finish + break + } + } + return out, nil +} diff --git a/internal/app/campaign/service.go b/internal/app/campaign/service.go index 1033844b..c2d709a8 100644 --- a/internal/app/campaign/service.go +++ b/internal/app/campaign/service.go @@ -19,9 +19,14 @@ const campaignCooldownSeconds = 60 type CampaignService interface { Create(ctx context.Context, userID string, orgID *uuid.UUID, data *models.CreateCampaign) (*models.Campaign, *errx.Error) - Get(ctx context.Context, userID, id string) (*models.Campaign, *errx.Error) - Search(ctx context.Context, userID, query, cursor, folder, status, limit string) (*models.CampaignsResult, *errx.Error) + // Get loads one of orgID's campaigns; any other id is not found. + Get(ctx context.Context, orgID, id string) (*models.Campaign, *errx.Error) + Search(ctx context.Context, userID, query, cursor, folder, status, kind, limit string) (*models.CampaignsResult, *errx.Error) Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, *errx.Error) + // Estimate projects how many contacts a set of segments reaches and how + // many sending days a mailbox pool needs under the per-mailbox caps. + // Read-only; the wizard shows it before a one-time email is created. + Estimate(ctx context.Context, orgID uuid.UUID, in *models.CampaignEstimate) (*models.CampaignEstimateResult, *errx.Error) Update(ctx context.Context, userID, id string, data *models.UpdateCampaign) (*models.Campaign, *errx.Error) // Delete removes an organization's campaign outright. A running campaign // is stopped as part of it: its pending tasks are cancelled in the same @@ -78,6 +83,24 @@ type campaignService struct { // campaignProgressRepo counts the leads verification refused, so a start // with nothing left to send can say why. Optional/nil-safe. campaignProgressRepo repository.CampaignProgressRepository + // segments counts an audience for Estimate. Optional: without it an + // estimate reports zero recipients. + segments SegmentCounter +} + +// SegmentCounter is the slice of the segment service Estimate needs. +// Satisfied structurally by segment.Service. +type SegmentCounter interface { + Preview(ctx context.Context, orgID uuid.UUID, in *models.SegmentPreview) (int, *errx.Error) +} + +// SegmentAware lets main hand the campaign service the segment counter. +type SegmentAware interface { + WireSegments(c SegmentCounter) +} + +func (s *campaignService) WireSegments(c SegmentCounter) { + s.segments = c } // WireAudience attaches the launch-time list gate. diff --git a/internal/app/cliauth/service.go b/internal/app/cliauth/service.go new file mode 100644 index 00000000..1a07b3c3 --- /dev/null +++ b/internal/app/cliauth/service.go @@ -0,0 +1,286 @@ +// Package cliauth is the device-code sign-in the `warmbly` CLI uses. +// +// The CLI has no credential of its own, so it opens a handshake, shows the +// user an eight character code, and polls. A signed-in member approves the +// code in the browser, and the approval mints an ordinary API key through the +// existing service: same hash, same scopes, same revocation, visible under +// Settings > API keys like every other key. Nothing here is a new credential +// type and nothing here is a new authentication path. +package cliauth + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "net/url" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/app/apikey" + "github.com/warmbly/warmbly/internal/app/organization" + "github.com/warmbly/warmbly/internal/app/user" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +var ( + ErrCodeNotFound = errx.NewWithIdentifier(errx.NotFound, "cli_auth_code_not_found", "That code is unknown or has expired. Run `warmbly auth login` again for a fresh one.") + ErrCodeNotPending = errx.NewWithIdentifier(errx.Conflict, "cli_auth_code_used", "That code has already been used.") + ErrBadRequest = errx.NewWithIdentifier(errx.BadRequest, "cli_auth_request", "device_code is required.") + ErrBadScopes = errx.NewWithIdentifier(errx.BadRequest, "cli_auth_scopes", "The requested scopes include bits this instance does not grant.") + ErrForbidden = errx.NewWithIdentifier(errx.Forbidden, "cli_auth_forbidden", "Managing API keys is required to authorize a CLI in this workspace.") +) + +type Service interface { + // StartCode opens a handshake for a CLI that holds no key yet. + StartCode(ctx context.Context, req models.CLIAuthStartRequest) (*models.CLIAuthStartResponse, *errx.Error) + // PollCode is what the CLI calls until a member decides. + PollCode(ctx context.Context, deviceCode string) (*models.CLIAuthPollResponse, *errx.Error) + // DescribeCode is what the approving member sees before deciding. + DescribeCode(ctx context.Context, userCode string) (*models.CLIAuthCode, *errx.Error) + // ApproveCode mints the key into the named workspace. + ApproveCode(ctx context.Context, userCode string, orgID, userID uuid.UUID) (*models.CLIAuthCode, *errx.Error) + DenyCode(ctx context.Context, userCode string) *errx.Error +} + +type service struct { + repo repository.CLIAuthRepository + keys apikey.APIKeyService + orgs organization.OrganizationService + users user.UserService + orgRep repository.OrganizationRepository +} + +func NewService( + repo repository.CLIAuthRepository, + keys apikey.APIKeyService, + orgs organization.OrganizationService, + users user.UserService, + orgRep repository.OrganizationRepository, +) Service { + return &service{repo: repo, keys: keys, orgs: orgs, users: users, orgRep: orgRep} +} + +// Unambiguous alphabet: no 0/O, 1/I/L. Same as the pool link handshake, because +// both codes get read off one screen and typed into another. +const userCodeAlphabet = "ABCDEFGHJKMNPQRSTUVWXYZ23456789" + +func randomUserCode() (string, error) { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + return "", err + } + out := make([]byte, 0, 9) + for i, v := range b { + if i == 4 { + out = append(out, '-') + } + out = append(out, userCodeAlphabet[int(v)%len(userCodeAlphabet)]) + } + return string(out), nil +} + +func randomDeviceCode() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +func hashDeviceCode(t string) string { + sum := sha256.Sum256([]byte(t)) + return hex.EncodeToString(sum[:]) +} + +// NormalizeUserCode accepts any case, with or without the dash, so a user who +// retypes the code by hand is not punished for the formatting. +func NormalizeUserCode(raw string) string { + raw = strings.ToUpper(strings.TrimSpace(raw)) + raw = strings.ReplaceAll(raw, "-", "") + raw = strings.ReplaceAll(raw, " ", "") + if len(raw) != 8 { + return raw + } + return raw[:4] + "-" + raw[4:] +} + +func clip(s string, n int) string { + s = strings.TrimSpace(s) + if len(s) > n { + return s[:n] + } + return s +} + +func (s *service) StartCode(ctx context.Context, req models.CLIAuthStartRequest) (*models.CLIAuthStartResponse, *errx.Error) { + req.ClientName = clip(req.ClientName, 60) + if req.ClientName == "" { + req.ClientName = "Warmbly CLI" + } + req.Hostname = clip(req.Hostname, 80) + req.CLIVersion = clip(req.CLIVersion, 40) + + // An unknown bit would grant a scope the approval screen never showed. + if req.Scopes&^models.AllAPIPermissionsMask != 0 { + return nil, ErrBadScopes + } + if req.Scopes == 0 { + req.Scopes = models.APIPermFullAccess + } + + deviceCode, err := randomDeviceCode() + if err != nil { + return nil, errx.InternalError() + } + _ = s.repo.DeleteExpiredCodes(ctx) + + var code *models.CLIAuthCode + for attempt := 0; attempt < 3; attempt++ { + userCode, uerr := randomUserCode() + if uerr != nil { + return nil, errx.InternalError() + } + created, cerr := s.repo.CreateCode(ctx, hashDeviceCode(deviceCode), userCode, req, time.Now().Add(config.CLIAuthCodeTTLMinutes*time.Minute)) + if cerr == nil && created != nil { + code = created + break + } + // A user-code collision is the only expected failure; retry with a new one. + } + if code == nil { + return nil, errx.InternalError() + } + + verify := config.AppBaseURL() + "/cli" + return &models.CLIAuthStartResponse{ + DeviceCode: deviceCode, + UserCode: code.UserCode, + VerificationURL: verify, + VerificationURLComplete: verify + "?code=" + url.QueryEscape(code.UserCode), + ExpiresIn: config.CLIAuthCodeTTLMinutes * 60, + Interval: config.CLIAuthPollIntervalSeconds, + }, nil +} + +func (s *service) PollCode(ctx context.Context, deviceCode string) (*models.CLIAuthPollResponse, *errx.Error) { + deviceCode = strings.TrimSpace(deviceCode) + if deviceCode == "" { + return nil, ErrBadRequest + } + code, secret, err := s.repo.ClaimCode(ctx, hashDeviceCode(deviceCode)) + if err != nil { + return nil, errx.InternalError() + } + if code == nil { + // Expired and unknown are the same answer on purpose: a poller that + // can tell them apart can probe for live handshakes. + return nil, ErrCodeNotFound + } + res := &models.CLIAuthPollResponse{Status: code.Status} + if secret == "" { + return res, nil + } + + res.Token = secret + res.Scopes = code.Scopes + res.ScopeNames = code.ScopeNames + res.OrganizationID = code.OrganizationID + + // Identity is a convenience for `warmbly auth status`, not part of the + // grant, so a lookup failure must not lose the user their token. + if key, kerr := s.keys.ValidateKey(ctx, secret); kerr == nil && key != nil { + res.APIKeyID = &key.ID + res.UserID = &key.UserID + if u, uerr := s.users.GetUser(ctx, key.UserID); uerr == nil && u != nil { + res.UserEmail = u.Email + res.UserName = strings.TrimSpace(u.FirstName + " " + u.LastName) + } + } + if code.OrganizationID != nil && s.orgRep != nil { + if org, oerr := s.orgRep.GetByID(ctx, *code.OrganizationID); oerr == nil && org != nil { + res.OrganizationName = org.Name + } + } + return res, nil +} + +func (s *service) DescribeCode(ctx context.Context, userCode string) (*models.CLIAuthCode, *errx.Error) { + code, err := s.repo.GetCodeByUserCode(ctx, NormalizeUserCode(userCode)) + if err != nil { + return nil, errx.InternalError() + } + if code == nil { + return nil, ErrCodeNotFound + } + return code, nil +} + +func (s *service) ApproveCode(ctx context.Context, userCode string, orgID, userID uuid.UUID) (*models.CLIAuthCode, *errx.Error) { + userCode = NormalizeUserCode(userCode) + code, xerr := s.DescribeCode(ctx, userCode) + if xerr != nil { + return nil, xerr + } + if code.Status != models.CLIAuthCodePending { + return nil, ErrCodeNotPending + } + + allowed, xerr := s.orgs.HasPermission(ctx, orgID, userID, models.PermManageAPIKeys) + if xerr != nil { + return nil, xerr + } + if !allowed { + return nil, ErrForbidden + } + + // The key is named for the machine that asked, so Settings > API keys shows + // which laptop a key belongs to and revoking the right one is possible. + name := code.ClientName + if code.Hostname != "" { + name += " on " + code.Hostname + } + desc := fmt.Sprintf("Created by `warmbly auth login` for code %s", code.UserCode) + created, xerr := s.keys.Create(ctx, orgID, userID, &models.CreateAPIKey{ + Name: clip(name, 255), + Description: &desc, + Permissions: code.Scopes, + }) + if xerr != nil { + return nil, xerr + } + + ok, err := s.repo.ApproveCode(ctx, userCode, orgID, userID, created.ID, created.Secret) + if err != nil { + return nil, errx.InternalError() + } + if !ok { + // Someone approved or denied between the read and the write. The key + // would otherwise be an orphan nobody asked for. + _ = s.keys.Revoke(ctx, orgID, created.ID, "cli authorization was resolved elsewhere") + return nil, ErrCodeNotPending + } + + code.Status = models.CLIAuthCodeApproved + code.OrganizationID = &orgID + code.APIKeyID = &created.ID + return code, nil +} + +func (s *service) DenyCode(ctx context.Context, userCode string) *errx.Error { + ok, err := s.repo.DenyCode(ctx, NormalizeUserCode(userCode)) + if err != nil { + return errx.InternalError() + } + if !ok { + return ErrCodeNotFound + } + return nil +} diff --git a/internal/app/consumer/dead_worker.go b/internal/app/consumer/dead_worker.go index cafa59aa..4af152c0 100644 --- a/internal/app/consumer/dead_worker.go +++ b/internal/app/consumer/dead_worker.go @@ -3,6 +3,7 @@ package jobs import ( "context" "fmt" + "strconv" "time" "github.com/google/uuid" @@ -17,6 +18,39 @@ type OrgNotifier interface { NotifyOrg(ctx context.Context, orgID uuid.UUID, perm models.OrganizationPermission, exclude uuid.UUID, category models.NotificationCategory, title, body, link string, meta map[string]any, groupKey string) } +// OperatorNotifier is the instance-wide operator alert surface, declared here +// so this package needs no import of it. Nil disables it. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + +// notifyOperatorWorkerDown alerts the operator that a worker is gone. It shares +// the same once-per-incident SetNX guard shape as the tenant notice, under its +// own key so the two audiences are independent. +func (s *JobsService) notifyOperatorWorkerDown(ctx context.Context, workerID uuid.UUID, mailboxes int, reassigned bool) { + if s.OpsNotifier == nil || s.Cache == nil { + return + } + ok, err := s.Cache.SetNX(ctx, "worker:opsnotify:"+workerID.String(), "1", 6*time.Hour).Result() + if err != nil || !ok { + return + } + outcome := "Mailboxes were moved to a healthy worker automatically." + if !reassigned { + outcome = "No healthy replacement of the same tier was available, so sending from those mailboxes is paused." + } + s.OpsNotifier.NotifyOperator( + "worker.offline", + "Worker stopped responding", + outcome, + map[string]string{ + "Worker": workerID.String(), + "Mailboxes": strconv.Itoa(mailboxes), + "Reassigned": map[bool]string{true: "yes", false: "no"}[reassigned], + }, + ) +} + // notifyWorkerDown tells each affected org's manage_emails members about a // dead worker, at most once per worker incident: detection reruns every // interval while the worker stays down, and the SetNX guard keeps that from @@ -116,6 +150,7 @@ func (s *JobsService) detectDeadWorkers(ctx context.Context) { if err != nil || replacement == nil { log.Warn().Str("worker_id", w.ID.String()).Msg("no healthy replacement worker found") s.notifyWorkerDown(ctx, w.ID, s.accountOrgs(ctx, accountIDs), false) + s.notifyOperatorWorkerDown(ctx, w.ID, len(accountIDs), false) continue } @@ -184,6 +219,7 @@ func (s *JobsService) detectDeadWorkers(ctx context.Context) { } s.notifyWorkerDown(ctx, w.ID, affectedOrgs, true) + s.notifyOperatorWorkerDown(ctx, w.ID, reassigned, true) } if reassigned == len(accountIDs) { diff --git a/internal/app/consumer/event_tracking.go b/internal/app/consumer/event_tracking.go index 459d8139..45638f84 100644 --- a/internal/app/consumer/event_tracking.go +++ b/internal/app/consumer/event_tracking.go @@ -4,16 +4,22 @@ import ( "context" "crypto/sha256" "encoding/hex" + "net/netip" + "strings" "time" "github.com/google/uuid" + "github.com/mileusna/useragent" "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/app/instancesettings" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/events" "github.com/warmbly/warmbly/internal/infrastructure/codec" "github.com/warmbly/warmbly/internal/infrastructure/eventbus" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/geo" "github.com/warmbly/warmbly/internal/repository" ) @@ -30,13 +36,44 @@ type TrackingConsumer struct { evidence advanced.EvidenceRecorder streamingPublisher *pubsub.StreamingPublisher dedupeRepo repository.TrackingDedupeRepository + trackedLinks repository.TrackedLinkRepository + linkClicks repository.LinkClickRepository + // afterBurstWindow runs fn once the click burst window has passed, so a + // human click's side effects wait for the burst rule's verdict. + afterBurstWindow func(fn func()) // advancedService fires INSTANT open/click action chains the moment a // tracking event lands (the open/click analog of the reply path in // ProcessIncomingReply). Best-effort and nil-safe: when unset, opens/clicks // are still recorded and routed at the next step boundary by the scheduler. advancedService advanced.Service - topic string - group string + // opens is the per-event open log; geo resolves a source network to a + // location for opens and clicks. Both optional. + opens repository.EmailOpenRepository + geo *geo.Client + // retention is the operator-editable window the engagement prune obeys. + // Injected post-construction; nil keeps the compiled default. + retention RetentionSource + topic string + group string +} + +// RetentionSource is the operator-editable retention section, satisfied by +// instancesettings.Service. Read on every prune pass, so an edit in the admin +// panel takes effect on the next sweep rather than at the next restart. +type RetentionSource interface { + RetentionWindows(ctx context.Context) instancesettings.Retention +} + +// WireRetention attaches the instance settings the engagement prune reads its +// window from. +func (tc *TrackingConsumer) WireRetention(src RetentionSource) { tc.retention = src } + +// engagementRetentionDays is the window the next prune pass uses. +func (tc *TrackingConsumer) engagementRetentionDays(ctx context.Context) int { + if tc.retention == nil { + return config.EngagementEventRetentionDaysDefault + } + return tc.retention.RetentionWindows(ctx).EngagementEventDays } // NewTrackingConsumer wires the tracking consumer to the shared event bus. @@ -50,8 +87,12 @@ func NewTrackingConsumer( contactRepo repository.ContactRepository, streamingPublisher *pubsub.StreamingPublisher, dedupeRepo repository.TrackingDedupeRepository, + trackedLinks repository.TrackedLinkRepository, + linkClicks repository.LinkClickRepository, advancedService advanced.Service, evidence advanced.EvidenceRecorder, + opens repository.EmailOpenRepository, + geoClient *geo.Client, ) (*TrackingConsumer, error) { return &TrackingConsumer{ bus: bus, @@ -62,18 +103,104 @@ func NewTrackingConsumer( contactRepo: contactRepo, streamingPublisher: streamingPublisher, dedupeRepo: dedupeRepo, - advancedService: advancedService, - evidence: evidence, - topic: topic, - group: group, + trackedLinks: trackedLinks, + linkClicks: linkClicks, + afterBurstWindow: func(fn func()) { + // One second past the window covers event-time skew between the + // tracking service and the consumer. + time.AfterFunc(time.Duration(config.TrackingClickBurstSeconds+1)*time.Second, fn) + }, + advancedService: advancedService, + evidence: evidence, + opens: opens, + geo: geoClient, + topic: topic, + group: group, }, nil } // Start subscribes to the tracking topic and blocks until ctx is cancelled. +// It also runs the daily prune of the open and click logs. func (tc *TrackingConsumer) Start(ctx context.Context) error { + if tc.opens != nil || tc.linkClicks != nil { + go tc.pruneEngagementLogs(ctx) + } + if tc.linkClicks != nil { + go tc.sweepPendingClicks(ctx) + } return tc.bus.Subscribe(ctx, []string{tc.topic}, tc.group, tc.receive) } +// sweepPendingClicks fires the effects of human clicks whose timer never +// ran or never finished: the consumer restarted inside the burst window, a +// claim failed, or an attempt died mid-way and its lease expired. Runs at +// start and every minute until ctx ends. A click the timer completed is not +// offered, and one under a live lease is not offered twice. +func (tc *TrackingConsumer) sweepPendingClicks(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + before := time.Now().Add(-time.Duration(config.TrackingClickBurstSeconds+1) * time.Second) + pending, err := tc.linkClicks.ListPendingAnnouncements(ctx, before, 200) + if err != nil { + log.Warn().Err(err).Msg("could not list pending click announcements") + } + for i := range pending { + c := &pending[i] + task := &repository.CampaignTask{TaskID: c.TaskID, CampaignID: &c.CampaignID, ContactID: &c.ContactID, SequenceID: &c.SequenceID} + destination := c.Destination + event := events.TrackingEvent{ + EventType: events.EventTypeEmailClicked, + TaskID: c.TaskID.String(), + OriginalURL: &destination, + Timestamp: c.ClickedAt.Format(time.RFC3339Nano), + } + if c.TrackedLinkID != nil { + id := c.TrackedLinkID.String() + event.LinkID = &id + } + tc.finishHumanClick(task, event, c.ID, c.Label, c.Origin) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// pruneEngagementLogs deletes opens and clicks older than the retention +// window, at start and then daily. The progress-row summary stays, so +// nothing a count, filter or branch reads is affected. +func (tc *TrackingConsumer) pruneEngagementLogs(ctx context.Context) { + ticker := time.NewTicker(24 * time.Hour) + defer ticker.Stop() + for { + pctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + days := tc.engagementRetentionDays(pctx) + if tc.opens != nil { + if n, err := tc.opens.Cleanup(pctx, days); err != nil { + log.Warn().Err(err).Msg("open log prune failed") + } else if n > 0 { + log.Info().Int64("deleted", n).Msg("open log pruned") + } + } + if tc.linkClicks != nil { + if n, err := tc.linkClicks.Cleanup(pctx, days); err != nil { + log.Warn().Err(err).Msg("click log prune failed") + } else if n > 0 { + log.Info().Int64("deleted", n).Msg("click log pruned") + } + } + cancel() + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + // Close is a no-op: the event bus lifecycle is owned by the consumer main, // which subscribes both worker-events and tracking on the same bus. func (tc *TrackingConsumer) Close() {} @@ -88,7 +215,19 @@ func (tc *TrackingConsumer) receive(_ context.Context, msg eventbus.Message) err return tc.HandleTrackingEvent(context.Background(), &event) } -// HandleTrackingEvent processes a tracking event +// HandleTrackingEvent processes a tracking event. +// +// Opens and clicks are classified before they count. The edge already drops +// crawlers and security scanners it can name; here the ones it cannot are +// caught by what they do: a fetch with no browser, a fetch inside the +// machine window after dispatch (nobody reads that fast), and clicks on +// several links of one email within seconds (a gateway walking the message). +// A machine open is still recorded, labelled, because it proves delivery. A +// machine click is logged per link with its reason but never stamps the step +// as clicked, fires no automation, and sends no webhook: "clicked" keeps +// meaning a person. Because a burst is only recognisable from its second +// click, a human click's side effects wait out the burst window before +// firing, on the classification the click has by then. func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *events.TrackingEvent) error { // Parse and validate task ID taskID, err := uuid.Parse(event.TaskID) @@ -97,37 +236,15 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even return nil } - // Calculate URL hash for click event deduplication + // Click dedupe identity: the ticket, so two links sharing a destination + // are two clicks; the URL only for events from an older tracking build. urlHash := "" - if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil && *event.OriginalURL != "" { - urlHash = hashURL(*event.OriginalURL) - } - - // Classify opens: machine fetches (Apple MPP prefetch, UA-less clients) - // still count as delivery signal but are labeled, and must never fire - // open-triggered automations (a prefetch is not intent). - machineOpen := event.EventType == events.EventTypeEmailOpened && isMachineOpen(event.UserAgent) - - // Check for duplicate at consumer level (belt and suspenders with Rust service) - if tc.dedupeRepo != nil { - processed, err := tc.dedupeRepo.IsProcessed(ctx, taskID, event.EventType, urlHash) - if err != nil { - // Log but continue - allow processing on dedupe errors - log.Warn().Err(err).Str("task_id", event.TaskID).Msg("tracking dedupe check failed") - } else if processed { - // A HUMAN open after a machine-labeled one upgrades the label - // (MPP prefetched at delivery; the person actually read it later - // from another network). Quiet write only: the open was already - // counted once, so no automations and no re-publish. - if event.EventType == events.EventTypeEmailOpened && !machineOpen { - if campaignTask, terr := tc.taskRepo.GetCampaignTask(ctx, taskID); terr == nil && - campaignTask != nil && campaignTask.CampaignID != nil && - campaignTask.ContactID != nil && campaignTask.SequenceID != nil { - _ = tc.campaignProgressRepo.RecordEmailOpened(ctx, - *campaignTask.CampaignID, *campaignTask.ContactID, *campaignTask.SequenceID, false) - } - } - return nil + if event.EventType == events.EventTypeEmailClicked { + switch { + case event.LinkID != nil && *event.LinkID != "": + urlHash = hashURL("link:" + *event.LinkID) + case event.OriginalURL != nil && *event.OriginalURL != "": + urlHash = hashURL(*event.OriginalURL) } } @@ -137,16 +254,65 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even log.Warn().Err(err).Str("task_id", event.TaskID).Msg("failed to get campaign task for tracking event") return nil } + if campaignTask == nil || campaignTask.CampaignID == nil || campaignTask.ContactID == nil || campaignTask.SequenceID == nil { + // Task not found, not a campaign task, or missing its linkage: skip + return nil + } + campaignID, contactID, sequenceID := *campaignTask.CampaignID, *campaignTask.ContactID, *campaignTask.SequenceID - if campaignTask == nil || campaignTask.CampaignID == nil { - // Task not found or not a campaign task, skip + at := eventTime(event.Timestamp) + sentAt, err := tc.campaignProgressRepo.GetStepSentAt(ctx, campaignID, contactID, sequenceID) + if err != nil { + log.Warn().Err(err).Str("task_id", event.TaskID).Msg("failed to read step dispatch time; classifying by user agent only") + sentAt = nil + } + + // Classify. Machine opens (Apple MPP prefetch, UA-less clients, a fetch + // inside the machine window) still count as delivery signal but are + // labelled, and must never fire open-triggered automations. + var machine bool + var reason string + switch event.EventType { + case events.EventTypeEmailOpened: + machine, reason = classifyOpen(event.UserAgent, sentAt, at) + case events.EventTypeEmailClicked: + machine, reason = classifyClick(event.UserAgent, sentAt, at) + default: + // Unknown event type, skip return nil } - // Ensure we have contact_id and sequence_id - if campaignTask.ContactID == nil || campaignTask.SequenceID == nil { - // Missing required fields, skip - return nil + // What the request said about where it came from, for the logs and the + // live feed. The source network is resolved here and goes no further. + origin := tc.originOf(event) + + // Check for duplicate at consumer level (belt and suspenders with Rust service) + if tc.dedupeRepo != nil { + processed, err := tc.dedupeRepo.IsProcessed(ctx, taskID, event.EventType, urlHash) + if err != nil { + // Log but continue - allow processing on dedupe errors + log.Warn().Err(err).Str("task_id", event.TaskID).Msg("tracking dedupe check failed") + } else if processed { + // A HUMAN engagement after a machine-labelled one upgrades the + // label (a gateway scanned at delivery; the person acted later). + // Quiet write only: the event was already counted once, so no + // automations and no re-publish. + if event.EventType == events.EventTypeEmailOpened { + // Every open is logged, repeats and machines included: a + // second open from another device is worth seeing. + tc.logOpen(ctx, campaignTask, event, at, machine, reason, origin) + } + if machine { + return nil + } + switch event.EventType { + case events.EventTypeEmailOpened: + _ = tc.campaignProgressRepo.RecordEmailOpened(ctx, campaignID, contactID, sequenceID, false) + case events.EventTypeEmailClicked: + tc.upgradeClick(ctx, campaignTask, event, at, origin) + } + return nil + } } // Record the event, then fire any INSTANT open/click action chain for the @@ -155,33 +321,41 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even // to the matcher's eventKind. Firing happens AFTER the Record* write so the // matcher reads the just-stamped opened_at / clicked_at off the progress row. var instantKind string + var linkLabel string + var deferred bool switch event.EventType { case events.EventTypeEmailOpened: - err = tc.campaignProgressRepo.RecordEmailOpened(ctx, - *campaignTask.CampaignID, - *campaignTask.ContactID, - *campaignTask.SequenceID, - machineOpen) - if !machineOpen { + err = tc.campaignProgressRepo.RecordEmailOpened(ctx, campaignID, contactID, sequenceID, machine) + tc.logOpen(ctx, campaignTask, event, at, machine, reason, origin) + if !machine { instantKind = "open" // A human open proves the mailbox is live; a prefetch proves // only that a proxy fetched an image. if tc.evidence != nil { - tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "opened", campaignTask.SequenceID.String(), "") + tc.evidence.RecordEvidence(ctx, contactID, "opened", sequenceID.String(), "") } } case events.EventTypeEmailClicked: - err = tc.campaignProgressRepo.RecordEmailClicked(ctx, - *campaignTask.CampaignID, - *campaignTask.ContactID, - *campaignTask.SequenceID) - instantKind = "click" - if tc.evidence != nil { - tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "clicked", campaignTask.SequenceID.String(), "") + var click *repository.LinkClick + machine, reason, click, err = tc.recordClick(ctx, campaignTask, event, at, machine, reason, origin) + if click != nil { + linkLabel = click.Label + } + if err == nil && !machine { + // The stamp is stored state a burst can walk back; the effects + // cannot be recalled, so they wait for the window to close. + err = tc.campaignProgressRepo.RecordEmailClicked(ctx, campaignID, contactID, sequenceID) + if err == nil && click != nil && tc.afterBurstWindow != nil { + deferred = true + task, ev, clickID, label := campaignTask, *event, click.ID, linkLabel + tc.afterBurstWindow(func() { tc.finishHumanClick(task, ev, clickID, label, origin) }) + } else if err == nil { + instantKind = "click" + if tc.evidence != nil { + tc.evidence.RecordEvidence(ctx, contactID, "clicked", sequenceID.String(), "") + } + } } - default: - // Unknown event type, skip - return nil } if err != nil { @@ -195,11 +369,7 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even // opened/clicked branch at the next step boundary. Exactly-once per (step, // eventKind) is enforced inside FireInstantActions via ClaimInstantFire. if tc.advancedService != nil && instantKind != "" { - tc.advancedService.FireInstantActions(ctx, - *campaignTask.CampaignID, - *campaignTask.ContactID, - *campaignTask.SequenceID, - instantKind) + tc.advancedService.FireInstantActions(ctx, campaignID, contactID, sequenceID, instantKind) } // Mark as processed for deduplication @@ -209,15 +379,266 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even } } - // Publish to Pub/Sub for realtime updates - tc.publishTrackingEvent(ctx, campaignTask, *event, machineOpen) + if machine { + log.Debug().Str("task_id", event.TaskID).Str("event_type", string(event.EventType)).Str("reason", reason).Msg("tracking event classified as machine") + } + + // Publish to Pub/Sub for realtime updates (a deferred human click + // publishes once its verdict is final) + if !deferred { + tc.publishTrackingEvent(ctx, campaignTask, *event, machine, linkLabel, origin) + } return nil } +// finishHumanClick runs the effects of a click that looked human when it +// landed, once the burst window has passed: if a burst relabelled it in the +// meantime it is announced as automated and nothing else fires. The click +// row carries the pending flag, written before the event was marked +// processed, so a consumer restart inside the window hands the click to +// sweepPendingClicks instead of losing it, and a redelivery cannot fire it +// twice. When the claim cannot be made at all, nothing fires here and the +// sweep retries: an automation for a scanner's click is worse than a late +// one, and the step boundary still routes on the stored stamp. +func (tc *TrackingConsumer) finishHumanClick(task *repository.CampaignTask, event events.TrackingEvent, clickID uuid.UUID, label string, origin models.EngagementOrigin) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // The claim is the verdict and the lease in one write: a burst that + // relabelled the click in the meantime shows up as machine, and a click + // already announced, or under another attempt's live lease, is not + // claimed. The row is completed only after the effects ran, so a crash + // in between is retried by the sweep once the lease expires (at-least- + // once: the instant actions claim their own once-only fire, the rest + // may repeat after a crash). + var claimed, machine bool + var err error + for attempt := 0; attempt < 5; attempt++ { + if claimed, machine, err = tc.linkClicks.ClaimAnnounce(ctx, clickID); err == nil { + break + } + select { + case <-ctx.Done(): + case <-time.After(time.Duration(attempt+1) * 2 * time.Second): + } + } + if err != nil { + log.Error().Err(err).Str("click_id", clickID.String()).Msg("could not claim click announcement; left for the sweep") + return + } + if !claimed { + return + } + if machine { + tc.publishTrackingEvent(ctx, task, event, true, label, origin) + } else { + if tc.evidence != nil { + tc.evidence.RecordEvidence(ctx, *task.ContactID, "clicked", task.SequenceID.String(), "") + } + if tc.advancedService != nil { + tc.advancedService.FireInstantActions(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID, "click") + } + tc.publishTrackingEvent(ctx, task, event, false, label, origin) + } + if err := tc.linkClicks.CompleteAnnounce(ctx, clickID); err != nil { + log.Warn().Err(err).Str("click_id", clickID.String()).Msg("could not mark click announcement complete; the sweep may repeat it after the lease") + } +} + +// resolveLink names the clicked link: the minted ticket when the event +// carries one (destination and anchor text as stored at send time), else the +// URL the event reports. nil ticket id means the click log row stands alone. +func (tc *TrackingConsumer) resolveLink(ctx context.Context, event *events.TrackingEvent) (*uuid.UUID, string, string) { + var destination string + if event.OriginalURL != nil { + destination = *event.OriginalURL + } + if event.LinkID == nil || tc.trackedLinks == nil { + return nil, destination, "" + } + id, err := uuid.Parse(*event.LinkID) + if err != nil { + return nil, destination, "" + } + link, err := tc.trackedLinks.GetByID(ctx, id) + if err != nil || link == nil { + return nil, destination, "" + } + if link.Destination != "" { + destination = link.Destination + } + return &link.ID, destination, link.Label +} + +// recordClick logs the click per link and applies the burst rule: a click on +// a second link of the same email from the same source inside the burst +// window turns this click AND the earlier ones into machine clicks. When +// that leaves the step with no human click, the clicked stamp the first +// click already wrote is walked back. Returns the final classification and +// the logged row (nil when nothing could be logged). +func (tc *TrackingConsumer) recordClick(ctx context.Context, task *repository.CampaignTask, event *events.TrackingEvent, at time.Time, machine bool, reason string, origin models.EngagementOrigin) (bool, string, *repository.LinkClick, error) { + if tc.linkClicks == nil { + return machine, reason, nil, nil + } + linkID, destination, label := tc.resolveLink(ctx, event) + if destination == "" { + return machine, reason, nil, nil + } + ipHash := "" + if event.IPHash != nil { + ipHash = *event.IPHash + } + userAgent := "" + if event.UserAgent != nil { + userAgent = *event.UserAgent + } + + burstSince := at.Add(-time.Duration(config.TrackingClickBurstSeconds) * time.Second) + burst := false + if !machine && ipHash != "" { + n, err := tc.linkClicks.CountRecentOtherLinks(ctx, task.TaskID, ipHash, linkID, destination, burstSince) + if err != nil { + log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("burst check failed; treating click as a person's") + } else if n > 0 { + machine, reason, burst = true, repository.LinkClickReasonBurst, true + } + } + + click := &repository.LinkClick{ + TrackedLinkID: linkID, + TaskID: task.TaskID, + CampaignID: *task.CampaignID, + ContactID: *task.ContactID, + SequenceID: *task.SequenceID, + Destination: destination, + Label: label, + UserAgent: userAgent, + IPHash: ipHash, + Machine: machine, + MachineReason: reason, + ClickedAt: at, + Origin: origin, + // A person's click waits out the burst window; the flag is the + // durable record of that, written before the event is marked + // processed, so a restart or a redelivery fires it exactly once. + AnnouncePending: !machine && tc.afterBurstWindow != nil, + } + if err := tc.linkClicks.Insert(ctx, click); err != nil { + return machine, reason, click, err + } + + if burst { + if _, err := tc.linkClicks.MarkBurst(ctx, task.TaskID, ipHash, burstSince); err != nil { + log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("failed to relabel burst clicks") + } + if err := tc.campaignProgressRepo.UnrecordEmailClicked(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID); err != nil { + log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("failed to walk back the click stamp after a burst") + } + } + return machine, reason, click, nil +} + +// upgradeClick handles a human click on a link this email was already +// credited for: the step is stamped clicked if only machines had clicked so +// far, and the click is logged once so the timeline shows the person's. +func (tc *TrackingConsumer) upgradeClick(ctx context.Context, task *repository.CampaignTask, event *events.TrackingEvent, at time.Time, origin models.EngagementOrigin) { + _ = tc.campaignProgressRepo.RecordEmailClicked(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID) + if tc.linkClicks == nil { + return + } + linkID, destination, label := tc.resolveLink(ctx, event) + if destination == "" { + return + } + if seen, err := tc.linkClicks.HasHumanClickOn(ctx, task.TaskID, linkID, destination); err != nil || seen { + return + } + ipHash, userAgent := "", "" + if event.IPHash != nil { + ipHash = *event.IPHash + } + if event.UserAgent != nil { + userAgent = *event.UserAgent + } + _ = tc.linkClicks.Insert(ctx, &repository.LinkClick{ + TrackedLinkID: linkID, + TaskID: task.TaskID, + CampaignID: *task.CampaignID, + ContactID: *task.ContactID, + SequenceID: *task.SequenceID, + Destination: destination, + Label: label, + UserAgent: userAgent, + IPHash: ipHash, + ClickedAt: at, + Origin: origin, + }) +} + +// logOpen writes one row to the open log for this event, whatever it was +// classified as; the label travels with it. +func (tc *TrackingConsumer) logOpen(ctx context.Context, task *repository.CampaignTask, event *events.TrackingEvent, at time.Time, machine bool, reason string, origin models.EngagementOrigin) { + if tc.opens == nil { + return + } + open := &repository.EmailOpen{ + TaskID: task.TaskID, + CampaignID: *task.CampaignID, + ContactID: *task.ContactID, + SequenceID: *task.SequenceID, + OpenedAt: at, + Machine: machine, + MachineReason: reason, + Origin: origin, + } + if event.UserAgent != nil { + open.UserAgent = clipString(*event.UserAgent, 512) + } + if event.IPHash != nil { + open.IPHash = *event.IPHash + } + if err := tc.opens.Insert(ctx, open); err != nil { + log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("failed to log open") + } +} + +// originOf reads what the event says about its source: the user agent parsed +// to client, browser and device, and the source network resolved to a +// location. The network is used here and dropped. +func (tc *TrackingConsumer) originOf(event *events.TrackingEvent) models.EngagementOrigin { + var o models.EngagementOrigin + if event.UserAgent != nil && strings.TrimSpace(*event.UserAgent) != "" { + ua := useragent.Parse(*event.UserAgent) + o.OS, o.Browser, o.BrowserVersion = ua.OS, ua.Name, ua.Version + o.DeviceType = deviceType(ua) + o.Client = clientName(*event.UserAgent) + } + if event.ClientIP != nil && tc.geo != nil { + if addr, err := netip.ParseAddr(strings.TrimSpace(*event.ClientIP)); err == nil && !addr.IsPrivate() && !addr.IsLoopback() { + if info, err := tc.geo.Lookup(addr); err == nil && info != nil { + o.CountryCode = info.CountryCode + o.Region = info.Region + if info.City != "Unknown" { + o.City = info.City + } + } + } + } + return o +} + +func clipString(s string, n int) string { + s = strings.TrimSpace(s) + if len(s) > n { + return s[:n] + } + return s +} + // publishTrackingEvent publishes the tracking event to Pub/Sub for realtime UI // updates AND fans an opt-in firehose webhook (campaign.email_opened/clicked). -func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repository.CampaignTask, event events.TrackingEvent, machine bool) { +func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repository.CampaignTask, event events.TrackingEvent, machine bool, linkLabel string, origin models.EngagementOrigin) { // Get campaign to find user ID + org campaign, err := tc.campaignRepo.GetByID(ctx, *task.CampaignID) if err != nil || campaign == nil { @@ -233,14 +654,14 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo } } - // Fan an opt-in firehose webhook for the open/click (org-scoped). Human opens - // only — a machine prefetch is not engagement intent — clicks always. - if tc.advancedService != nil && campaign.OrganizationID != nil { + // Fan an opt-in firehose webhook for the open/click (org-scoped). People + // only: a prefetch or a gateway walking the links is not engagement. + if tc.advancedService != nil && campaign.OrganizationID != nil && !machine { var whType models.WebhookEventType - switch { - case event.EventType == events.EventTypeEmailOpened && !machine: + switch event.EventType { + case events.EventTypeEmailOpened: whType = models.WebhookEventCampaignEmailOpened - case event.EventType == events.EventTypeEmailClicked: + case events.EventTypeEmailClicked: whType = models.WebhookEventCampaignEmailClicked } if whType != "" { @@ -252,6 +673,9 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo } if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil { data["url"] = *event.OriginalURL + if linkLabel != "" { + data["link_label"] = linkLabel + } } tc.advancedService.EmitCampaignEvent(ctx, *campaign.OrganizationID, whType, data) } @@ -290,10 +714,16 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo ContactEmail: contactEmail, SequenceID: task.SequenceID.String(), Machine: machine, + OccurredAt: eventTime(event.Timestamp), + Client: origin.Client, + DeviceType: origin.DeviceType, + CountryCode: origin.CountryCode, + City: origin.City, } if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil { trackingPayload.OriginalURL = *event.OriginalURL + trackingPayload.LinkLabel = linkLabel } tc.streamingPublisher.PublishTrackingEvent(ctx, trackingPayload) diff --git a/internal/app/consumer/open_class.go b/internal/app/consumer/open_class.go index 43f492b6..e451aeea 100644 --- a/internal/app/consumer/open_class.go +++ b/internal/app/consumer/open_class.go @@ -1,6 +1,13 @@ package jobs -import "strings" +import ( + "strings" + "time" + + "github.com/mileusna/useragent" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/repository" +) // isMachineOpen reports whether an open event came from an automated fetcher // rather than a human-rendered view. The edge already filters crawlers and @@ -25,3 +32,97 @@ func isMachineOpen(userAgent *string) bool { } return strings.HasSuffix(ua, "(khtml, like gecko)") } + +// isInstant reports whether an engagement arrived so soon after the step was +// dispatched that no person could have read the email yet. Security +// gateways (Safe Links, Proofpoint, Mimecast) open the pixel and walk every +// link at delivery time with an ordinary browser UA, which is exactly what +// the UA rules cannot see. An unknown dispatch time never counts as instant. +func isInstant(sentAt *time.Time, at time.Time) bool { + if sentAt == nil { + return false + } + return at.Sub(*sentAt) < time.Duration(config.TrackingMachineWindowSeconds)*time.Second +} + +// classifyClick applies the per-event click rules (the burst rule needs the +// click log and lives in the consumer). It returns whether the click is +// automated and the reason recorded with it; an empty reason is a person. +func classifyClick(userAgent *string, sentAt *time.Time, at time.Time) (bool, string) { + if userAgent == nil || strings.TrimSpace(*userAgent) == "" { + return true, repository.LinkClickReasonPrefetch + } + if isInstant(sentAt, at) { + return true, repository.LinkClickReasonInstant + } + return false, "" +} + +// eventTime is when the tracking service saw the event, falling back to now +// when the stamp is missing or unreadable, so consumer lag never turns a +// delivery-time scan into a plausible human open. +func eventTime(stamp string) time.Time { + if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil { + return t + } + if t, err := time.Parse(time.RFC3339, stamp); err == nil { + return t + } + return time.Now() +} + +// classifyOpen applies the per-event open rules and names the one that +// caught it: prefetch for a mail proxy or a fetch with no browser, instant +// for a fetch inside the machine window after dispatch. An empty reason is +// a person. +func classifyOpen(userAgent *string, sentAt *time.Time, at time.Time) (bool, string) { + if isMachineOpen(userAgent) { + return true, repository.EmailOpenReasonPrefetch + } + if isInstant(sentAt, at) { + return true, repository.EmailOpenReasonInstant + } + return false, "" +} + +// clientName names the mail client or image proxy behind a user agent when +// it says so; empty for a plain browser, which the parsed fields describe. +func clientName(userAgent string) string { + ua := strings.ToLower(strings.TrimSpace(userAgent)) + switch { + case ua == "": + return "" + case strings.Contains(ua, "googleimageproxy"): + return "Gmail" + case strings.Contains(ua, "yahoomailproxy"), strings.Contains(ua, "yahoo mail"): + return "Yahoo Mail" + case strings.Contains(ua, "outlook"), strings.Contains(ua, "microsoft office"): + return "Outlook" + case strings.Contains(ua, "thunderbird"): + return "Thunderbird" + case strings.Contains(ua, "superhuman"): + return "Superhuman" + case strings.Contains(ua, "protonmail"), strings.Contains(ua, "proton mail"): + return "Proton Mail" + case strings.Contains(ua, "hey.com"): + return "HEY" + case strings.HasSuffix(ua, "(khtml, like gecko)"): + // Apple Mail Privacy Protection's prefetch fingerprint. + return "Apple Mail" + } + return "" +} + +// deviceType folds the parser's flags into desktop, mobile, tablet or unknown. +func deviceType(ua useragent.UserAgent) string { + switch { + case ua.Tablet: + return "tablet" + case ua.Mobile: + return "mobile" + case ua.Desktop: + return "desktop" + default: + return "unknown" + } +} diff --git a/internal/app/consumer/open_class_test.go b/internal/app/consumer/open_class_test.go new file mode 100644 index 00000000..ef9333b2 --- /dev/null +++ b/internal/app/consumer/open_class_test.go @@ -0,0 +1,48 @@ +package jobs + +import ( + "testing" + "time" + + "github.com/warmbly/warmbly/internal/repository" +) + +func strp(s string) *string { return &s } + +func TestIsInstantUsesTheDispatchClock(t *testing.T) { + sent := time.Now() + if !isInstant(&sent, sent.Add(3*time.Second)) { + t.Fatal("three seconds after dispatch is a machine") + } + if isInstant(&sent, sent.Add(45*time.Second)) { + t.Fatal("forty-five seconds after dispatch can be a person") + } + if isInstant(nil, sent) { + t.Fatal("an unknown dispatch time must never count as instant") + } +} + +func TestClassifyClick(t *testing.T) { + sent := time.Now() + chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") + + if m, r := classifyClick(nil, &sent, sent.Add(time.Minute)); !m || r != repository.LinkClickReasonPrefetch { + t.Fatalf("no user agent = prefetch, got %v %q", m, r) + } + if m, r := classifyClick(chrome, &sent, sent.Add(2*time.Second)); !m || r != repository.LinkClickReasonInstant { + t.Fatalf("a browser UA two seconds after dispatch = instant, got %v %q", m, r) + } + if m, r := classifyClick(chrome, &sent, sent.Add(time.Minute)); m || r != "" { + t.Fatalf("a browser a minute later is a person, got %v %q", m, r) + } +} + +func TestEventTimeFallsBackToNow(t *testing.T) { + stamp := "2026-09-03T10:00:00Z" + if got := eventTime(stamp); !got.Equal(time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC)) { + t.Fatalf("unexpected parse: %v", got) + } + if d := time.Since(eventTime("garbage")); d < 0 || d > time.Minute { + t.Fatalf("unreadable stamp should fall back to now, got %v ago", d) + } +} diff --git a/internal/app/consumer/service.go b/internal/app/consumer/service.go index 6e1d4269..4cf6252b 100644 --- a/internal/app/consumer/service.go +++ b/internal/app/consumer/service.go @@ -74,6 +74,11 @@ type JobsService struct { // dead-worker job moves or strands their mailboxes. Nil disables it. Notifier OrgNotifier + // OpsNotifier raises instance-wide operator alerts, which are a different + // audience from Notifier: the operator hears about the fleet, the tenant + // hears about their mailboxes. Nil disables it. + OpsNotifier OperatorNotifier + // Send-outcome handling (EMAIL_SENT / EMAIL_FAILED from workers). The task // and campaign progress the control plane stamped at hand-off are walked // back here when a worker reports it could not send. Nil TaskRepo disables diff --git a/internal/app/contact/events.go b/internal/app/contact/events.go new file mode 100644 index 00000000..ffa309c6 --- /dev/null +++ b/internal/app/contact/events.go @@ -0,0 +1,85 @@ +package contact + +import ( + "context" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +type skipCreatedEventsKey struct{} + +// contactCreatedEventMaxBatch is the largest single write that still raises +// contact.created per row. A request adding more contacts at once is a bulk +// arrival like a file import, and stays silent for the same reason. +const contactCreatedEventMaxBatch = 100 + +// WithoutCreatedEvents marks a context whose contact writes must not fire +// contact.created. Bulk arrivals (file import, sheet sync) use it so one +// upload cannot flood an organization's automations and webhooks. +func WithoutCreatedEvents(ctx context.Context) context.Context { + return context.WithValue(ctx, skipCreatedEventsKey{}, true) +} + +func createdEventsSuppressed(ctx context.Context) bool { + v, _ := ctx.Value(skipCreatedEventsKey{}).(bool) + return v +} + +// emitCreated fires contact.created for every row the upsert inserted. Rows +// that matched an existing contact stay silent: their first touch already +// happened, and so does a batch past contactCreatedEventMaxBatch. in and out +// are index-aligned, as the repository returns them. +func (s *contactService) emitCreated(ctx context.Context, orgID uuid.UUID, in []models.AddContact, out []models.Contact) { + if s.webhooks == nil || createdEventsSuppressed(ctx) || len(in) > contactCreatedEventMaxBatch { + return + } + for i := range out { + if !out[i].IsNew { + continue + } + var src models.AddContact + if i < len(in) { + src = in[i] + } + _, _ = s.webhooks.Dispatch(ctx, orgID, models.WebhookEventContactCreated, ContactCreatedPayload(out[i], src)) + } +} + +// ContactCreatedPayload is the contact.created event body: the contact's +// fields plus its first-touch source, flat so automation conditions and +// templates read them as {{.contact_email}}, {{.first_name}}, {{.source}}. +func ContactCreatedPayload(c models.Contact, src models.AddContact) map[string]any { + source := src.Source + if source == "" { + source = models.ContactSourceUnknown + } + custom := c.CustomFields + if custom == nil { + custom = map[string]string{} + } + campaignIDs := make([]string, 0, len(c.Campaigns)) + for _, cm := range c.Campaigns { + campaignIDs = append(campaignIDs, cm.ID) + } + categoryIDs := make([]string, 0, len(c.Categories)) + for _, cat := range c.Categories { + categoryIDs = append(categoryIDs, cat.ID.String()) + } + return map[string]any{ + "contact_id": c.ID.String(), + "contact_email": c.Email, + "first_name": c.FirstName, + "last_name": c.LastName, + "company": c.Company, + "phone": c.Phone, + "subscribed": c.Subscribed, + "custom_fields": custom, + "source": string(source), + "source_detail": src.SourceDetail, + "campaign_ids": campaignIDs, + "category_ids": categoryIDs, + "created_at": c.CreatedAt, + } +} diff --git a/internal/app/contact/events_test.go b/internal/app/contact/events_test.go new file mode 100644 index 00000000..9e547de9 --- /dev/null +++ b/internal/app/contact/events_test.go @@ -0,0 +1,77 @@ +package contact + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +type recordingDispatcher struct { + events []models.WebhookEventType + data []map[string]any +} + +func (r *recordingDispatcher) Dispatch(_ context.Context, _ uuid.UUID, t models.WebhookEventType, data any) (uuid.UUID, error) { + r.events = append(r.events, t) + m, _ := data.(map[string]any) + r.data = append(r.data, m) + return uuid.New(), nil +} + +func TestEmitCreated_OnlyNewRowsAndNotWhenSuppressed(t *testing.T) { + rec := &recordingDispatcher{} + svc := &contactService{webhooks: rec} + in := []models.AddContact{ + {Email: "new@example.com", Source: models.ContactSourceForm, SourceDetail: "Demo request"}, + {Email: "old@example.com", Source: models.ContactSourceForm}, + } + out := []models.Contact{ + {ID: uuid.New(), Email: "new@example.com", IsNew: true, Campaigns: []models.MiniCampaign{{ID: "c1", Name: "Q3"}}}, + {ID: uuid.New(), Email: "old@example.com", IsNew: false}, + } + svc.emitCreated(context.Background(), uuid.New(), in, out) + if len(rec.events) != 1 || rec.events[0] != models.WebhookEventContactCreated { + t.Fatalf("events = %v", rec.events) + } + got := rec.data[0] + if got["contact_email"] != "new@example.com" || got["source"] != "form" || got["source_detail"] != "Demo request" { + t.Fatalf("payload = %v", got) + } + if ids, _ := got["campaign_ids"].([]string); len(ids) != 1 || ids[0] != "c1" { + t.Fatalf("campaign_ids = %v", got["campaign_ids"]) + } + + rec.events = nil + svc.emitCreated(WithoutCreatedEvents(context.Background()), uuid.New(), in, out) + if len(rec.events) != 0 { + t.Fatalf("suppressed context still emitted %v", rec.events) + } + + quiet := &contactService{} + quiet.emitCreated(context.Background(), uuid.New(), in, out) // nil dispatcher is a no-op + + bigIn := make([]models.AddContact, contactCreatedEventMaxBatch+1) + bigOut := make([]models.Contact, len(bigIn)) + for i := range bigIn { + bigIn[i] = models.AddContact{Email: "x@example.com"} + bigOut[i] = models.Contact{ID: uuid.New(), IsNew: true} + } + rec.events = nil + svc.emitCreated(context.Background(), uuid.New(), bigIn, bigOut) + if len(rec.events) != 0 { + t.Fatalf("a batch past the cap must stay silent, emitted %d", len(rec.events)) + } +} + +func TestContactCreatedPayload_DefaultsUnknownSourceAndEmptyMaps(t *testing.T) { + p := ContactCreatedPayload(models.Contact{ID: uuid.New(), Email: "x@y.z"}, models.AddContact{}) + if p["source"] != "unknown" { + t.Fatalf("source = %v", p["source"]) + } + if m, ok := p["custom_fields"].(map[string]string); !ok || m == nil { + t.Fatalf("custom_fields = %v", p["custom_fields"]) + } +} diff --git a/internal/app/contact/handler.go b/internal/app/contact/handler.go index 053967f3..77e9af32 100644 --- a/internal/app/contact/handler.go +++ b/internal/app/contact/handler.go @@ -76,6 +76,7 @@ func (s *contactService) Add(ctx context.Context, userID string, orgID uuid.UUID } s.wakeCampaigns(ctx, orgID, attached) s.syncSegmentCampaigns(ctx, orgID) + s.emitCreated(ctx, orgID, contacts, created) return created, nil } @@ -218,6 +219,6 @@ func (s *contactService) ListSentEmails(ctx context.Context, userID, contactID u return s.contactRepository.ListSentEmails(ctx, userID, contactID, limit, beforeSentAt, beforeTaskID) } -func (s *contactService) ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, before *time.Time) (*models.ContactTimelineResult, *errx.Error) { - return s.contactRepository.ListTimeline(ctx, userID, orgID, contactID, limit, before) +func (s *contactService) ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, cursor *models.ContactTimelineKey) (*models.ContactTimelineResult, *errx.Error) { + return s.contactRepository.ListTimeline(ctx, userID, orgID, contactID, limit, cursor) } diff --git a/internal/app/contact/import.go b/internal/app/contact/import.go index 1daa2a5a..436ee472 100644 --- a/internal/app/contact/import.go +++ b/internal/app/contact/import.go @@ -445,6 +445,11 @@ func (s *contactService) ImportCommit( return nil, xerr } + // A file or sheet is a bulk arrival, not fifty thousand "new contact" + // moments: the per-contact event stays quiet so automations and webhooks + // are not flooded by a single import. + ctx = WithoutCreatedEvents(ctx) + // Insert in chunks so a 50k row import doesn't blow up a single // pgx batch. 500 lines up with the Search page size. for start := 0; start < len(toInsert); start += 500 { diff --git a/internal/app/contact/service.go b/internal/app/contact/service.go index 6cc28ba8..e634db57 100644 --- a/internal/app/contact/service.go +++ b/internal/app/contact/service.go @@ -67,7 +67,7 @@ type ContactService interface { // ListTimeline returns a merged, reverse-chronological feed of all // engagement + CRM events for the contact. - ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, before *time.Time) (*models.ContactTimelineResult, *errx.Error) + ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, cursor *models.ContactTimelineKey) (*models.ContactTimelineResult, *errx.Error) // SetCampaignWaker wires the campaign service so attaching a lead to a // running campaign wakes that campaign's parked send chain. Optional: with @@ -109,6 +109,17 @@ type SegmentAware interface { WireSegments(linker SegmentLinker, syncer SegmentCampaignSyncer) } +// WebhookDispatcher delivers contact.created to customer webhooks and +// automations. Satisfied structurally by webhook.Service. +type WebhookDispatcher interface { + Dispatch(ctx context.Context, orgID uuid.UUID, eventType models.WebhookEventType, data any) (uuid.UUID, error) +} + +// WebhookAware is the optional capability the caller uses to attach it. +type WebhookAware interface { + WireWebhooks(w WebhookDispatcher) +} + type contactService struct { contactRepository repository.ContactRepository subRepo repository.SubscriptionRepository @@ -123,6 +134,8 @@ type contactService struct { orgRisk orgrisk.Service // explainer builds the verification "why" for the contact drawer. explainer VerificationExplainer + // webhooks fans contact.created out; nil-safe (no events). + webhooks WebhookDispatcher } // VerificationAware is implemented by the contact service so main can hand @@ -142,6 +155,9 @@ func (s *contactService) WireVerification(e VerificationExplainer) { s.explainer // WireOrgRisk attaches the organization risk posture. func (s *contactService) WireOrgRisk(r orgrisk.Service) { s.orgRisk = r } +// WireWebhooks attaches the event dispatcher behind contact.created. +func (s *contactService) WireWebhooks(w WebhookDispatcher) { s.webhooks = w } + // OrgRiskAware is the optional capability the caller uses to attach it. type OrgRiskAware interface { WireOrgRisk(r orgrisk.Service) diff --git a/internal/app/dailythrottle/service.go b/internal/app/dailythrottle/service.go index 55db5627..dda10650 100644 --- a/internal/app/dailythrottle/service.go +++ b/internal/app/dailythrottle/service.go @@ -29,7 +29,6 @@ type Resource string const ( ResourceCampaign Resource = "campaign" - ResourceMailbox Resource = "mailbox" ResourceOrg Resource = "org" ResourceScheduledSend Resource = "scheduled_send" ) diff --git a/internal/app/email/broker.go b/internal/app/email/broker.go index 76a9a9fb..2f294319 100644 --- a/internal/app/email/broker.go +++ b/internal/app/email/broker.go @@ -27,7 +27,8 @@ func (s *emailService) OAuthConnectWithCode(ctx context.Context, userID string, if code = strings.TrimSpace(code); code == "" { return nil, errx.ErrEmailOnboardCode } - if xerr := s.guardInboxLimit(ctx, orgID); xerr != nil { + allowance, xerr := s.guardInboxLimit(ctx, orgID) + if xerr != nil { return nil, xerr } cfg, xerr := s.oauthConfigFor(provider) @@ -51,11 +52,9 @@ func (s *emailService) OAuthConnectWithCode(ctx context.Context, userID string, if name == "" { name = deriveNameFromEmail(owner.Email) } - if xerr := s.guardMailboxThrottle(ctx, orgID); xerr != nil { - return nil, xerr - } acc, xerr := s.emailRepository.NewOauthAccount(ctx, userID, models.NewOauthAccount{ OrganizationID: orgID, + Allowance: allowance, Provider: provider, Name: name, Email: owner.Email, diff --git a/internal/app/email/bulk.go b/internal/app/email/bulk.go new file mode 100644 index 00000000..9c8c2006 --- /dev/null +++ b/internal/app/email/bulk.go @@ -0,0 +1,156 @@ +package email + +import ( + "context" + "errors" + "strings" + "sync" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// OnboardSMTPIMAPBulk connects up to config.MailboxBulkBatchMax mailboxes. +// +// The allowance is resolved once up front: rows past what fits are answered +// with mailbox_allowance_reached without dialling anything, and the rows that +// do fit are validated concurrently. Each of those still runs the single +// connect path, so a row is never connected twice and every side effect of a +// single connect (worker load, warmup pool, webhook) happens per mailbox. +func (s *emailService) OnboardSMTPIMAPBulk(ctx context.Context, userID string, orgID *uuid.UUID, rows []models.NewSMTPIMAPAccount) *models.MailboxBulkResult { + res := &models.MailboxBulkResult{Data: make([]models.MailboxBulkRow, len(rows))} + res.Summary.Total = len(rows) + if len(rows) == 0 { + return res + } + + fail := func(i int, xerr *errx.Error) { + res.Data[i] = models.MailboxBulkRow{ + Row: i, Email: rows[i].Email, Status: models.MailboxBulkFailed, + Code: bulkCode(xerr), Message: xerr.Message, + } + } + + // A batch-wide refusal (no org, allowance unreadable) fails every row the + // same way rather than pretending some rows were tried. + remaining := len(rows) + var allowance *models.MailboxAllowance + if orgID == nil { + for i := range rows { + fail(i, errx.ErrNoOrganization) + } + res.Summary.Failed = len(rows) + return res + } + if s.allowance != nil { + a, xerr := s.allowance.MailboxAllowance(ctx, *orgID) + if xerr != nil { + for i := range rows { + fail(i, xerr) + } + res.Summary.Failed = len(rows) + return res + } + allowance = a + if a.Remaining != nil && *a.Remaining < remaining { + remaining = *a.Remaining + } + } + + // Duplicates inside the file and mailboxes that are already connected are + // settled before anything competes for the allowance, so a re-uploaded + // file never spends a slot on a row that would create nothing. + seen := make(map[string]bool, len(rows)) + eligible := make([]int, 0, len(rows)) + for i := range rows { + key := strings.ToLower(strings.TrimSpace(rows[i].Email)) + if key != "" && seen[key] { + fail(i, errx.NewWithIdentifier(errx.BadRequest, "duplicate_row", "This address appears earlier in the same file.")) + continue + } + seen[key] = true + if exists, xerr := s.emailRepository.ExistsForUser(ctx, userID, strings.TrimSpace(rows[i].Email)); xerr != nil { + fail(i, xerr) + continue + } else if exists { + res.Data[i] = models.MailboxBulkRow{ + Row: i, Email: rows[i].Email, Status: models.MailboxBulkSkipped, + Code: "already_connected", Message: errx.ErrEmailOnboardAlreadyExists.Message, + } + continue + } + if len(eligible) >= remaining { + used, limit := 0, 0 + paid := true + if allowance != nil { + used, paid = allowance.Used, allowance.Paid + if allowance.Allowance != nil { + limit = *allowance.Allowance + } + } + fail(i, errx.MailboxAllowanceReached(used, limit, paid)) + continue + } + eligible = append(eligible, i) + } + + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, config.MailboxBulkConcurrency) + for _, i := range eligible { + wg.Add(1) + go func(i int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + row := rows[i] + acc, xerr := s.OnboardSMTPIMAP(ctx, userID, orgID, &row) + mu.Lock() + defer mu.Unlock() + switch { + case xerr == nil: + res.Data[i] = models.MailboxBulkRow{Row: i, Email: acc.Email, Status: models.MailboxBulkConnected, ID: &acc.ID} + case errors.Is(xerr, errx.ErrEmailOnboardAlreadyExists): + res.Data[i] = models.MailboxBulkRow{ + Row: i, Email: row.Email, Status: models.MailboxBulkSkipped, + Code: "already_connected", Message: xerr.Message, + } + default: + res.Data[i] = models.MailboxBulkRow{ + Row: i, Email: row.Email, Status: models.MailboxBulkFailed, + Code: bulkCode(xerr), Message: xerr.Message, + } + } + }(i) + } + wg.Wait() + + for _, r := range res.Data { + switch r.Status { + case models.MailboxBulkConnected: + res.Summary.Connected++ + case models.MailboxBulkSkipped: + res.Summary.Skipped++ + default: + res.Summary.Failed++ + } + } + if s.allowance != nil { + if a, xerr := s.allowance.MailboxAllowance(ctx, *orgID); xerr == nil { + res.Allowance = a + } + } + return res +} + +// bulkCode is the stable per-row code: the error's own identifier when it +// has one, otherwise the generic one for its HTTP class. +func bulkCode(xerr *errx.Error) string { + if xerr == nil { + return "" + } + return xerr.ResponseCode() +} diff --git a/internal/app/email/onboarding.go b/internal/app/email/onboarding.go index 36e37fe9..54b11aeb 100644 --- a/internal/app/email/onboarding.go +++ b/internal/app/email/onboarding.go @@ -11,8 +11,6 @@ import ( "github.com/getsentry/sentry-go" "github.com/google/uuid" - "github.com/warmbly/warmbly/internal/app/dailythrottle" - "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/models" @@ -30,7 +28,7 @@ func (s *emailService) OAuthStart(ctx context.Context, userID string, orgID *uui // Refuse early so we don't waste an OAuth round-trip on a request // that the inbox-limit guard would reject after callback. - if xerr := s.guardInboxLimit(ctx, orgID); xerr != nil { + if _, xerr := s.guardInboxLimit(ctx, orgID); xerr != nil { return nil, xerr } @@ -57,47 +55,42 @@ func (s *emailService) OAuthStart(ctx context.Context, userID string, orgID *uui return &models.EmailOnboardingStartResponse{URL: url, State: state}, nil } -// guardMailboxThrottle bounds new-mailbox connection rate per org per -// day so abuse paths (or accidents) can't connect 200 mailboxes in -// one tab session. The budget is keyed by org, so a request without -// one is refused rather than exempted. The check fires only on the -// actual create paths, not on OAuthStart, so retrying a failed flow -// doesn't consume the day's budget. -func (s *emailService) guardMailboxThrottle(ctx context.Context, orgID *uuid.UUID) *errx.Error { +// guardInboxLimit refuses a connect that would take the workspace past its +// mailbox allowance (fair use for paid plans, FreeWorkspaceMailboxLimit for +// free ones, unlimited without billing) and returns the resolved allowance so +// the insert can enforce it again under the organization's lock. The +// allowance is counted per org, so no org means it cannot be applied and the +// connect is refused. Without an allowance source wired, the feature gate's +// free-or-paid split stands in and the insert is not re-checked. +func (s *emailService) guardInboxLimit(ctx context.Context, orgID *uuid.UUID) (*models.MailboxAllowance, *errx.Error) { if orgID == nil { - return errx.ErrNoOrganization + return nil, errx.ErrNoOrganization } - if s.throttle == nil { - return nil - } - return s.throttle.CheckAndIncrement(ctx, *orgID, dailythrottle.ResourceMailbox, config.DailyThrottleNewMailboxes) -} - -// guardInboxLimit enforces the per-org inbox cap for free-trial users. -// Returns nil (allowed) for paid orgs and for trial orgs under the cap. -// Trial orgs that have already connected one inbox get -// ErrEmailOnboardInboxLimit; orgs without an active subscription or trial -// get ErrEmailOnboardTrialExpired. The cap is counted per org, so no org -// means the cap cannot be applied and the connect is refused. -func (s *emailService) guardInboxLimit(ctx context.Context, orgID *uuid.UUID) *errx.Error { - if orgID == nil { - return errx.ErrNoOrganization + if s.allowance != nil { + a, xerr := s.allowance.MailboxAllowance(ctx, *orgID) + if xerr != nil { + return nil, xerr + } + if a.CanAdd(1) { + return a, nil + } + return nil, errx.MailboxAllowanceReached(a.Used, *a.Allowance, a.Paid) } if s.featureGate == nil { - return nil + return nil, nil } count, xerr := s.emailRepository.CountForOrganization(ctx, *orgID) if xerr != nil { - return xerr + return nil, xerr } allowed, xerr := s.featureGate.CanAddInbox(ctx, *orgID, count) if xerr != nil { - return xerr + return nil, xerr } if allowed { - return nil + return nil, nil } - return errx.ErrEmailOnboardInboxLimit + return nil, errx.MailboxAllowanceReached(count, models.FreeWorkspaceMailboxLimit, false) } // OAuthFinish validates the state, exchanges the code for tokens, fetches the @@ -120,10 +113,13 @@ func (s *emailService) OAuthFinish(ctx context.Context, userID, code, state stri } // A reauth adds no mailbox, so an org over its inbox cap can still fix one. + var allowance *models.MailboxAllowance if sess.EmailAccountID == nil { - if xerr := s.guardInboxLimit(ctx, sess.OrganizationID); xerr != nil { + a, xerr := s.guardInboxLimit(ctx, sess.OrganizationID) + if xerr != nil { return nil, false, xerr } + allowance = a } provider := models.InboxProvider(sess.Provider) @@ -158,12 +154,9 @@ func (s *emailService) OAuthFinish(ctx context.Context, userID, code, state stri name = deriveNameFromEmail(owner.Email) } - if xerr := s.guardMailboxThrottle(ctx, sess.OrganizationID); xerr != nil { - return nil, false, xerr - } - acc, xerr := s.emailRepository.NewOauthAccount(ctx, userID, models.NewOauthAccount{ OrganizationID: sess.OrganizationID, + Allowance: allowance, Provider: provider, Name: name, Email: owner.Email, @@ -189,7 +182,8 @@ func (s *emailService) OnboardSMTPIMAP(ctx context.Context, userID string, orgID return nil, xerr } - if xerr := s.guardInboxLimit(ctx, orgID); xerr != nil { + allowance, xerr := s.guardInboxLimit(ctx, orgID) + if xerr != nil { return nil, xerr } @@ -222,11 +216,8 @@ func (s *emailService) OnboardSMTPIMAP(ctx context.Context, userID string, orgID return nil, xerr } - if xerr := s.guardMailboxThrottle(ctx, orgID); xerr != nil { - return nil, xerr - } - data.OrganizationID = orgID + data.Allowance = allowance acc, xerr := s.emailRepository.NewSMTPIMAPAccount(ctx, userID, *data) if xerr != nil { diff --git a/internal/app/email/service.go b/internal/app/email/service.go index bd880b47..34b7b5f0 100644 --- a/internal/app/email/service.go +++ b/internal/app/email/service.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/cipher" - "github.com/warmbly/warmbly/internal/app/dailythrottle" "github.com/warmbly/warmbly/internal/app/feature" warmupapp "github.com/warmbly/warmbly/internal/app/warmup" "github.com/warmbly/warmbly/internal/app/webhook" @@ -65,6 +64,10 @@ type EmailService interface { OAuthStart(ctx context.Context, userID string, orgID *uuid.UUID, provider models.InboxProvider) (*models.EmailOnboardingStartResponse, *errx.Error) OAuthFinish(ctx context.Context, userID, code, state string) (*models.Email, bool, *errx.Error) OnboardSMTPIMAP(ctx context.Context, userID string, orgID *uuid.UUID, data *models.NewSMTPIMAPAccount) (*models.Email, *errx.Error) + // OnboardSMTPIMAPBulk connects many SMTP/IMAP mailboxes in one call and + // answers per row, so one bad password never fails the file. Rows past the + // workspace's allowance are refused before any credential is dialled. + OnboardSMTPIMAPBulk(ctx context.Context, userID string, orgID *uuid.UUID, rows []models.NewSMTPIMAPAccount) *models.MailboxBulkResult // OAuthReauth starts an OAuth round trip that renews the tokens of an // existing Gmail/Outlook mailbox after the provider invalidated them. OAuthReauth(ctx context.Context, userID string, orgID *uuid.UUID, accountID uuid.UUID) (*models.EmailOnboardingStartResponse, *errx.Error) @@ -75,7 +78,9 @@ type EmailService interface { // Optional: wire in the webhook dispatcher after construction. Once // set, account-lifecycle events fan out to customer webhook endpoints. WireWebhooks(w webhook.Service) - WireThrottle(t dailythrottle.Service) + // WireMailboxAllowance attaches the allowance resolver every connect path + // checks. Without it the feature gate's free-or-paid split stands in. + WireMailboxAllowance(src MailboxAllowanceSource) // WireGraphDelta attaches the Graph delta-cursor repository so the worker // reconciler can seed a mailbox's saved cursors when loading it. WireGraphDelta(repo repository.EmailGraphDeltaRepository) @@ -124,7 +129,7 @@ type emailService struct { r *cache.Cache oauthInbox *config.Oauth2Inbox workerAssignment worker.WorkerAssignmentService - throttle dailythrottle.Service + allowance MailboxAllowanceSource graphDelta repository.EmailGraphDeltaRepository historyID repository.EmailHistoryIDRepository syncState repository.EmailSyncStateRepository @@ -210,11 +215,15 @@ func (s *emailService) WirePoolLink(repo repository.PoolLinkRepository) { s.poolLink = repo } -// WireThrottle attaches the daily-creation throttle after construction -// so callers without a Redis cache (jobs, tests) need not provide one. -// When unset, guardMailboxThrottle is a no-op. -func (s *emailService) WireThrottle(t dailythrottle.Service) { - s.throttle = t +// MailboxAllowanceSource answers how many mailboxes a workspace may hold. +// Satisfied by the organization service; injected post-construction so this +// package needs no import of it. +type MailboxAllowanceSource interface { + MailboxAllowance(ctx context.Context, orgID uuid.UUID) (*models.MailboxAllowance, *errx.Error) +} + +func (s *emailService) WireMailboxAllowance(src MailboxAllowanceSource) { + s.allowance = src } // WireWebhooks attaches the webhook dispatcher after construction. Done diff --git a/internal/app/feature/gate.go b/internal/app/feature/gate.go index 879c2078..971ad471 100644 --- a/internal/app/feature/gate.go +++ b/internal/app/feature/gate.go @@ -86,6 +86,10 @@ type featureGateService struct { selfHost bool // poolLink entitles a linked workspace to warm without a paid plan; nil when not wired. poolLink PoolLinkReader + // overrides is the per-org limit override row, so an approved daily-send + // increase raises what the sender enforces and not only what the + // dashboard shows. Nil when not wired. + overrides LimitOverrideReader } // PoolLinkReader answers whether a workspace has a live self-hosted link. @@ -93,6 +97,12 @@ type PoolLinkReader interface { HasActiveLink(ctx context.Context, orgID uuid.UUID) bool } +// LimitOverrideReader reads the operator override row for an organization. +// Satisfied by the organization repository. +type LimitOverrideReader interface { + GetOrganizationLimitOverrides(ctx context.Context, orgID uuid.UUID) (*models.OrganizationLimitOverrides, error) +} + func NewService(subRepo repository.SubscriptionRepository, planRepo repository.PlanRepository) FeatureGateService { return &featureGateService{ subRepo: subRepo, @@ -104,6 +114,21 @@ func NewService(subRepo repository.SubscriptionRepository, planRepo repository.P // WirePoolLink attaches the pool-link entitlement after construction. func (s *featureGateService) WirePoolLink(r PoolLinkReader) { s.poolLink = r } +// WireLimitOverrides attaches the override reader after construction. +func (s *featureGateService) WireLimitOverrides(r LimitOverrideReader) { s.overrides = r } + +// dailyOverride is the operator-granted daily send cap, or 0 when none. +func (s *featureGateService) dailyOverride(ctx context.Context, orgID uuid.UUID) int { + if s.overrides == nil { + return 0 + } + o, err := s.overrides.GetOrganizationLimitOverrides(ctx, orgID) + if err != nil || o == nil { + return 0 + } + return o.DailyCampaignLimit +} + // CanSendCampaignEmail checks if an organization can send campaign emails func (s *featureGateService) CanSendCampaignEmail(ctx context.Context, orgID uuid.UUID) (bool, *errx.Error) { if s.selfHost { @@ -204,8 +229,11 @@ func (s *featureGateService) GetDailyEmailLimit(ctx context.Context, orgID uuid. return FreeTierDailyEmailLimit, nil } - // Paid users = plan limit or unlimited + // Paid users = approved override, else plan limit, else unlimited if sub.HasPaidSubscription() { + if ov := s.dailyOverride(ctx, orgID); ov > 0 { + return ov, nil + } plan, err := s.planRepo.GetByID(ctx, sub.PlanID) if err != nil || plan == nil { return UnlimitedEmails, nil // Default to unlimited if plan not found @@ -262,7 +290,9 @@ func (s *featureGateService) GetSubscriptionStatus(ctx context.Context, orgID uu if status.IsInFreeTrial && !status.IsPaidSubscriber { status.DailyEmailLimit = FreeTierDailyEmailLimit } else if status.IsPaidSubscriber { - if plan != nil && plan.DailyCampaignLimit != nil { + if ov := s.dailyOverride(ctx, orgID); ov > 0 { + status.DailyEmailLimit = ov + } else if plan != nil && plan.DailyCampaignLimit != nil { status.DailyEmailLimit = *plan.DailyCampaignLimit } else { status.DailyEmailLimit = UnlimitedEmails diff --git a/internal/app/form/service.go b/internal/app/form/service.go index 224dff87..9dd1cfae 100644 --- a/internal/app/form/service.go +++ b/internal/app/form/service.go @@ -443,6 +443,15 @@ func (s *service) Submit(ctx context.Context, publicID string, answers map[strin if contactID != "" { payload["contact_id"] = contactID } + // The mapped contact columns ride along flat so an automation can read + // {{.contact_email}} or {{.first_name}} without digging into data. + if lead != nil { + payload["contact_email"] = lead.Email + payload["first_name"] = lead.FirstName + payload["last_name"] = lead.LastName + payload["company"] = lead.Company + payload["phone"] = lead.Phone + } if sub.CampaignID != nil { payload["campaign_id"] = sub.CampaignID.String() } diff --git a/internal/app/instancecheck/checks_updates.go b/internal/app/instancecheck/checks_updates.go new file mode 100644 index 00000000..c42efeca --- /dev/null +++ b/internal/app/instancecheck/checks_updates.go @@ -0,0 +1,60 @@ +package instancecheck + +import ( + "context" + "fmt" +) + +const docsUpdates = "/development/updates/" + +func updateChecks() []check { + return []check{ + {id: "update_available", run: checkUpdateAvailable}, + {id: "updater_unreachable", run: checkUpdaterUnreachable}, + } +} + +// checkUpdateAvailable is the health-page twin of the top-bar indicator, so an +// operator who only reads findings still learns a newer Warmbly exists. +func checkUpdateAvailable(ctx context.Context, d Deps, _ Input) *Finding { + if d.Updates == nil { + return nil + } + st := d.Updates.State(ctx, false) + if !st.UpdateAvailable { + return nil + } + var msg string + switch { + case st.Reason == "release" && st.Latest != nil: + msg = fmt.Sprintf("Warmbly %s is available and this instance runs %s. ", st.Latest.Tag, st.Running.Version) + case st.Updater.Checkout != nil: + msg = fmt.Sprintf("The checkout is %d commits behind %s. ", st.Updater.Checkout.Behind, st.Updater.Checkout.Branch) + default: + msg = "A newer version is available. " + } + if st.Updater.Status == "ok" { + msg += "Open the version pill in the top bar and choose Update and restart; the stack restarts and resumes on its own." + } else if st.Updater.Release != nil { + msg += "On the host, run docker compose pull && docker compose up -d in the install directory, or enable the updater to do it from this panel." + } else { + msg += "Run make upgrade on the host, or enable the updater to do it from this panel." + } + return result(CategoryUpdates, SeverityInfo, "An update is available", msg, docsUpdates) +} + +// checkUpdaterUnreachable fires only when an updater is configured and not +// answering; an instance that never configured one is not misconfigured. +func checkUpdaterUnreachable(ctx context.Context, d Deps, _ Input) *Finding { + if d.Updates == nil { + return nil + } + st := d.Updates.State(ctx, false) + if st.Updater.Status != "unreachable" { + return nil + } + return result(CategoryUpdates, SeverityWarning, "The updater is not answering", + "UPDATER_URL is set but the backend cannot reach the updater, so Update and restart in the panel cannot work. "+ + st.Updater.Error+" Remove UPDATER_URL if you update by hand.", + docsUpdates) +} diff --git a/internal/app/instancecheck/instancecheck.go b/internal/app/instancecheck/instancecheck.go index 226ca237..39433062 100644 --- a/internal/app/instancecheck/instancecheck.go +++ b/internal/app/instancecheck/instancecheck.go @@ -14,6 +14,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/warmbly/warmbly/internal/app/instanceconfig" + "github.com/warmbly/warmbly/internal/app/updates" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/infrastructure/cache" "github.com/warmbly/warmbly/internal/notify" @@ -40,6 +41,7 @@ const ( CategoryAccess = "access" CategoryWorkers = "workers" CategoryData = "data" + CategoryUpdates = "updates" ) // Finding is one thing that is wrong, plus how to fix it. @@ -80,6 +82,8 @@ type Deps struct { Policy *config.AuthPolicy DB *pgxpool.Pool Cache *cache.Cache + // Updates is the release and updater state; nil skips the update checks. + Updates *updates.Service } type check struct { @@ -101,6 +105,7 @@ func New(deps Deps) *Registry { r.checks = append(r.checks, mailChecks()...) r.checks = append(r.checks, accessChecks()...) r.checks = append(r.checks, infraChecks()...) + r.checks = append(r.checks, updateChecks()...) return r } diff --git a/internal/app/instanceconfig/entries.go b/internal/app/instanceconfig/entries.go index f2b7b1a6..57fb9c42 100644 --- a/internal/app/instanceconfig/entries.go +++ b/internal/app/instanceconfig/entries.go @@ -26,6 +26,10 @@ const ( docsCaptcha = "/development/configuration/#captcha" docsSSO = "/development/accounts-and-access/#single-sign-on" docsFirstOwner = "/development/accounts-and-access/#first-owner" + docsUpdates = "/development/updates/" + // The database-backed settings document, which is the one tier the + // environment does not own. + docsSettingsDoc = "/development/configuration/#settings-stored-in-the-database" ) // table is the static inventory. Declaration order is display order. @@ -507,6 +511,12 @@ var table = []Entry{ DocsAnchor: docsFirstOwner, Resolve: envValue("WARMBLY_BOOTSTRAP_ORG"), }, + { + Key: "WARMBLY_SETTINGS_BOOTSTRAP", Group: GroupDeployment, RuntimeChangeable: ChangeBootOnly, + Effect: "Seeds the database-backed settings document (sync budgets, retention windows) on an instance that has never saved one. A no-op from the first save in Instance settings onwards, so leaving it here cannot undo an edit made there.", + DocsAnchor: docsSettingsDoc, + Resolve: envValue("WARMBLY_SETTINGS_BOOTSTRAP"), + }, // Captcha. { @@ -773,6 +783,49 @@ var table = []Entry{ DocsAnchor: docsDeployment, Resolve: envValue("SENTRY_DSN"), }, + + // Updates. + { + Key: "UPDATE_CHECK_ENABLED", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "Polls GitHub Releases for a newer Warmbly and shows it in the admin panel's top bar and on Setup and health.", + DocsAnchor: docsUpdates, + Resolve: boolOr("UPDATE_CHECK_ENABLED", true), + }, + { + Key: "UPDATE_CHECK_INTERVAL", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "How often the release check runs. Minimum 5m.", + DocsAnchor: docsUpdates, + Resolve: envOr("UPDATE_CHECK_INTERVAL", "30m"), + }, + { + Key: "UPDATE_CHANNEL", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "stable follows releases; dev also offers prereleases.", + DocsAnchor: docsUpdates, + Resolve: envOr("UPDATE_CHANNEL", "stable"), + }, + { + Key: "RELEASES_GITHUB_REPO", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "The owner/repo whose releases count as Warmbly versions. Point a fork's instance at the fork.", + DocsAnchor: docsUpdates, + Resolve: envOr("RELEASES_GITHUB_REPO", "warmbly/warmbly"), + }, + { + Key: "UPDATER_URL", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "The host-side updater that applies an update (pull, rebuild, restart). Unset leaves the panel report-only.", + DocsAnchor: docsUpdates, + Resolve: envValue("UPDATER_URL"), + }, + { + Key: "UPDATER_TOKEN", Group: GroupUpdates, RuntimeChangeable: ChangeBootOnly, + Effect: "The bearer token the backend presents to the updater. Falls back to INTERNAL_API_TOKEN.", + DocsAnchor: docsUpdates, WhenUnset: SourceDerived, + Resolve: func(*Runtime) string { + if v := trimmed("UPDATER_TOKEN"); v != "" { + return v + } + return trimmed("INTERNAL_API_TOKEN") + }, + }, } // ssoRedirect mirrors the backend's own derivation, so the configuration page diff --git a/internal/app/instanceconfig/instanceconfig.go b/internal/app/instanceconfig/instanceconfig.go index 8f1748ba..b11fdfbb 100644 --- a/internal/app/instanceconfig/instanceconfig.go +++ b/internal/app/instanceconfig/instanceconfig.go @@ -28,6 +28,7 @@ const ( GroupTracking = "tracking" GroupCaptcha = "captcha" GroupObservability = "observability" + GroupUpdates = "updates" ) // Where a resolved value came from. diff --git a/internal/app/instanceconfig/limits.go b/internal/app/instanceconfig/limits.go index d26315a1..ca5dfcd1 100644 --- a/internal/app/instanceconfig/limits.go +++ b/internal/app/instanceconfig/limits.go @@ -4,6 +4,7 @@ import ( "strconv" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/models" ) // LimitEntry is one number an operator may be looking for. @@ -47,10 +48,18 @@ func Limits() []LimitGroup { {"Warmup ramp", "+" + n(config.WarmupIncreaseDefault), "emails/day", "Added each day while the mailbox stays healthy."}, }, }, + { + Title: "Mailbox allowance", + Entries: []LimitEntry{ + {"Fair-use sends per mailbox", n(config.FairUseSendsPerMailbox), "sends/day", "A paid plan holds one mailbox for every this many daily sends it includes; a plan with no daily cap holds unlimited mailboxes."}, + {"Free workspace mailboxes", n(models.FreeWorkspaceMailboxLimit), "per organization", "Without a paid plan."}, + {"Bulk connect batch", n(config.MailboxBulkBatchMax), "rows/request", "The most SMTP/IMAP rows one bulk connect call carries; the dashboard streams a CSV through batches of this size."}, + {"Bulk connect concurrency", n(config.MailboxBulkConcurrency), "validations", "Credentials dialled at the same time within one batch."}, + }, + }, { Title: "Organization hard caps", Entries: []LimitEntry{ - {"Connected mailboxes", n(config.HardCapMailboxes), "per organization", "The backstop when neither a plan nor an override sets one."}, {"Campaigns created", n(config.HardCapCampaignsTotal), "per organization", "Total campaigns ever created."}, {"Active campaigns", n(config.HardCapCampaignsActive), "per organization", "Running at the same time."}, {"Team members", n(config.HardCapTeamMembers), "seats", "Members in one organization."}, @@ -62,7 +71,6 @@ func Limits() []LimitGroup { Title: "Daily creation throttles", Entries: []LimitEntry{ {"New campaigns", n(config.DailyThrottleNewCampaigns), "per organization/day", "Resets at UTC midnight. Not raisable per organization."}, - {"Newly connected mailboxes", n(config.DailyThrottleNewMailboxes), "per organization/day", "Resets at UTC midnight."}, {"New workspaces", n(config.DailyThrottleNewOrgs), "per owner/day", "Resets at UTC midnight."}, }, }, diff --git a/internal/app/instancesettings/document.go b/internal/app/instancesettings/document.go index fc6a1a2c..1933a779 100644 --- a/internal/app/instancesettings/document.go +++ b/internal/app/instancesettings/document.go @@ -48,6 +48,27 @@ type Sync struct { DailyMessagesPerOrg int `json:"daily_messages_per_org"` } +// Retention holds how long event-level history is kept. Every window here +// bounds personal data: opens and clicks carry a client, a device and a +// location, funnel events carry a visitor's path, and the audit trail carries +// IP addresses, user agents and change payloads. Shortening one is the only +// way an operator can hold less without patching the binary. +// +// None of them affect a count, a filter or a routing decision: campaign +// progress keeps its own summary of opens and clicks, which outlives the +// per-event log. +type Retention struct { + // EngagementEventDays is how long the per-event open and click logs are + // kept. + EngagementEventDays int `json:"engagement_event_days"` + // FormEventDays is how long form funnel events are kept. Forms analytics + // ranges top out at 90 days, so anything shorter than that shortens what + // the funnel report can show. + FormEventDays int `json:"form_event_days"` + // AuditLogDays is how long the audit trail is kept. + AuditLogDays int `json:"audit_log_days"` +} + // Bounds on the domain-authentication grace window. One hour is the shortest // window that still absorbs a resolver blip; 30 days is the longest a domain // should keep sending cold mail unauthenticated while being warned about it. @@ -76,7 +97,9 @@ type Document struct { Invitations Invitations `json:"invitations"` Access Access `json:"access"` Sync Sync `json:"sync"` + Retention Retention `json:"retention"` Deliverability Deliverability `json:"deliverability"` + Notifications Notifications `json:"notifications"` } // Defaults is the document a fresh instance behaves as if it had. @@ -90,6 +113,7 @@ func Defaults() Document { AllowInvitedSignup: true, }, Sync: DefaultSync(), + Retention: DefaultRetention(), Deliverability: DefaultDeliverability(), } } @@ -115,6 +139,37 @@ func DefaultSync() Sync { } } +// DefaultRetention is the compiled retention window for each event log. +func DefaultRetention() Retention { + return Retention{ + EngagementEventDays: config.EngagementEventRetentionDaysDefault, + FormEventDays: config.FormEventsRetentionDaysDefault, + AuditLogDays: config.AuditLogRetentionDaysDefault, + } +} + +// Normalize clamps every window into its accepted range. Zero and negative +// resolve to the compiled default rather than to "keep nothing": a document +// written before this section existed must not silently start deleting +// everything on the next sweep. +func (r *Retention) Normalize() { + clamp := func(v, def int) int { + if v <= 0 { + return def + } + if v < config.RetentionDaysMin { + return config.RetentionDaysMin + } + if v > config.RetentionDaysMax { + return config.RetentionDaysMax + } + return v + } + r.EngagementEventDays = clamp(r.EngagementEventDays, config.EngagementEventRetentionDaysDefault) + r.FormEventDays = clamp(r.FormEventDays, config.FormEventsRetentionDaysDefault) + r.AuditLogDays = clamp(r.AuditLogDays, config.AuditLogRetentionDaysDefault) +} + // Normalize clamps a document into its accepted range. It is applied on read // as well as on write, so a row written by an older version still resolves. func (d *Document) Normalize() { @@ -128,7 +183,9 @@ func (d *Document) Normalize() { d.Invitations.TTLHours = TTLHoursMax } d.Sync.Normalize() + d.Retention.Normalize() d.Deliverability.Normalize() + d.Notifications.Normalize() } // Normalize clamps the grace window. Zero and negative resolve to the compiled @@ -192,10 +249,21 @@ type Patch struct { DailyMessagesPerMailbox *int `json:"daily_messages_per_mailbox"` DailyMessagesPerOrg *int `json:"daily_messages_per_org"` } `json:"sync"` + Retention *struct { + EngagementEventDays *int `json:"engagement_event_days"` + FormEventDays *int `json:"form_event_days"` + AuditLogDays *int `json:"audit_log_days"` + } `json:"retention"` Deliverability *struct { EnforceDomainAuth *bool `json:"enforce_domain_auth"` AuthGraceHours *int `json:"auth_grace_hours"` } `json:"deliverability"` + // Channels replaces the whole list when present. A channel that comes back + // with a masked target or secret keeps the stored value, so the admin panel + // can round-trip a redacted read without wiping credentials. + Notifications *struct { + Channels *[]NotifyChannel `json:"channels"` + } `json:"notifications"` } // Apply merges a patch onto a document. @@ -230,6 +298,17 @@ func (p Patch) Apply(doc Document) Document { doc.Sync.DailyMessagesPerOrg = *p.Sync.DailyMessagesPerOrg } } + if p.Retention != nil { + if p.Retention.EngagementEventDays != nil { + doc.Retention.EngagementEventDays = *p.Retention.EngagementEventDays + } + if p.Retention.FormEventDays != nil { + doc.Retention.FormEventDays = *p.Retention.FormEventDays + } + if p.Retention.AuditLogDays != nil { + doc.Retention.AuditLogDays = *p.Retention.AuditLogDays + } + } if p.Deliverability != nil { if p.Deliverability.EnforceDomainAuth != nil { doc.Deliverability.EnforceDomainAuth = *p.Deliverability.EnforceDomainAuth @@ -243,5 +322,8 @@ func (p Patch) Apply(doc Document) Document { } } } + if p.Notifications != nil && p.Notifications.Channels != nil { + doc.Notifications.Channels = mergeChannels(doc.Notifications.Channels, *p.Notifications.Channels) + } return doc } diff --git a/internal/app/instancesettings/notifications.go b/internal/app/instancesettings/notifications.go new file mode 100644 index 00000000..c231d63e --- /dev/null +++ b/internal/app/instancesettings/notifications.go @@ -0,0 +1,305 @@ +package instancesettings + +import ( + "net" + "net/url" + "os" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/pkg/safehttp" +) + +// Operator notification channels. These are instance-wide, not per +// organization: they answer "tell me when something happens on my deployment", +// which is why they live in the settings document rather than in the +// customer-facing webhook tables. + +// Channel transports. +const ( + ChannelDiscord = "discord" + ChannelSlack = "slack" + ChannelWebhook = "webhook" + ChannelEmail = "email" +) + +// MaxChannels bounds the fan-out. Each delivery is an outbound request the +// backend makes on the instance's behalf, so the list is not unbounded. +const MaxChannels = 25 + +// Masked is what a secret-bearing field reads as on the way out. A client that +// sends it back unchanged means "keep the stored value". +const Masked = "••••••••" + +// NotifyChannel is one operator destination. +type NotifyChannel struct { + ID string `json:"id"` + Name string `json:"name"` + // Type is one of the Channel* constants. + Type string `json:"type"` + // Target is the incoming-webhook URL, or the address for an email channel. + // It is a bearer credential for every webhook transport (anyone holding a + // Discord webhook URL can post to that room), so it is masked on read. + Target string `json:"target"` + // Secret signs the generic webhook payload (HMAC-SHA256, same scheme as + // customer webhooks). Ignored by the other transports. + Secret string `json:"secret,omitempty"` + // Events subscribed to, by key. Empty means every event. + Events []string `json:"events"` + Enabled bool `json:"enabled"` +} + +// Notifications is the operator notification section. +type Notifications struct { + Channels []NotifyChannel `json:"channels"` +} + +// Wants reports whether this channel should receive the given event. +func (c NotifyChannel) Wants(eventKey string) bool { + if !c.Enabled { + return false + } + if len(c.Events) == 0 { + return true + } + for _, e := range c.Events { + if e == eventKey { + return true + } + } + return false +} + +// IsWebhookTransport reports whether Target is a URL rather than an address. +func (c NotifyChannel) IsWebhookTransport() bool { + return c.Type == ChannelDiscord || c.Type == ChannelSlack || c.Type == ChannelWebhook +} + +// Redacted is the channel as it leaves the process: the target reduced to a +// recognisable preview and the secret replaced by a set/unset flag, so an +// admin can tell channels apart without the response handing out credentials. +func (c NotifyChannel) Redacted() NotifyChannel { + out := c + out.Target = previewTarget(c.Type, c.Target) + if c.Secret != "" { + out.Secret = Masked + } + return out +} + +// previewTarget keeps an email address readable (it is not a credential) and +// reduces a webhook URL to its host plus the tail of its path. +func previewTarget(kind, target string) string { + target = strings.TrimSpace(target) + if target == "" || kind == ChannelEmail { + return target + } + u, err := url.Parse(target) + if err != nil || u.Host == "" { + return Masked + } + tail := u.Path + if len(tail) > 6 { + tail = "…" + tail[len(tail)-6:] + } + return u.Host + tail +} + +// NormalizeChannels validates, de-duplicates and bounds the channel list. +// Invalid entries are dropped rather than rejected so one bad row saved by an +// older client cannot make the whole document unreadable. +func (n *Notifications) Normalize() { + seen := make(map[string]bool, len(n.Channels)) + out := make([]NotifyChannel, 0, len(n.Channels)) + for _, ch := range n.Channels { + ch.Name = strings.TrimSpace(ch.Name) + ch.Target = strings.TrimSpace(ch.Target) + ch.Secret = strings.TrimSpace(ch.Secret) + ch.Type = strings.ToLower(strings.TrimSpace(ch.Type)) + + if !validChannelType(ch.Type) || ch.Target == "" { + continue + } + if ch.IsWebhookTransport() && ValidateChannelURL(ch.Target) != nil { + continue + } + if ch.Type == ChannelEmail && !looksLikeEmail(ch.Target) { + continue + } + if ch.ID == "" { + ch.ID = uuid.NewString() + } + if seen[ch.ID] { + continue + } + seen[ch.ID] = true + if ch.Name == "" { + ch.Name = defaultChannelName(ch.Type) + } + ch.Events = dedupe(ch.Events) + out = append(out, ch) + if len(out) >= MaxChannels { + break + } + } + n.Channels = out +} + +func validChannelType(t string) bool { + switch t { + case ChannelDiscord, ChannelSlack, ChannelWebhook, ChannelEmail: + return true + } + return false +} + +func defaultChannelName(t string) string { + switch t { + case ChannelDiscord: + return "Discord" + case ChannelSlack: + return "Slack" + case ChannelEmail: + return "Email" + default: + return "Webhook" + } +} + +func dedupe(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]bool, len(in)) + out := make([]string, 0, len(in)) + for _, v := range in { + v = strings.TrimSpace(v) + if v == "" || seen[v] { + continue + } + seen[v] = true + out = append(out, v) + } + return out +} + +func looksLikeEmail(v string) bool { + at := strings.IndexByte(v, '@') + return at > 0 && at < len(v)-1 && strings.Contains(v[at+1:], ".") && !strings.ContainsAny(v, " \t\r\n") +} + +// ValidateChannelURL applies the same SSRF posture as customer webhooks: +// HTTPS to a publicly routable host, no inline credentials. A self-hosted or +// development deployment opts out with WARMBLY_ALLOW_UNSAFE_WEBHOOK_URLS, +// which is how an operator points a channel at a LAN chat server. +func ValidateChannelURL(raw string) error { + raw = strings.TrimSpace(raw) + if raw == "" { + return errChannelURL("url is required") + } + u, err := url.Parse(raw) + if err != nil { + return errChannelURL("invalid url") + } + allowUnsafe := strings.EqualFold(os.Getenv("WARMBLY_ALLOW_UNSAFE_WEBHOOK_URLS"), "true") + if u.Scheme != "https" && !(allowUnsafe && u.Scheme == "http") { + return errChannelURL("url scheme must be https") + } + if u.Host == "" { + return errChannelURL("url must have a host") + } + if u.User != nil { + return errChannelURL("url must not contain credentials") + } + if !allowUnsafe && isPrivateHost(u.Hostname()) { + return errChannelURL("url host must be publicly routable") + } + return nil +} + +func isPrivateHost(host string) bool { + host = strings.Trim(strings.ToLower(host), "[]") + if host == "" || host == "localhost" || strings.HasSuffix(host, ".localhost") { + return true + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + return safehttp.IsBlockedIP(ip) +} + +type channelURLError string + +func (e channelURLError) Error() string { return string(e) } + +func errChannelURL(msg string) error { return channelURLError(msg) } + +// mergeChannels applies an incoming list over the stored one. The admin panel +// reads channels with their target and secret redacted, so an unchanged field +// comes back as the mask (or empty) and must resolve to what is already +// stored; otherwise saving an unrelated toggle would wipe every credential. +func mergeChannels(stored, incoming []NotifyChannel) []NotifyChannel { + byID := make(map[string]NotifyChannel, len(stored)) + for _, ch := range stored { + byID[ch.ID] = ch + } + out := make([]NotifyChannel, 0, len(incoming)) + for _, ch := range incoming { + prev, existed := byID[ch.ID] + if existed { + // Only the redacted form means "unchanged". An empty field is a + // deliberate clear: the panel empties the target when the channel + // type changes, and restoring the old one there would post Slack + // payloads to a Discord webhook. An emptied target fails + // validation in Normalize, which is the intended outcome. + if ch.Target == Masked || ch.Target == prev.Redacted().Target { + ch.Target = prev.Target + } + // Same for the signing secret, so it can actually be removed. + if ch.Secret == Masked { + ch.Secret = prev.Secret + } + // A type change invalidates a target carried over from the old + // transport even when the string was sent back unchanged. + if ch.Type != prev.Type { + ch.Target = strings.TrimSpace(ch.Target) + if ch.Target == Masked || ch.Target == prev.Redacted().Target || ch.Target == prev.Target { + ch.Target = "" + } + } + } + out = append(out, ch) + } + return out +} + +// RedactedChannels is the channel list as it leaves the process. +func (n Notifications) RedactedChannels() []NotifyChannel { + out := make([]NotifyChannel, 0, len(n.Channels)) + for _, ch := range n.Channels { + out = append(out, ch.Redacted()) + } + return out +} + +// Subscribers returns every enabled channel that wants the given event. +func (n Notifications) Subscribers(eventKey string) []NotifyChannel { + out := make([]NotifyChannel, 0, len(n.Channels)) + for _, ch := range n.Channels { + if ch.Wants(eventKey) { + out = append(out, ch) + } + } + return out +} + +// Find returns the channel with the given id. +func (n Notifications) Find(id string) (NotifyChannel, bool) { + for _, ch := range n.Channels { + if ch.ID == id { + return ch, true + } + } + return NotifyChannel{}, false +} diff --git a/internal/app/instancesettings/service.go b/internal/app/instancesettings/service.go index 0e74294c..05281af6 100644 --- a/internal/app/instancesettings/service.go +++ b/internal/app/instancesettings/service.go @@ -27,12 +27,23 @@ type Service interface { // settings row is missing. Get(ctx context.Context) Document Put(ctx context.Context, patch Patch, updatedBy *uuid.UUID) (Document, error) + // Bootstrap applies patch only on an instance whose settings row has never + // been written, and reports whether it did. It is how an unattended + // install ships the data-control answers with the rest of the + // environment; from the first write onwards the admin panel is + // authoritative and this is a no-op, so leaving the variable in place does + // not undo an operator's later edit. + Bootstrap(ctx context.Context, patch Patch) (bool, error) InvitationTTL(ctx context.Context) time.Duration InviteLinksEnabled(ctx context.Context) bool AllowInvitedSignup(ctx context.Context) bool // SyncBudget is the mailbox sync fair-use section, already normalized. SyncBudget(ctx context.Context) Sync + // RetentionWindows is the event-log retention section, already normalized. + // The retention sweeps read it on every pass, so an edit takes effect on + // the next one rather than at the next restart. + RetentionWindows(ctx context.Context) Retention // DomainAuth is the sending-domain authentication gate: whether it is // enforced at all, and how long a domain must stay failing first. DomainAuth(ctx context.Context) (enforce bool, grace time.Duration) @@ -90,6 +101,29 @@ func (s *service) Put(ctx context.Context, patch Patch, updatedBy *uuid.UUID) (D return next, nil } +func (s *service) Bootstrap(ctx context.Context, patch Patch) (bool, error) { + if s.store == nil { + return false, nil + } + next := patch.Apply(Defaults()) + next.Normalize() + // One statement, not a read then a write: two processes booting together + // must not both find the row absent and have the loser overwrite whatever + // the winner wrote. The insert is skipped entirely when a row exists, so a + // document an admin has already saved is never touched. + inserted, err := s.store.Insert(ctx, next) + if err != nil { + return false, err + } + if !inserted { + return false, nil + } + s.mu.Lock() + s.cached, s.cachedAt, s.loaded = next, time.Now(), true + s.mu.Unlock() + return true, nil +} + func (s *service) InvitationTTL(ctx context.Context) time.Duration { return s.Get(ctx).TTL() } func (s *service) InviteLinksEnabled(ctx context.Context) bool { @@ -106,6 +140,12 @@ func (s *service) SyncBudget(ctx context.Context) Sync { return sync } +func (s *service) RetentionWindows(ctx context.Context) Retention { + r := s.Get(ctx).Retention + r.Normalize() + return r +} + func (s *service) DomainAuth(ctx context.Context) (bool, time.Duration) { d := s.Get(ctx).Deliverability d.Normalize() diff --git a/internal/app/instancesettings/store.go b/internal/app/instancesettings/store.go index f366acee..68f1128c 100644 --- a/internal/app/instancesettings/store.go +++ b/internal/app/instancesettings/store.go @@ -13,6 +13,11 @@ import ( type Store interface { Get(ctx context.Context) (Document, error) Put(ctx context.Context, doc Document, updatedBy *uuid.UUID) error + // Insert writes the document only when no row exists yet, and reports + // whether it did. It is one statement rather than a read followed by a + // write, so two processes booting together cannot both decide the row is + // absent and have the loser overwrite the winner. + Insert(ctx context.Context, doc Document) (bool, error) } type pgStore struct { @@ -43,6 +48,22 @@ func (s *pgStore) Get(ctx context.Context) (Document, error) { return doc, nil } +func (s *pgStore) Insert(ctx context.Context, doc Document) (bool, error) { + raw, err := json.Marshal(doc) + if err != nil { + return false, err + } + tag, err := s.db.Exec(ctx, ` + INSERT INTO instance_settings (id, doc, updated_at) + VALUES (true, $1::jsonb, NOW()) + ON CONFLICT (id) DO NOTHING + `, raw) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + func (s *pgStore) Put(ctx context.Context, doc Document, updatedBy *uuid.UUID) error { raw, err := json.Marshal(doc) if err != nil { diff --git a/internal/app/integration/dispatch.go b/internal/app/integration/dispatch.go index 1e0cd06d..fc4348bc 100644 --- a/internal/app/integration/dispatch.go +++ b/internal/app/integration/dispatch.go @@ -39,6 +39,14 @@ func (s *service) Dispatch(ctx context.Context, orgID uuid.UUID, eventType model if err != nil { log.Warn().Err(err).Str("event", string(eventType)).Msg("integration dispatch: failed to load automations") } + // An event raised by an automation's own action (a contact it created, a + // deal it opened) carries its depth. Past the chain cap it still reaches + // customer webhooks and legacy subscriptions, but runs no more flows, so + // "on new contact, create a contact" cannot loop. + if depth := int(toFloat(data[automationDepthKey])); depth >= maxAutomationChainDepth && len(autos) > 0 { + log.Warn().Str("event", string(eventType)).Int("depth", depth).Msg("integration dispatch: automation chain depth limit reached; not running automations") + autos = nil + } if len(targets) == 0 && len(autos) == 0 { return } @@ -290,6 +298,11 @@ func renderEventMessage(sub models.IntegrationEventSubscription, data map[string m.Title = "✅ Campaign completed" case models.WebhookEventContactCreated: m.Title = "🧑 Contact created" + case models.WebhookEventFormSubmitted: + m.Title = "📝 Form submitted" + if n := stringFromMap(data, "form_name"); n != "" { + m.Title = "📝 Form submitted: " + n + } case models.WebhookEventContactUpdated: m.Title = "✏️ Contact updated" case models.WebhookEventCRMDealCreated: diff --git a/internal/app/integration/graph_executor.go b/internal/app/integration/graph_executor.go index 287e6e72..e9b7e228 100644 --- a/internal/app/integration/graph_executor.go +++ b/internal/app/integration/graph_executor.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/webhook" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" @@ -43,6 +44,11 @@ func (s *service) executeAutomationGraph(ctx context.Context, a models.Automatio } } + // Anything an action does under this context (a contact it creates, a deal + // it opens) is one hop deeper, so the events those writes fire carry the + // depth and Dispatch can stop a flow that keeps re-triggering itself. + ctx = webhook.WithAutomationDepth(ctx, int(toFloat(data[automationDepthKey]))+1) + // Best-effort run record — observability, never blocks the walk. run := &models.AutomationRun{AutomationID: a.ID, OrganizationID: a.OrganizationID, TriggerEvent: eventType, Status: "running"} _ = s.repo.CreateAutomationRun(ctx, run) @@ -178,7 +184,7 @@ func conditionSummary(c *models.AutomationCondition) string { // automation, that launcher MUST forward this key so a misconfigured loop // (campaign -> automation -> campaign -> ...) is bounded instead of infinite. const ( - automationDepthKey = "_automation_depth" + automationDepthKey = webhook.AutomationDepthKey maxAutomationChainDepth = 5 ) @@ -386,6 +392,27 @@ func actionPreview(n models.AutomationNode, data map[string]any) map[string]any if cfg.TaskTitle != "" { p["task_title"] = renderTemplate(cfg.TaskTitle, data) } + if n.Action == models.IntegrationActionUpsertContact { + for k, v := range map[string]string{ + "email": cfg.Email, "first_name": cfg.FirstName, "last_name": cfg.LastName, + "company": cfg.Company, "phone": cfg.Phone, + } { + if v != "" { + p[k] = renderTemplate(v, data) + } + } + for _, f := range cfg.CustomFields { + if k := strings.TrimSpace(f.Key); k != "" { + p["custom:"+k] = renderTemplate(f.Value, data) + } + } + if cfg.CampaignID != "" { + p["campaign_id"] = cfg.CampaignID + } + } + if n.Action == models.IntegrationActionAddToCampaign && cfg.CampaignID != "" { + p["campaign_id"] = cfg.CampaignID + } if n.Action == models.IntegrationActionFireEvent { p["event"] = renderTemplate(cfg.EventName, data) for _, f := range cfg.EventFields { @@ -406,6 +433,9 @@ func actionPreview(n models.AutomationNode, data map[string]any) map[string]any func actionRunOutput(n models.AutomationNode, data map[string]any) map[string]any { out := actionPreview(n, data) switch { + case n.Action == models.IntegrationActionUpsertContact: + out["contact_id"] = valueString(data["contact_id"]) + out["contact_created"] = valueString(data["contact_created"]) case n.Action == models.IntegrationActionSetVariables: cfg := parseNativeConfig(n.Config) for _, v := range cfg.SetVars { @@ -459,6 +489,27 @@ func sampleEventData(triggerEvent string) map[string]any { base["new_state"] = "watch" base["previous_state"] = "healthy" base["reason"] = "spam placement rising" + case "contact.created": + delete(base, "campaign_id") + delete(base, "campaign_name") + base["phone"] = "+1 555 0100" + base["subscribed"] = true + base["custom_fields"] = map[string]any{"industry": "SaaS"} + base["source"] = "form" + base["source_detail"] = "Demo request" + base["campaign_ids"] = []any{} + base["category_ids"] = []any{} + case "form.submitted": + delete(base, "campaign_name") + base["form_id"] = "00000000-0000-0000-0000-000000000003" + base["form_name"] = "Demo request" + base["submission_id"] = "00000000-0000-0000-0000-000000000004" + base["phone"] = "+1 555 0100" + base["source_url"] = "https://example.com/pricing" + base["data"] = map[string]any{ + "email": "jane@example.com", "first_name": "Jane", "last_name": "Doe", + "company": "Example Inc", "team_size": "10-50", + } } return base } diff --git a/internal/app/integration/native_actions.go b/internal/app/integration/native_actions.go index 0b0b7bc0..250d5b4f 100644 --- a/internal/app/integration/native_actions.go +++ b/internal/app/integration/native_actions.go @@ -48,6 +48,12 @@ func validateNativeActionConfig(action models.IntegrationAction, raw json.RawMes if strings.TrimSpace(cfg.EventName) == "" { return fmt.Errorf("a fire-event action needs an event name") } + case models.IntegrationActionUpsertContact: + return validateUpsertContactConfig(cfg) + case models.IntegrationActionAddToCampaign: + if _, err := uuid.Parse(strings.TrimSpace(cfg.CampaignID)); err != nil { + return fmt.Errorf("an add-to-campaign action needs a campaign") + } case models.IntegrationActionAIStep: return validateAIStepConfig(raw) case models.IntegrationActionAISwitch: @@ -169,6 +175,12 @@ type NativeActions interface { // "label_email" action; userID + threadID come from the reply event data. LabelThread(ctx context.Context, userID uuid.UUID, threadID string, categoryIDs []uuid.UUID) error + // UpsertContact creates the contact or enriches the one already holding + // its email (the same write the contacts API does), owned by actorID. + UpsertContact(ctx context.Context, orgID, actorID uuid.UUID, in models.AddContact) (*models.Contact, error) + // AddToCampaign enrols an existing contact in a campaign and wakes it. + AddToCampaign(ctx context.Context, orgID, actorID, contactID, campaignID uuid.UUID) error + // ListCategories / CreateCategory / ListPipelines back the AI agent step's // argument-based tools: the model picks a tag/label/pipeline by name and the // executor resolves it live (empty pool = any of the owner's tags). Keyed by @@ -205,6 +217,99 @@ type nativeActionConfig struct { // gateway. EventName + each field value are Go-templated against the event data. EventName string `json:"event_name"` EventFields []setVar `json:"event_fields"` + // upsert_contact: templated contact fields, the tags and campaign the + // contact lands in, and what to do when the email already exists. + Email string `json:"email"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Company string `json:"company"` + Phone string `json:"phone"` + CustomFields []setVar `json:"custom_fields"` + CategoryIDs []string `json:"category_ids"` + SegmentIDs []string `json:"segment_ids"` + IfExists string `json:"if_exists"` + // campaign_id is shared by upsert_contact (optional) and add_to_campaign + // (required). + CampaignID string `json:"campaign_id"` +} + +// Values of nativeActionConfig.IfExists for upsert_contact. +const ( + upsertIfExistsUpdate = "update" + upsertIfExistsSkip = "skip" +) + +// validateUpsertContactConfig checks the lead-intake action at write time: an +// email template is the one thing it cannot do without, and every id it +// references must at least parse. +func validateUpsertContactConfig(cfg nativeActionConfig) error { + if strings.TrimSpace(cfg.Email) == "" { + return fmt.Errorf("a create-or-update-contact action needs an email") + } + switch strings.TrimSpace(cfg.IfExists) { + case "", upsertIfExistsUpdate, upsertIfExistsSkip: + default: + return fmt.Errorf("unknown if_exists value %q", cfg.IfExists) + } + if id := strings.TrimSpace(cfg.CampaignID); id != "" { + if _, err := uuid.Parse(id); err != nil { + return fmt.Errorf("the campaign id is not valid") + } + } + for _, raw := range append(append([]string{}, cfg.CategoryIDs...), cfg.SegmentIDs...) { + if strings.TrimSpace(raw) == "" { + continue + } + if _, err := uuid.Parse(strings.TrimSpace(raw)); err != nil { + return fmt.Errorf("a tag or segment id is not valid") + } + } + return nil +} + +// buildUpsertContact renders the action's templates against the event data +// into the contact to write. A blank rendered field is left empty so the +// upsert's enrich-never-erase rule keeps whatever the contact already has. +func buildUpsertContact(a models.Automation, cfg nativeActionConfig, data map[string]any) (models.AddContact, error) { + email := strings.ToLower(strings.TrimSpace(renderTemplate(cfg.Email, data))) + if email == "" || !strings.Contains(email, "@") { + return models.AddContact{}, fmt.Errorf("the email rendered empty or invalid (%q)", truncate(email, 80)) + } + custom := map[string]string{} + for _, f := range cfg.CustomFields { + key := strings.TrimSpace(f.Key) + if key == "" { + continue + } + if v := strings.TrimSpace(renderTemplate(f.Value, data)); v != "" { + custom[key] = v + } + } + in := models.AddContact{ + Email: email, + FirstName: strings.TrimSpace(renderTemplate(cfg.FirstName, data)), + LastName: strings.TrimSpace(renderTemplate(cfg.LastName, data)), + Company: strings.TrimSpace(renderTemplate(cfg.Company, data)), + Phone: strings.TrimSpace(renderTemplate(cfg.Phone, data)), + CustomFields: custom, + Categories: uuidStrings(cfg.CategoryIDs), + Segments: uuidStrings(cfg.SegmentIDs), + Source: models.ContactSourceAutomation, + SourceDetail: a.Name, + } + if id := strings.TrimSpace(cfg.CampaignID); id != "" { + in.Campaigns = []string{id} + } + return in, nil +} + +// uuidStrings keeps the entries of a saved id list that parse, trimmed. +func uuidStrings(ids []string) []string { + out := make([]string, 0, len(ids)) + for _, id := range parseUUIDList(ids) { + out = append(out, id.String()) + } + return out } type setVar struct { @@ -295,12 +400,63 @@ func (s *service) execNativeAction(ctx context.Context, a models.Automation, n m contactID := stringFromMap(data, "contact_id") email := stringFromMap(data, "contact_email", "invitee_email", "email") + // upsert_contact is the one action that may run with no contact yet: it + // makes one from the event. The written contact becomes the event's + // contact so the nodes after it (tag, task, deal) find it. + if n.Action == models.IntegrationActionUpsertContact { + in, berr := buildUpsertContact(a, cfg, data) + if berr != nil { + return berr + } + if strings.TrimSpace(cfg.IfExists) == upsertIfExistsSkip { + existing, rerr := s.native.ResolveContact(ctx, a.OrganizationID, "", in.Email) + if rerr != nil { + return fmt.Errorf("look up the existing contact: %w", rerr) + } + if existing != nil { + data["contact_id"] = existing.ID.String() + data["contact_email"] = existing.Email + data["contact_created"] = false + return nil + } + } + owner, oerr := s.native.OrgOwner(ctx, a.OrganizationID) + if oerr != nil { + return oerr + } + written, werr := s.native.UpsertContact(ctx, a.OrganizationID, owner, in) + if werr != nil { + return werr + } + if written == nil { + return fmt.Errorf("the contact write returned nothing") + } + data["contact_id"] = written.ID.String() + data["contact_email"] = written.Email + data["contact_created"] = written.IsNew + return nil + } + c, err := s.native.ResolveContact(ctx, a.OrganizationID, contactID, email) - if err != nil || c == nil { + if err != nil { + return fmt.Errorf("look up the event's contact: %w", err) + } + if c == nil { return fmt.Errorf("no contact matched the event (need contact_id or contact_email)") } switch n.Action { + case models.IntegrationActionAddToCampaign: + campID, perr := uuid.Parse(strings.TrimSpace(cfg.CampaignID)) + if perr != nil { + return fmt.Errorf("an add-to-campaign action needs a campaign") + } + owner, oerr := s.native.OrgOwner(ctx, a.OrganizationID) + if oerr != nil { + return oerr + } + return s.native.AddToCampaign(ctx, a.OrganizationID, owner, c.ID, campID) + case models.IntegrationActionUnsubscribe: campID, perr := uuid.Parse(stringFromMap(data, "campaign_id")) if perr != nil { diff --git a/internal/app/integration/native_actions_test.go b/internal/app/integration/native_actions_test.go new file mode 100644 index 00000000..ecc939a8 --- /dev/null +++ b/internal/app/integration/native_actions_test.go @@ -0,0 +1,126 @@ +package integration + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +func TestValidateNativeActionConfig_UpsertContact(t *testing.T) { + campaign := uuid.New().String() + cases := []struct { + name string + cfg string + wantErr bool + }{ + {"needs an email", `{}`, true}, + {"email alone is enough", `{"email":"{{.email}}"}`, false}, + {"unknown if_exists", `{"email":"{{.email}}","if_exists":"merge"}`, true}, + {"skip is allowed", `{"email":"{{.email}}","if_exists":"skip"}`, false}, + {"bad campaign id", `{"email":"{{.email}}","campaign_id":"nope"}`, true}, + {"good campaign id", `{"email":"{{.email}}","campaign_id":"` + campaign + `"}`, false}, + {"bad tag id", `{"email":"{{.email}}","category_ids":["x"]}`, true}, + {"blank tag id is ignored", `{"email":"{{.email}}","category_ids":[""]}`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateNativeActionConfig(models.IntegrationActionUpsertContact, json.RawMessage(tc.cfg)) + if (err != nil) != tc.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} + +func TestValidateNativeActionConfig_AddToCampaign(t *testing.T) { + if err := validateNativeActionConfig(models.IntegrationActionAddToCampaign, json.RawMessage(`{}`)); err == nil { + t.Fatal("expected an error without a campaign") + } + if err := validateNativeActionConfig(models.IntegrationActionAddToCampaign, json.RawMessage(`{"campaign_id":"`+uuid.New().String()+`"}`)); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestBuildUpsertContact_RendersTemplatesAgainstEvent(t *testing.T) { + campaign := uuid.New().String() + tag := uuid.New().String() + cfg := parseNativeConfig(json.RawMessage(`{ + "email": "{{.data.email}}", + "first_name": "{{.data.first_name}}", + "company": "{{.data.company}}", + "custom_fields": [{"key":"team_size","value":"{{.data.team_size}}"},{"key":"empty","value":"{{.data.missing}}"},{"key":"","value":"x"}], + "category_ids": ["` + tag + `", "junk"], + "campaign_id": "` + campaign + `" + }`)) + data := map[string]any{ + "data": map[string]any{"email": " Jane@Example.com ", "first_name": "Jane", "company": "Example Inc", "team_size": "10-50"}, + } + in, err := buildUpsertContact(models.Automation{Name: "Lead ads intake"}, cfg, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if in.Email != "jane@example.com" { + t.Fatalf("email = %q", in.Email) + } + if in.FirstName != "Jane" || in.Company != "Example Inc" || in.LastName != "" { + t.Fatalf("fields = %+v", in) + } + if in.CustomFields["team_size"] != "10-50" { + t.Fatalf("custom fields = %v", in.CustomFields) + } + if _, ok := in.CustomFields["empty"]; ok { + t.Fatal("a blank rendered custom field must be dropped, not written as empty") + } + if len(in.Categories) != 1 || in.Categories[0] != tag { + t.Fatalf("categories = %v", in.Categories) + } + if len(in.Campaigns) != 1 || in.Campaigns[0] != campaign { + t.Fatalf("campaigns = %v", in.Campaigns) + } + if in.Source != models.ContactSourceAutomation || in.SourceDetail != "Lead ads intake" { + t.Fatalf("source = %s / %s", in.Source, in.SourceDetail) + } +} + +func TestBuildUpsertContact_RejectsEmptyOrInvalidEmail(t *testing.T) { + cfg := parseNativeConfig(json.RawMessage(`{"email":"{{.email}}"}`)) + if _, err := buildUpsertContact(models.Automation{}, cfg, map[string]any{}); err == nil { + t.Fatal("expected an error when the email renders empty") + } + if _, err := buildUpsertContact(models.Automation{}, cfg, map[string]any{"email": "not-an-email"}); err == nil { + t.Fatal("expected an error for an address without @") + } +} + +func TestSampleEventData_NewTriggersCarryTheirFields(t *testing.T) { + created := sampleEventData("contact.created") + if created["source"] != "form" || created["contact_email"] == "" { + t.Fatalf("contact.created sample = %v", created) + } + if _, ok := created["campaign_id"]; ok { + t.Fatal("a new contact carries no single campaign_id") + } + form := sampleEventData("form.submitted") + if form["form_name"] == "" { + t.Fatalf("form.submitted sample = %v", form) + } + answers, ok := form["data"].(map[string]any) + if !ok || answers["email"] == "" { + t.Fatalf("form answers = %v", form["data"]) + } +} + +func TestActionPreview_UpsertContactRendersFields(t *testing.T) { + n := models.AutomationNode{ + Type: models.AutomationNodeAction, + Action: models.IntegrationActionUpsertContact, + Config: json.RawMessage(`{"email":"{{.email}}","company":"{{.company}}","custom_fields":[{"key":"plan","value":"{{.plan}}"}]}`), + } + p := actionPreview(n, map[string]any{"email": "a@b.co", "company": "Acme", "plan": "pro"}) + if p["email"] != "a@b.co" || p["company"] != "Acme" || p["custom:plan"] != "pro" { + t.Fatalf("preview = %v", p) + } +} diff --git a/internal/app/nativeactions/adapter.go b/internal/app/nativeactions/adapter.go index 55368e92..605238ce 100644 --- a/internal/app/nativeactions/adapter.go +++ b/internal/app/nativeactions/adapter.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/app/contact" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -21,20 +22,34 @@ type Adapter struct { Adv advanced.Service Contacts repository.ContactRepository Orgs repository.OrganizationRepository + // ContactSvc is the contact service behind the lead-intake actions: its + // upsert runs the plan check, wakes campaigns and fires contact.created, + // which a bare repository write would not. + ContactSvc contact.ContactService } func (a Adapter) ResolveContact(ctx context.Context, orgID uuid.UUID, contactID, email string) (*models.Contact, error) { // Both lookups are ORG-SCOPED — never resolve a contact id from another org, - // even if a stale/crafted id reaches the event data. + // even if a stale/crafted id reaches the event data. A failed lookup is an + // error, not a miss: "skip if it exists" must not write over a contact it + // could not see. if contactID != "" { if id, perr := uuid.Parse(contactID); perr == nil { - if cs, e := a.Contacts.GetByIDsAndOrganization(ctx, orgID, []uuid.UUID{id}); e == nil && len(cs) > 0 { + cs, e := a.Contacts.GetByIDsAndOrganization(ctx, orgID, []uuid.UUID{id}) + if e != nil { + return nil, e + } + if len(cs) > 0 { return &cs[0], nil } } } if email != "" { - if c, e := a.Contacts.GetByEmailAndOrganization(ctx, orgID, email); e == nil && c != nil { + c, e := a.Contacts.GetByEmailAndOrganization(ctx, orgID, email) + if e != nil { + return nil, e + } + if c != nil { return c, nil } } @@ -98,6 +113,37 @@ func (a Adapter) Unsubscribe(ctx context.Context, campaignID, contactID uuid.UUI return nil } +// UpsertContact writes one contact through the contact service (upsert by +// email, tags, campaign and segment links, contact.created for a new row). +func (a Adapter) UpsertContact(ctx context.Context, orgID, actorID uuid.UUID, in models.AddContact) (*models.Contact, error) { + if a.ContactSvc == nil { + return nil, fmt.Errorf("contact writes are not available") + } + created, xerr := a.ContactSvc.Add(ctx, actorID.String(), orgID, []models.AddContact{in}) + if xerr != nil { + return nil, xerr + } + if len(created) == 0 { + return nil, fmt.Errorf("the contact write returned nothing") + } + return &created[0], nil +} + +// AddToCampaign enrols an existing contact in a campaign through the bulk +// edit path, which also wakes the campaign's parked send chain. +func (a Adapter) AddToCampaign(ctx context.Context, orgID, actorID, contactID, campaignID uuid.UUID) error { + if a.ContactSvc == nil { + return fmt.Errorf("contact writes are not available") + } + if _, xerr := a.ContactSvc.BulkUpdate(ctx, actorID.String(), orgID, &models.BulkEditContactsData{ + Contacts: []string{contactID.String()}, + AddCampaigns: []string{campaignID.String()}, + }); xerr != nil { + return xerr + } + return nil +} + // LabelThread applies unibox conversation labels to a thread on behalf of the // mailbox owner (the advanced service guards category ownership). The error is // already a plain error, so it passes straight through. diff --git a/internal/app/opsnotify/events.go b/internal/app/opsnotify/events.go new file mode 100644 index 00000000..846954b3 --- /dev/null +++ b/internal/app/opsnotify/events.go @@ -0,0 +1,136 @@ +// Package opsnotify delivers instance-wide operator alerts to the channels an +// admin configured in the settings document (Discord, Slack, a generic signed +// webhook, or email). +// +// This is the operator's channel, not the customer's. Customer-facing event +// delivery is internal/app/webhook, which is organization-scoped, queued and +// retried. Operator alerts are best-effort and fire-and-forget by design: an +// unreachable chat server must never fail or slow the request that triggered +// it, and a missed alert is not a correctness bug. +package opsnotify + +// Event keys. Adding one means adding it to Catalog too, otherwise the admin +// panel cannot subscribe a channel to it. +const ( + EventEnterpriseInquiry = "enterprise_inquiry.created" + EventLimitRequest = "limit_request.created" + EventWarmupAppeal = "warmup_appeal.created" + EventOrganizationNew = "organization.created" + EventUserRegistered = "user.registered" + EventWorkerOffline = "worker.offline" + EventOrgRisk = "org_risk.escalated" + EventSubscriptionIssue = "subscription.payment_failed" + EventTest = "test" +) + +// Severity drives the colour a chat transport renders. +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityWarning Severity = "warning" + SeverityUrgent Severity = "urgent" +) + +// EventDef describes one subscribable event for the admin panel. +type EventDef struct { + Key string `json:"key"` + Label string `json:"label"` + Description string `json:"description"` + Group string `json:"group"` + Severity Severity `json:"severity"` + // SelfHostRelevant is false for events that only mean something on a + // commercial deployment, so a self-hosted panel can hide them. + SelfHostRelevant bool `json:"self_host_relevant"` +} + +// Catalog is the inventory the admin panel renders. Declaration order is +// display order. +var Catalog = []EventDef{ + { + Key: EventEnterpriseInquiry, Group: "Sales", + Label: "Enterprise inquiry submitted", + Description: "Someone asked for enterprise pricing from the plan chooser.", + Severity: SeverityInfo, + }, + { + Key: EventLimitRequest, Group: "Sales", + Label: "Limit increase requested", + Description: "A workspace asked for more capacity than its plan allows.", + Severity: SeverityInfo, SelfHostRelevant: true, + }, + { + Key: EventSubscriptionIssue, Group: "Sales", + Label: "Payment failed", + Description: "A subscription went past due and sending is at risk.", + Severity: SeverityWarning, + }, + { + Key: EventOrganizationNew, Group: "Growth", + Label: "Workspace created", + Description: "A new organization was created on this instance.", + Severity: SeverityInfo, SelfHostRelevant: true, + }, + { + Key: EventUserRegistered, Group: "Growth", + Label: "User registered", + Description: "A new account finished signing up.", + Severity: SeverityInfo, SelfHostRelevant: true, + }, + { + Key: EventWorkerOffline, Group: "Infrastructure", + Label: "Worker went offline", + Description: "A worker stopped heartbeating, so its mailboxes cannot send until it returns or they are reassigned.", + Severity: SeverityUrgent, SelfHostRelevant: true, + }, + { + Key: EventWarmupAppeal, Group: "Abuse", + Label: "Warmup ban appealed", + Description: "A blocked mailbox asked to be let back into the warmup pool.", + Severity: SeverityInfo, SelfHostRelevant: true, + }, + { + Key: EventOrgRisk, Group: "Abuse", + Label: "Workspace risk escalated", + Description: "Risk scoring moved a workspace into a stricter posture.", + Severity: SeverityWarning, SelfHostRelevant: true, + }, +} + +// Def returns the catalog entry for a key. +func Def(key string) (EventDef, bool) { + for _, d := range Catalog { + if d.Key == key { + return d, true + } + } + return EventDef{}, false +} + +// Event is one alert on its way out. +type Event struct { + Key string + Title string + Summary string + Severity Severity + // Fields render as a small key/value table on chat transports and as the + // payload body on the generic webhook. + Fields []Field + // Link is an absolute URL an operator can click, usually into the admin panel. + Link string +} + +// Field is one labelled value on an event. +type Field struct { + Label string `json:"label"` + Value string `json:"value"` +} + +// NewEvent builds an event, defaulting its severity from the catalog. +func NewEvent(key, title, summary string, fields ...Field) Event { + sev := SeverityInfo + if d, ok := Def(key); ok { + sev = d.Severity + } + return Event{Key: key, Title: title, Summary: summary, Severity: sev, Fields: fields} +} diff --git a/internal/app/opsnotify/format.go b/internal/app/opsnotify/format.go new file mode 100644 index 00000000..7fd634fc --- /dev/null +++ b/internal/app/opsnotify/format.go @@ -0,0 +1,131 @@ +package opsnotify + +import ( + "fmt" + "html" + "strings" +) + +// Chat transports colour by severity. Discord takes a decimal integer, Slack +// takes a hex string on the attachment. +func severityColorInt(s Severity) int { + switch s { + case SeverityUrgent: + return 0xDC2626 // red-600 + case SeverityWarning: + return 0xD97706 // amber-600 + default: + return 0x0284C7 // sky-600 + } +} + +func severityColorHex(s Severity) string { + return fmt.Sprintf("#%06X", severityColorInt(s)) +} + +// discordPayload builds a single embed. Discord caps a field value at 1024 +// characters and an embed at 25 fields; both are enforced here so a long +// value cannot make the whole POST fail. +func discordPayload(e Event) map[string]any { + fields := make([]map[string]any, 0, len(e.Fields)) + for i, f := range e.Fields { + if i >= 25 { + break + } + fields = append(fields, map[string]any{ + "name": truncate(f.Label, 256), + "value": truncate(emptyDash(f.Value), 1024), + "inline": len(f.Value) <= 40, + }) + } + embed := map[string]any{ + "title": truncate(e.Title, 256), + "description": truncate(e.Summary, 4096), + "color": severityColorInt(e.Severity), + "footer": map[string]any{"text": "Warmbly"}, + } + if len(fields) > 0 { + embed["fields"] = fields + } + if e.Link != "" { + embed["url"] = e.Link + } + return map[string]any{"embeds": []any{embed}} +} + +// slackPayload uses an attachment so the severity colour shows as the bar on +// the left. The `text` fallback is what a notification preview renders. +func slackPayload(e Event) map[string]any { + var sb strings.Builder + sb.WriteString("*" + e.Title + "*") + if e.Summary != "" { + sb.WriteString("\n" + e.Summary) + } + for _, f := range e.Fields { + sb.WriteString("\n• *" + f.Label + ":* " + emptyDash(f.Value)) + } + if e.Link != "" { + sb.WriteString("\n<" + e.Link + "|Open in the admin panel>") + } + return map[string]any{ + "text": e.Title, + "attachments": []any{ + map[string]any{ + "color": severityColorHex(e.Severity), + "text": sb.String(), + "mrkdwn_in": []string{"text"}, + }, + }, + } +} + +func emailSubject(e Event) string { + prefix := "[Warmbly]" + if e.Severity == SeverityUrgent { + prefix = "[Warmbly] Urgent:" + } + return prefix + " " + e.Title +} + +// emailBodyHTML renders the alert. The transport derives the plain-text part +// from this, so it stays simple and table-free. +func emailBodyHTML(e Event) string { + var sb strings.Builder + sb.WriteString(`
`) + sb.WriteString(`

` + esc(e.Title) + `

`) + if e.Summary != "" { + sb.WriteString(`

` + esc(e.Summary) + `

`) + } + if len(e.Fields) > 0 { + sb.WriteString(`
    `) + for _, f := range e.Fields { + sb.WriteString(`
  • ` + esc(f.Label) + `: ` + esc(emptyDash(f.Value)) + `
    • `) + } + sb.WriteString(`
`) + } + if e.Link != "" { + sb.WriteString(`

Open in the admin panel

`) + } + sb.WriteString(`

You are receiving this because this address is an operator notification channel on your Warmbly instance.

`) + sb.WriteString(`
`) + return sb.String() +} + +func esc(v string) string { return html.EscapeString(v) } + +func emptyDash(v string) string { + if strings.TrimSpace(v) == "" { + return "—" + } + return v +} + +func truncate(v string, max int) string { + if len(v) <= max { + return v + } + if max <= 1 { + return v[:max] + } + return v[:max-1] + "…" +} diff --git a/internal/app/opsnotify/service.go b/internal/app/opsnotify/service.go new file mode 100644 index 00000000..0cdc7258 --- /dev/null +++ b/internal/app/opsnotify/service.go @@ -0,0 +1,240 @@ +package opsnotify + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/app/instancesettings" + "github.com/warmbly/warmbly/internal/app/webhook" + "github.com/warmbly/warmbly/internal/pkg/safehttp" +) + +// deliveryTimeout bounds one outbound call. Chat webhooks answer in +// milliseconds; anything slower is not worth holding a goroutine for. +const deliveryTimeout = 8 * time.Second + +// maxInFlight bounds concurrent deliveries. Notify is called from request +// paths that are open to the internet (signup emits user.registered), and each +// delivery can hold a goroutine for the full timeout against a slow endpoint. +// A bounded pool means a hostile or dead chat server costs a fixed amount of +// this process instead of one goroutine per event. +const maxInFlight = 8 + +// Settings is the slice of the settings service this package needs. +type Settings interface { + Get(ctx context.Context) instancesettings.Document +} + +// Mailer is the narrow slice of notify.EmailNotificationService this package +// needs, declared locally so opsnotify does not depend on the mail package. +// `message` is HTML; the transports derive the text part themselves. +// +// Operator alerts deliberately bypass internal/app/notification: that path is +// per-user, tenant-scoped, digest-coalesced and daily-capped, so an alert sent +// through it could be delayed or silently dropped. +type Mailer interface { + Send(ctx context.Context, to, cc, bcc []string, subject, message string) error +} + +// Notifier is the emit surface. Call sites depend on this, never on the +// concrete service, so a nil notifier is a no-op rather than a panic. +type Notifier interface { + // Notify delivers to every subscribed channel. It never blocks the caller + // and never returns an error: an operator alert must not be able to fail + // the request that produced it. + Notify(event Event) + // NotifyOperator is the plain-string form every emit site uses, so a + // package can declare a one-method local interface for it and stay free of + // any dependency on this one. + NotifyOperator(key, title, summary string, fields map[string]string) + // Deliver sends one event to one channel and reports the outcome. The + // admin panel's "send test" uses it; nothing else should. + Deliver(ctx context.Context, ch instancesettings.NotifyChannel, event Event) error +} + +type service struct { + settings Settings + mailer Mailer + client *http.Client + // baseURL is the admin panel origin, used to build the Link on events. + baseURL string + // slots bounds concurrent deliveries; an event that cannot claim one is + // dropped rather than queued. Operator alerts are best effort, and a + // backlog that outlives the incident is worse than a missed line. + slots chan struct{} + // dropped counts events shed under load, so the condition is observable + // instead of silent. + dropped atomic.Uint64 +} + +// NewService builds the notifier. A nil settings service disables delivery. +func NewService(settings Settings, mailer Mailer, baseURL string) Notifier { + return &service{ + settings: settings, + mailer: mailer, + client: safehttp.Client(deliveryTimeout), + baseURL: strings.TrimRight(baseURL, "/"), + slots: make(chan struct{}, maxInFlight), + } +} + +// Nop is the notifier used where none is configured. +type Nop struct{} + +func (Nop) Notify(Event) {} +func (Nop) Deliver(context.Context, instancesettings.NotifyChannel, Event) error { + return fmt.Errorf("notifications are not configured on this deployment") +} + +func (s *service) Notify(event Event) { + if s == nil || s.settings == nil { + return + } + // Claim a slot before detaching. Non-blocking: the caller is on a request + // path and must never wait on a chat server. + select { + case s.slots <- struct{}{}: + default: + if n := s.dropped.Add(1); n == 1 || n%100 == 0 { + log.Warn().Uint64("dropped", n).Str("event", event.Key). + Msg("operator notification dropped: delivery slots are full") + } + return + } + + // Detached context: the caller's request may finish (or be cancelled) + // before a chat server answers, and that must not drop the alert. + go func() { + defer func() { <-s.slots }() + ctx, cancel := context.WithTimeout(context.Background(), deliveryTimeout*2) + defer cancel() + + subs := s.settings.Get(ctx).Notifications.Subscribers(event.Key) + for _, ch := range subs { + // Sequential: the list is bounded at MaxChannels and this runs off + // the request path, so there is nothing to gain from more goroutines. + _ = s.Deliver(ctx, ch, event) + } + }() +} + +func (s *service) Deliver(ctx context.Context, ch instancesettings.NotifyChannel, event Event) error { + switch ch.Type { + case instancesettings.ChannelDiscord: + return s.post(ctx, ch, discordPayload(event), nil) + case instancesettings.ChannelSlack: + return s.post(ctx, ch, slackPayload(event), nil) + case instancesettings.ChannelWebhook: + return s.postSigned(ctx, ch, event) + case instancesettings.ChannelEmail: + if s.mailer == nil { + return fmt.Errorf("no mail transport is configured on this deployment") + } + return s.mailer.Send(ctx, []string{ch.Target}, nil, nil, emailSubject(event), emailBodyHTML(event)) + default: + return fmt.Errorf("unknown channel type %q", ch.Type) + } +} + +func (s *service) post(ctx context.Context, ch instancesettings.NotifyChannel, payload any, headers map[string]string) error { + body, err := json.Marshal(payload) + if err != nil { + return err + } + return s.send(ctx, ch.Target, body, headers) +} + +// postSigned is the generic webhook: the event as JSON, signed with the same +// HMAC scheme customer webhooks use so an existing verifier works unchanged. +func (s *service) postSigned(ctx context.Context, ch instancesettings.NotifyChannel, event Event) error { + payload := map[string]any{ + "event": event.Key, + "title": event.Title, + "summary": event.Summary, + "severity": string(event.Severity), + "fields": event.Fields, + "link": event.Link, + "timestamp": time.Now().UTC().Format(time.RFC3339), + } + body, err := json.Marshal(payload) + if err != nil { + return err + } + headers := map[string]string{} + if ch.Secret != "" { + now := time.Now() + headers["X-Warmbly-Signature"] = webhook.FormatSignatureHeader(now, webhook.Sign(ch.Secret, now, body)) + } + headers["X-Warmbly-Event"] = event.Key + return s.send(ctx, ch.Target, body, headers) +} + +func (s *service) send(ctx context.Context, url string, body []byte, headers map[string]string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "Warmbly-Ops-Notifier/1") + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + // Read and discard so the connection can be reused; cap it so a hostile + // endpoint cannot stream us an unbounded body. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("endpoint returned %d", resp.StatusCode) + } + return nil +} + +// WithLink returns a copy of the event pointing at an admin panel path. +func (s *service) WithLink(event Event, path string) Event { + if s.baseURL != "" && path != "" { + event.Link = s.baseURL + path + } + return event +} + +// NotifyOperator is the emit surface every call site uses. It takes plain +// strings rather than an Event so a package can declare a one-method local +// interface and stay free of any dependency on this one. +// +// Fields are rendered in sorted key order: a map has no order, and an alert +// whose lines move between deliveries is hard to read. +func (s *service) NotifyOperator(key, title, summary string, fields map[string]string) { + event := NewEvent(key, title, summary) + if len(fields) > 0 { + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if strings.TrimSpace(fields[k]) == "" { + continue + } + event.Fields = append(event.Fields, Field{Label: k, Value: fields[k]}) + } + } + s.Notify(event) +} + +// NotifyOperator on the no-op notifier discards the alert. +func (Nop) NotifyOperator(string, string, string, map[string]string) {} diff --git a/internal/app/organization/service.go b/internal/app/organization/service.go index 2272ce3d..7a14b3c7 100644 --- a/internal/app/organization/service.go +++ b/internal/app/organization/service.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "strconv" "strings" "time" @@ -34,11 +35,21 @@ type InstanceSettings interface { InviteLinksEnabled(ctx context.Context) bool } +// OperatorNotifier is the instance-wide operator alert surface, injected +// post-construction so this package needs no import of it. Nil disables every +// alert below; a deployment with no channels configured is the normal case. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + // OrganizationService defines the interface for organization management type OrganizationService interface { // WireAuthPolicy attaches the deployment auth policy after construction. WireAuthPolicy(p *config.AuthPolicy) + // WireOperatorNotifier attaches the operator alert channel. + WireOperatorNotifier(n OperatorNotifier) + // WireInstanceSettings attaches the database-backed instance settings // (post-construction; nil keeps the compiled defaults). WireInstanceSettings(s InstanceSettings) @@ -86,7 +97,9 @@ type OrganizationService interface { // Limit checks CanAddMember(ctx context.Context, orgID uuid.UUID) (bool, *errx.Error) CanAddCampaign(ctx context.Context, orgID uuid.UUID) (bool, *errx.Error) - CanAddEmailAccount(ctx context.Context, orgID uuid.UUID) (bool, *errx.Error) + // MailboxAllowance resolves how many mailboxes the workspace may hold and + // why; every connect path checks it and GET /emails/allowance returns it. + MailboxAllowance(ctx context.Context, orgID uuid.UUID) (*models.MailboxAllowance, *errx.Error) GetCampaignCounts(ctx context.Context, orgID uuid.UUID) (total int, active int, err *errx.Error) GetOrganizationLimits(ctx context.Context, orgID uuid.UUID) (*models.OrganizationLimits, *errx.Error) GetOrganizationCounts(ctx context.Context, orgID uuid.UUID) (*models.OrganizationCounts, *errx.Error) @@ -145,6 +158,21 @@ type organizationService struct { // settings is the operator-editable invite configuration, wired after // construction because it needs the database pool. settings InstanceSettings + // opsNotify raises instance-wide operator alerts. Nil is the default. + opsNotify OperatorNotifier +} + +// WireOperatorNotifier attaches the operator alert channel. +func (s *organizationService) WireOperatorNotifier(n OperatorNotifier) { + s.opsNotify = n +} + +// notifyOperator is the nil-safe emit helper. +func (s *organizationService) notifyOperator(key, title, summary string, fields map[string]string) { + if s.opsNotify == nil { + return + } + s.opsNotify.NotifyOperator(key, title, summary, fields) } // WireAuthPolicy attaches the deployment auth policy, so invitations answer to @@ -286,6 +314,16 @@ func (s *organizationService) Create(ctx context.Context, userID uuid.UUID, name } } + s.notifyOperator( + "organization.created", + "New workspace: "+org.Name, + "A new organization was created on this instance.", + map[string]string{ + "Workspace": org.Name, + "Owner": user.Email, + }, + ) + return org, nil } @@ -871,25 +909,90 @@ func (s *organizationService) CanAddCampaign(ctx context.Context, orgID uuid.UUI return true, nil } -// CanAddEmailAccount checks if the organization can add more email accounts based on plan limits -func (s *organizationService) CanAddEmailAccount(ctx context.Context, orgID uuid.UUID) (bool, *errx.Error) { - limits, err := s.GetEffectiveLimits(ctx, orgID) +// MailboxAllowance resolves the workspace's mailbox allowance. Resolution: +// +// 1. no billing provider: unlimited +// 2. an operator override: the override +// 3. no paid subscription: FreeWorkspaceMailboxLimit +// 4. the plan's explicit mailbox column, when it carries one +// 5. the plan's daily sends divided by FairUseSendsPerMailbox +// 6. a plan with no daily send cap: unlimited +// +// The count includes every connected mailbox, so a workspace that dropped to +// a smaller plan simply cannot add until it is back under; nothing is removed. +func (s *organizationService) MailboxAllowance(ctx context.Context, orgID uuid.UUID) (*models.MailboxAllowance, *errx.Error) { + count, err := s.orgRepo.GetEmailAccountCount(ctx, orgID) if err != nil { - return false, err + sentry.CaptureException(err) + return nil, errx.New(errx.Internal, "failed to get email account count") + } + a := &models.MailboxAllowance{Used: count, SendsPerMailbox: config.FairUseSendsPerMailbox} + + if config.BillingProvider() == "none" { + a.Basis = models.MailboxAllowanceUnlimited + a.Paid = true + return a, nil } - // No limit set = unlimited - if limits == nil || limits.MaxEmailAccounts == nil { - return true, nil + sub, serr := s.subRepo.GetByOrganizationID(ctx, orgID) + if serr != nil { + sentry.CaptureException(serr) + return nil, errx.New(errx.Internal, "failed to get subscription") + } + a.Paid = sub != nil && sub.HasPaidSubscription() + if sub != nil && sub.Plan != nil { + if sub.Plan.Name != nil { + a.PlanName = *sub.Plan.Name + } + if sub.Plan.DailyCampaignLimit != nil && *sub.Plan.DailyCampaignLimit > 0 { + v := *sub.Plan.DailyCampaignLimit + a.PlanDailySends = &v + } } - count, xerr := s.orgRepo.GetEmailAccountCount(ctx, orgID) + override, xerr := s.GetLimitOverrides(ctx, orgID) if xerr != nil { - sentry.CaptureException(xerr) - return false, errx.New(errx.Internal, "failed to get email account count") + return nil, xerr } - return count < *limits.MaxEmailAccounts, nil + set := func(v int, basis models.MailboxAllowanceBasis) { + a.Allowance = &v + rem := v - count + if rem < 0 { + rem = 0 + } + a.Remaining = &rem + a.Basis = basis + } + + switch { + case override != nil && override.MaxEmailAccounts > 0: + set(override.MaxEmailAccounts, models.MailboxAllowanceOverride) + case !a.Paid: + set(models.FreeWorkspaceMailboxLimit, models.MailboxAllowanceFree) + case sub.Plan != nil && sub.Plan.MaxEmailAccounts != nil && *sub.Plan.MaxEmailAccounts > 0: + set(*sub.Plan.MaxEmailAccounts, models.MailboxAllowancePlan) + case a.PlanDailySends != nil: + set((*a.PlanDailySends+config.FairUseSendsPerMailbox-1)/config.FairUseSendsPerMailbox, models.MailboxAllowanceFairUse) + default: + a.Basis = models.MailboxAllowanceUnlimited + } + + // The open request, so the dashboard can show "asked for 5,000, pending" + // instead of offering a form that would be refused as a duplicate. + if a.Allowance != nil { + rows, rerr := s.orgRepo.ListLimitRequestsForOrg(ctx, orgID) + if rerr != nil { + sentry.CaptureException(rerr) + } + for i := range rows { + if rows[i].Field == "max_email_accounts" && rows[i].Status == models.LimitRequestStatusPending { + a.PendingRequest = &rows[i] + break + } + } + } + return a, nil } // GetCampaignCounts returns total and active campaign counts @@ -971,6 +1074,18 @@ func (s *organizationService) CreateEnterpriseInquiry(ctx context.Context, inqui return nil, errx.New(errx.Internal, "failed to create enterprise inquiry") } + s.notifyOperator( + "enterprise_inquiry.created", + "Enterprise inquiry from "+inquiry.CompanyName, + "Someone asked for enterprise pricing.", + map[string]string{ + "Company": inquiry.CompanyName, + "Contact": inquiry.ContactName, + "Email": inquiry.ContactEmail, + "Notes": inquiry.Notes, + }, + ) + return inquiry, nil } @@ -1088,9 +1203,10 @@ func (s *organizationService) SetLimitOverrides(ctx context.Context, orgID uuid. // 2. plan != nil → use plan column // 3. otherwise → fall back to the product-level hard cap // -// Never returns nil values: even an "unlimited" plan is bounded by the -// product hard caps in config/constants.go. Admins can raise individual -// caps per-org by writing an override. +// Every field but mailboxes is never nil: an "unlimited" plan is bounded by +// the product hard caps in config/constants.go. Mailboxes follow +// MailboxAllowance instead, where nil really means unlimited. Admins can +// raise individual caps per-org by writing an override. func (s *organizationService) GetEffectiveLimits(ctx context.Context, orgID uuid.UUID) (*models.OrganizationLimits, *errx.Error) { plan, err := s.GetOrganizationLimits(ctx, orgID) if err != nil { @@ -1100,6 +1216,10 @@ func (s *organizationService) GetEffectiveLimits(ctx context.Context, orgID uuid if err != nil { return nil, err } + mailboxes, err := s.MailboxAllowance(ctx, orgID) + if err != nil { + return nil, err + } resolve := func(overrideVal int, planVal *int, hardCap int) *int { if overrideVal > 0 { @@ -1113,12 +1233,11 @@ func (s *organizationService) GetEffectiveLimits(ctx context.Context, orgID uuid return &v } - var ovMaxCampaigns, ovMaxActive, ovMaxMembers, ovMaxEmails, ovMaxContacts, ovDaily int + var ovMaxCampaigns, ovMaxActive, ovMaxMembers, ovMaxContacts, ovDaily int if override != nil { ovMaxCampaigns = override.MaxCampaigns ovMaxActive = override.MaxActiveCampaigns ovMaxMembers = override.MaxTeamMembers - ovMaxEmails = override.MaxEmailAccounts ovMaxContacts = override.MaxContacts ovDaily = override.DailyCampaignLimit } @@ -1132,7 +1251,7 @@ func (s *organizationService) GetEffectiveLimits(ctx context.Context, orgID uuid MaxCampaigns: resolve(ovMaxCampaigns, planLimits.MaxCampaigns, config.HardCapCampaignsTotal), MaxActiveCampaigns: resolve(ovMaxActive, planLimits.MaxActiveCampaigns, config.HardCapCampaignsActive), MaxTeamMembers: resolve(ovMaxMembers, planLimits.MaxTeamMembers, config.HardCapTeamMembers), - MaxEmailAccounts: resolve(ovMaxEmails, planLimits.MaxEmailAccounts, config.HardCapMailboxes), + MaxEmailAccounts: mailboxes.Allowance, MaxContacts: resolve(ovMaxContacts, planLimits.MaxContacts, config.HardCapContacts), DailyCampaignLimit: resolve(ovDaily, planLimits.DailyCampaignLimit, config.HardCapDailyCampaignSends), }, nil @@ -1151,10 +1270,12 @@ func (s *organizationService) WebhookDispatchLimit(ctx context.Context, orgID uu if err != nil || eff == nil { return limit } - if eff.MaxEmailAccounts != nil { - if scaled := *eff.MaxEmailAccounts * config.WebhookDispatchPerMailboxPerMinute; scaled > limit { - limit = scaled - } + if eff.MaxEmailAccounts == nil { + // Unlimited mailboxes: the ceiling is the only bound left. + return config.WebhookDispatchMaxPerMinute + } + if scaled := *eff.MaxEmailAccounts * config.WebhookDispatchPerMailboxPerMinute; scaled > limit { + limit = scaled } if limit > config.WebhookDispatchMaxPerMinute { limit = config.WebhookDispatchMaxPerMinute @@ -1225,6 +1346,9 @@ func (s *organizationService) SubmitLimitIncreaseRequest(ctx context.Context, or if xerr != nil { return nil, xerr } + if req.Field == "max_email_accounts" && effective.MaxEmailAccounts == nil { + return nil, errx.New(errx.BadRequest, "this workspace already holds unlimited mailboxes") + } current := limitFieldEffective(req.Field, effective) if req.Requested <= current { return nil, errx.New(errx.BadRequest, "requested value must exceed current effective limit") @@ -1253,6 +1377,20 @@ func (s *organizationService) SubmitLimitIncreaseRequest(ctx context.Context, or sentry.CaptureException(err) return nil, errx.New(errx.Internal, "failed to submit request") } + + s.notifyOperator( + "limit_request.created", + "Limit increase requested", + "A workspace asked for more capacity than its plan allows.", + map[string]string{ + "Workspace": orgID.String(), + "Field": req.Field, + "Current": strconv.Itoa(current), + "Requested": strconv.Itoa(req.Requested), + "Reason": req.Reason, + }, + ) + return lr, nil } diff --git a/internal/app/orgrisk/service.go b/internal/app/orgrisk/service.go index b9b06c1b..b8d04020 100644 --- a/internal/app/orgrisk/service.go +++ b/internal/app/orgrisk/service.go @@ -142,8 +142,9 @@ type Service interface { } type service struct { - repo repository.OrgRiskRepository - audit AuditLogger + repo repository.OrgRiskRepository + audit AuditLogger + opsNotify OperatorNotifier } // AuditLogger is the narrow slice of the audit service a transition needs. It @@ -162,6 +163,15 @@ func NewService(repo repository.OrgRiskRepository) Service { // every teammate's dashboard reflects it without a bespoke emit site. func (s *service) WireAudit(a AuditLogger) { s.audit = a } +// OperatorNotifier is the instance-wide operator alert surface, injected +// post-construction so this package needs no import of it. Nil disables it. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + +// WireOperatorNotifier attaches the operator alert channel. +func (s *service) WireOperatorNotifier(n OperatorNotifier) { s.opsNotify = n } + // AuditAware is the optional capability the caller uses to attach the logger. type AuditAware interface { WireAudit(a AuditLogger) @@ -187,6 +197,22 @@ func (s *service) auditTransition(ctx context.Context, orgID uuid.UUID, before, if before != nil { from = string(before.State) } + // The same "only a real change" guard already applied above, so this + // cannot re-fire while a detector keeps recording the same finding. + if s.opsNotify != nil { + s.opsNotify.NotifyOperator( + "org_risk.escalated", + "Workspace risk changed to "+string(after.State), + "Risk scoring moved a workspace into a different posture.", + map[string]string{ + "Workspace": orgID.String(), + "From": from, + "To": string(after.State), + "Score": strconv.Itoa(after.Score), + "Reason": after.Reason, + }, + ) + } s.audit.LogAction(ctx, orgID, uuid.Nil, models.AuditActionUpdate, models.AuditEntityOrgRisk, &orgID, "", "", map[string]string{"risk_state": from + " -> " + string(after.State)}, map[string]string{"reason": after.Reason, "score": strconv.Itoa(after.Score)}) diff --git a/internal/app/orgtransfer/spec.go b/internal/app/orgtransfer/spec.go index 4a5abff4..e6df3ba9 100644 --- a/internal/app/orgtransfer/spec.go +++ b/internal/app/orgtransfer/spec.go @@ -654,6 +654,20 @@ var Tables = []Table{ Scope: `campaign_id IN ` + orgCampaigns, Note: "Click tickets already in the wild keep resolving after the move, provided the tracking domain follows.", }, + { + // Campaign engagement like campaign_contact_progress, one row per + // link. Sits below tracked_links because of the nullable ticket + // reference, which the importer blanks when send history stays behind. + Name: "email_link_clicks", Group: models.OrgDataGroupCampaigns, + Scope: `campaign_id IN ` + orgCampaigns, + }, + { + // The per-event open log beside the click log: same keys, same + // scope, no ticket reference. task_id is an opaque id from the source + // instance, used only to group a step's opens. + Name: "email_opens", Group: models.OrgDataGroupCampaigns, + Scope: `campaign_id IN ` + orgCampaigns, + }, // ---------- delivery events ---------- { @@ -772,6 +786,7 @@ var ExcludedTables = map[string]string{ "dedicated_worker_assignments": "Worker topology, which is a property of the instance rather than the workspace.", "warmup_pools": "Instance-global pool definitions shared by every workspace on the instance.", "pool_link_codes": "In-flight link handshakes between a self-hosted instance and this cloud, valid for minutes.", + "cli_auth_codes": "In-flight `warmbly auth login` handshakes, valid for minutes. The API key an approval mints does travel, with the api_keys rows.", "pool_link_instances": "Self-hosted instances linked to this workspace's pool allowance. The token hash only authenticates against this instance, and the enrolled mailboxes are mirrors of mailboxes that live elsewhere.", "pool_link_mailboxes": "Which mailbox rows are warmup-only mirrors for a linked instance. They follow pool_link_instances, which does not travel.", "cloud_link": "This instance's own link to Warmbly Cloud: an instance property, not workspace data, and its token would be wrong on any other instance.", diff --git a/internal/app/releases/github.go b/internal/app/releases/github.go new file mode 100644 index 00000000..5511dfb3 --- /dev/null +++ b/internal/app/releases/github.go @@ -0,0 +1,89 @@ +package releases + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sort" + "time" +) + +// Release is one GitHub release as the update surfaces read it. +type Release struct { + TagName string `json:"tag_name"` + Name string `json:"name"` + Prerelease bool `json:"prerelease"` + Draft bool `json:"draft"` + PublishedAt time.Time `json:"published_at"` + HTMLURL string `json:"html_url"` +} + +// FetchReleases lists the newest releases of owner/repo. token is optional and +// only raises the unauthenticated rate limit. +func FetchReleases(ctx context.Context, client *http.Client, repo, token string) ([]Release, error) { + if repo == "" { + return nil, errors.New("RELEASES_GITHUB_REPO not set") + } + if client == nil { + client = &http.Client{Timeout: 15 * time.Second} + } + url := "https://api.github.com/repos/" + repo + "/releases?per_page=30" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("github %d: %s", resp.StatusCode, truncate(string(body), 200)) + } + var releases []Release + if err := json.Unmarshal(body, &releases); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return releases, nil +} + +// PickChannelHeads returns the most recent published release for each channel: +// - stable: latest non-prerelease, non-draft +// - dev: latest published (including prereleases) +func PickChannelHeads(releases []Release) (stable, dev *Release) { + sort.Slice(releases, func(i, j int) bool { + return releases[i].PublishedAt.After(releases[j].PublishedAt) + }) + for i := range releases { + r := &releases[i] + if r.Draft { + continue + } + if dev == nil { + dev = r + } + if !r.Prerelease && stable == nil { + stable = r + } + if dev != nil && stable != nil { + break + } + } + return +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/internal/app/releases/service.go b/internal/app/releases/service.go index 31108ba8..5837541b 100644 --- a/internal/app/releases/service.go +++ b/internal/app/releases/service.go @@ -20,10 +20,8 @@ import ( "encoding/json" "errors" "fmt" - "io" "log" "net/http" - "sort" "strings" "sync" "time" @@ -110,13 +108,13 @@ func (s *Service) CheckGitHub(ctx context.Context) (changed []ProfileUpdate, err return nil, errors.New("releases not enabled") } - releases, err := s.fetchReleases(ctx) + releases, err := FetchReleases(ctx, s.http, s.cfg.GithubRepo, s.cfg.GithubToken) if err != nil { s.recordError(err.Error()) return nil, err } - stable, dev := pickChannelHeads(releases) + stable, dev := PickChannelHeads(releases) now := time.Now() s.stateMu.Lock() @@ -132,7 +130,7 @@ func (s *Service) CheckGitHub(ctx context.Context) (changed []ProfileUpdate, err s.stateMu.Unlock() for _, channel := range []models.ReleaseChannel{models.ReleaseChannelStable, models.ReleaseChannelDev} { - var target *githubRelease + var target *Release switch channel { case models.ReleaseChannelStable: target = stable @@ -292,7 +290,7 @@ func (s *Service) imageFor(tag string) string { return repo + ":" + tag } -func (s *Service) channelView(name string, r *githubRelease) ChannelView { +func (s *Service) channelView(name string, r *Release) ChannelView { return ChannelView{ Channel: name, Tag: r.TagName, @@ -302,72 +300,6 @@ func (s *Service) channelView(name string, r *githubRelease) ChannelView { } } -// GitHub API - -type githubRelease struct { - TagName string `json:"tag_name"` - Name string `json:"name"` - Prerelease bool `json:"prerelease"` - Draft bool `json:"draft"` - PublishedAt time.Time `json:"published_at"` - HTMLURL string `json:"html_url"` -} - -func (s *Service) fetchReleases(ctx context.Context) ([]githubRelease, error) { - if s.cfg.GithubRepo == "" { - return nil, errors.New("RELEASES_GITHUB_REPO not set") - } - url := "https://api.github.com/repos/" + s.cfg.GithubRepo + "/releases?per_page=30" - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("X-GitHub-Api-Version", "2022-11-28") - if s.cfg.GithubToken != "" { - req.Header.Set("Authorization", "Bearer "+s.cfg.GithubToken) - } - resp, err := s.http.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - if resp.StatusCode != 200 { - return nil, fmt.Errorf("github %d: %s", resp.StatusCode, truncate(string(body), 200)) - } - var releases []githubRelease - if err := json.Unmarshal(body, &releases); err != nil { - return nil, fmt.Errorf("decode: %w", err) - } - return releases, nil -} - -// pickChannelHeads returns the most recent published release for each channel: -// - stable: latest non-prerelease, non-draft -// - dev: latest published (including prereleases) -func pickChannelHeads(releases []githubRelease) (stable, dev *githubRelease) { - sort.Slice(releases, func(i, j int) bool { - return releases[i].PublishedAt.After(releases[j].PublishedAt) - }) - for i := range releases { - r := &releases[i] - if r.Draft { - continue - } - if dev == nil { - dev = r - } - if !r.Prerelease && stable == nil { - stable = r - } - if dev != nil && stable != nil { - break - } - } - return -} - // HMAC verification func verifySignature(secret string, body []byte, header string) bool { @@ -382,10 +314,3 @@ func verifySignature(secret string, body []byte, header string) bool { mac.Write(body) return hmac.Equal(got, mac.Sum(nil)) } - -func truncate(s string, n int) string { - if len(s) <= n { - return s - } - return s[:n] + "..." -} diff --git a/internal/app/replyclassify/lexicon.go b/internal/app/replyclassify/lexicon.go index 964316f1..2ab01907 100644 --- a/internal/app/replyclassify/lexicon.go +++ b/internal/app/replyclassify/lexicon.go @@ -1,6 +1,9 @@ package replyclassify -import "strings" +import ( + "regexp" + "strings" +) // classifyLexicon is Layer 2: a deterministic, offline keyword scan over the // subject + body. It returns (result, true) only on a CLEAR signal; ambiguous @@ -13,16 +16,14 @@ import "strings" // 2. Clear interest phrases => positive. // 3. Clear rejection phrases => negative. func classifyLexicon(in Input) (Result, bool) { - text := strings.ToLower(strings.TrimSpace(in.Subject + "\n" + in.BodyText)) + text := quoteFolder.Replace(strings.ToLower(strings.TrimSpace(in.Subject + "\n" + StripQuoted(in.BodyText)))) if text == "" { return Result{}, false } // 1. Compliance / opt-out (highest priority). - for _, kw := range unsubscribeKeywords { - if strings.Contains(text, kw) { - return Result{Class: ClassUnsubscribe, Confidence: 0.9, Source: SourceLexicon}, true - } + if matchesOptOut(text) { + return Result{Class: ClassUnsubscribe, Confidence: 0.9, Source: SourceLexicon}, true } // 2. Clear interest => positive. @@ -43,20 +44,103 @@ func classifyLexicon(in Input) (Result, bool) { } // unsubscribeKeywords are explicit opt-out requests. Compliance-first: any of -// these short-circuits to "unsubscribe" before sentiment is considered. +// these short-circuits to "unsubscribe" before sentiment is considered. Each +// is matched on word boundaries, so "stop" alone never fires on "stop by". var unsubscribeKeywords = []string{ "unsubscribe", "opt out", "opt-out", "remove me", + "remove my email", + "remove my address", "take me off", "stop emailing", + "stop sending", "stop contacting", + "stop these emails", + "no more emails", "do not contact", "don't contact", "do not email", "don't email", + "do not send", + "don't send", "please stop", + "delete my details", + "delete my data", + "delete my information", +} + +var optOutPatterns = compileWordPatterns(unsubscribeKeywords) + +// compileWordPatterns anchors each phrase on word boundaries; "don't" and +// "opt-out" keep their apostrophe and hyphen literal. +func compileWordPatterns(phrases []string) []*regexp.Regexp { + out := make([]*regexp.Regexp, 0, len(phrases)) + for _, p := range phrases { + out = append(out, regexp.MustCompile(`(^|[^a-z0-9])`+regexp.QuoteMeta(p)+`($|[^a-z0-9])`)) + } + return out +} + +// quoteFolder folds the typographic apostrophes mail clients substitute while +// typing, so "don’t email me" matches the straight-quote phrase. +var quoteFolder = strings.NewReplacer("\u2019", "'", "\u2018", "'", "\u02bc", "'", "\u00b4", "'", "`", "'") + +func matchesOptOut(lowerText string) bool { + lowerText = quoteFolder.Replace(lowerText) + for _, re := range optOutPatterns { + if re.MatchString(lowerText) { + return true + } + } + return false +} + +// IsOptOut reports whether a reply, read without its quoted history, asks to +// stop being emailed. It is the single check behind automatic suppression: +// the quoted original carries the sender's own opt-out line, so matching the +// whole body would opt out everyone who replies. +func IsOptOut(subject, body string) bool { + text := strings.ToLower(strings.TrimSpace(subject + "\n" + StripQuoted(body))) + if text == "" { + return false + } + return matchesOptOut(text) +} + +// quoteMarkers begin the quoted history a mail client appends to a reply. +var quoteMarkers = []*regexp.Regexp{ + regexp.MustCompile(`(?im)^\s*on .{0,200}wrote:\s*$`), + regexp.MustCompile(`(?im)^\s*-{2,}\s*original message\s*-{2,}\s*$`), + regexp.MustCompile(`(?im)^\s*-{2,}\s*forwarded message\s*-{2,}\s*$`), + regexp.MustCompile(`(?im)^\s*from:\s.+$`), + regexp.MustCompile(`(?im)^\s*le .{0,200}a écrit\s*:\s*$`), + regexp.MustCompile(`(?im)^\s*am .{0,200}schrieb .{0,200}:\s*$`), +} + +// StripQuoted drops the quoted history from a reply body: everything from the +// first reply marker on, plus any line that is itself a ">" quote. +func StripQuoted(body string) string { + if body == "" { + return "" + } + cut := len(body) + for _, re := range quoteMarkers { + if loc := re.FindStringIndex(body); loc != nil && loc[0] < cut { + cut = loc[0] + } + } + body = body[:cut] + lines := strings.Split(body, "\n") + kept := lines[:0] + for _, ln := range lines { + if strings.HasPrefix(strings.TrimSpace(ln), ">") { + continue + } + kept = append(kept, ln) + } + return strings.TrimSpace(strings.Join(kept, "\n")) } // positiveKeywords are clear buying / interest signals. Kept conservative so the diff --git a/internal/app/replyclassify/optout_test.go b/internal/app/replyclassify/optout_test.go new file mode 100644 index 00000000..28d20a32 --- /dev/null +++ b/internal/app/replyclassify/optout_test.go @@ -0,0 +1,37 @@ +package replyclassify + +import "testing" + +func TestIsOptOut(t *testing.T) { + cases := []struct { + name string + subject string + body string + want bool + }{ + {"plain", "", "Please unsubscribe me from this list.", true}, + {"remove me", "Re: hi", "remove me from your list", true}, + {"stop emailing", "", "Stop emailing me.", true}, + {"hyphen", "", "I'd like to opt-out.", true}, + {"word boundary", "", "Stop by our booth next week!", false}, + {"substring", "", "We have an unsubscribed model in beta", false}, + {"quoted footer only", "", "Sounds interesting, let's talk.\n\nOn Tue, Sep 1, 2026 Jane wrote:\n> If this isn't relevant, just reply and I'll stop.\n> Unsubscribe: https://example.com/u/abc", false}, + {"quote lines only", "", "Yes please\n> unsubscribe here", false}, + {"before quote", "", "Please remove me\n\nOn Mon Jane wrote:\n> hello", true}, + {"outlook header", "", "not interested\r\nFrom: Jane\r\nSent: Monday\r\nunsubscribe", false}, + {"curly apostrophe", "", "Please don\u2019t email me again.", true}, + {"empty", "", "", false}, + } + for _, c := range cases { + if got := IsOptOut(c.subject, c.body); got != c.want { + t.Errorf("%s: IsOptOut=%v want %v", c.name, got, c.want) + } + } +} + +func TestLexiconIgnoresQuotedHistory(t *testing.T) { + r, ok := classifyLexicon(Input{BodyText: "Sounds good, let's talk.\n\nOn Mon, X wrote:\n> reply STOP or unsubscribe to opt out"}) + if !ok || r.Class != ClassPositive { + t.Fatalf("got %+v ok=%v, want positive", r, ok) + } +} diff --git a/internal/app/segment/service.go b/internal/app/segment/service.go index d8ed18ad..93322638 100644 --- a/internal/app/segment/service.go +++ b/internal/app/segment/service.go @@ -7,6 +7,7 @@ package segment import ( "context" "fmt" + "strconv" "strings" "sync" "time" @@ -18,17 +19,12 @@ import ( "github.com/warmbly/warmbly/internal/repository" ) -// CampaignWaker wakes a campaign's parked send chain after leads are added. +// CampaignWaker wakes a campaign's parked send chain after leads are added, +// and restarts a finished one. Satisfied structurally by campaign.CampaignService. type CampaignWaker interface { WakeCampaigns(ctx context.Context, orgID uuid.UUID, campaignIDs []string) } -// CampaignStarter restarts a completed campaign whose linked segments grew. -// Satisfied structurally by campaign.CampaignService. -type CampaignStarter interface { - StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string, opts models.StartCampaignOptions) *errx.Error -} - type Service interface { List(ctx context.Context, orgID uuid.UUID) ([]models.Segment, *errx.Error) Get(ctx context.Context, orgID, id uuid.UUID) (*models.Segment, *errx.Error) @@ -56,7 +52,13 @@ type Service interface { // membership drift (dates, engagement, nested segments) still enrols. StartCampaignSegmentSync(ctx context.Context, interval time.Duration) SetCampaignWaker(w CampaignWaker) - SetCampaignStarter(st CampaignStarter) + SetEnrolmentAuditor(a EnrolmentAuditor) +} + +// EnrolmentAuditor records an automatic enrolment as a campaign update, so the +// audit spine refreshes every teammate's Leads tab when the sweep adds leads. +type EnrolmentAuditor interface { + LogAction(ctx context.Context, orgID, actorID uuid.UUID, action models.AuditAction, entityType models.AuditEntityType, entityID *uuid.UUID, ipAddress, userAgent string, changes, metadata map[string]string) } // CustomFieldLister is the slice of the contact repository Fields needs. @@ -65,10 +67,10 @@ type CustomFieldLister interface { } type service struct { - repo repository.SegmentRepository - fields CustomFieldLister - waker CampaignWaker - starter CampaignStarter + repo repository.SegmentRepository + fields CustomFieldLister + waker CampaignWaker + audit EnrolmentAuditor // orgSync coalesces org-wide enrolment passes, one entry per org that is // currently syncing. Guarded by syncMu, which owns every transition so an // entry is only dropped when nothing is running or queued. @@ -88,8 +90,8 @@ func NewService(repo repository.SegmentRepository, fields CustomFieldLister) Ser return &service{repo: repo, fields: fields, orgSync: map[uuid.UUID]*orgSyncState{}} } -func (s *service) SetCampaignWaker(w CampaignWaker) { s.waker = w } -func (s *service) SetCampaignStarter(st CampaignStarter) { s.starter = st } +func (s *service) SetCampaignWaker(w CampaignWaker) { s.waker = w } +func (s *service) SetEnrolmentAuditor(a EnrolmentAuditor) { s.audit = a } func (s *service) List(ctx context.Context, orgID uuid.UUID) ([]models.Segment, *errx.Error) { return s.repo.List(ctx, orgID) @@ -321,9 +323,7 @@ func (s *service) AddToCampaign(ctx context.Context, orgID uuid.UUID, actor stri if xerr != nil { return nil, xerr } - if s.waker != nil && res.Added > 0 { - s.waker.WakeCampaigns(ctx, orgID, []string{campaignID.String()}) - } + s.reactToEnrolment(ctx, models.LinkedCampaign{CampaignID: campaignID, OrganizationID: orgID, Status: res.Status}, res.Added) return res, nil } @@ -348,21 +348,13 @@ func (s *service) SetCampaignSegments(ctx context.Context, orgID, campaignID uui if len(ids) > models.CampaignSegmentsMax { return nil, 0, errx.New(errx.BadRequest, fmt.Sprintf("a campaign can link at most %d segments", models.CampaignSegmentsMax)) } - if xerr := s.repo.SetForCampaign(ctx, orgID, campaignID, ids); xerr != nil { + // Links and enrolment commit together: the user is waiting on this one, + // and a failed enrolment must not answer 200 with "added 0". + added, status, xerr := s.repo.ReplaceForCampaign(ctx, orgID, campaignID, ids) + if xerr != nil { return nil, 0, xerr } - added := 0 - if len(ids) > 0 { - links, xerr := s.repo.LinkedCampaignsForSegments(ctx, orgID, ids) - if xerr != nil { - return nil, 0, xerr - } - for _, lc := range links { - if lc.CampaignID == campaignID { - added = s.syncLinkedCampaign(ctx, lc) - } - } - } + s.reactToEnrolment(ctx, models.LinkedCampaign{CampaignID: campaignID, OrganizationID: orgID, Status: status}, added) out, xerr := s.repo.ListForCampaign(ctx, orgID, campaignID) if xerr != nil { return nil, 0, xerr @@ -372,32 +364,34 @@ func (s *service) SetCampaignSegments(ctx context.Context, orgID, campaignID uui // syncLinkedCampaign enrols missing leads for one linked campaign, waking an // active chain and restarting a completed one when anything was added. -func (s *service) syncLinkedCampaign(ctx context.Context, lc models.LinkedCampaign) int { +func (s *service) syncLinkedCampaign(ctx context.Context, lc models.LinkedCampaign) (int, *errx.Error) { added, xerr := s.repo.SyncCampaignSegments(ctx, lc.OrganizationID, lc.CampaignID) if xerr != nil { log.Warn().Str("campaign_id", lc.CampaignID.String()).Str("error", xerr.Message).Msg("segment sync: enrol failed") - return 0 + return 0, xerr } - if added == 0 { - return 0 + s.reactToEnrolment(ctx, lc, added) + if added > 0 && s.audit != nil { + // The request paths audit themselves; this is the platform acting + // (zero actor), and it is what tells open Leads tabs to refresh. + s.audit.LogAction(ctx, lc.OrganizationID, uuid.Nil, models.AuditActionUpdate, models.AuditEntityCampaign, &lc.CampaignID, "", "", + map[string]string{"leads_added": strconv.Itoa(added)}, map[string]string{"source": "segment_sync"}) + } + return added, nil +} + +// reactToEnrolment wakes an active campaign and restarts a finished one when +// new leads arrived. WakeCampaigns owns both (the finished case goes through +// the full launch checks), so a segment enrolment, a direct add and an +// automation all behave the same way. Paused and draft campaigns accumulate. +func (s *service) reactToEnrolment(ctx context.Context, lc models.LinkedCampaign, added int) { + if added == 0 || s.waker == nil { + return } switch lc.Status { - case "active": - if s.waker != nil { - s.waker.WakeCampaigns(ctx, lc.OrganizationID, []string{lc.CampaignID.String()}) - } - case "completed": - // Completed only means the campaign ran out of leads; new segment - // members are exactly the reason to pick it back up. StartCampaign - // re-runs every launch check, so a campaign past its end date, over - // plan limits, or with a risky list stays closed. - if s.starter != nil { - if xerr := s.starter.StartCampaign(ctx, lc.OrganizationID, lc.CampaignID.String(), models.StartCampaignOptions{Automatic: true}); xerr != nil { - log.Info().Str("campaign_id", lc.CampaignID.String()).Str("reason", xerr.Message).Msg("segment sync: completed campaign not restarted") - } - } + case "active", "completed": + s.waker.WakeCampaigns(ctx, lc.OrganizationID, []string{lc.CampaignID.String()}) } - return added } // syncLinkedCampaignsForSegments re-enrols the campaigns linked to any of the @@ -415,7 +409,7 @@ func (s *service) syncLinkedCampaignsForSegments(ctx context.Context, orgID uuid return } for _, lc := range links { - s.syncLinkedCampaign(rctx, lc) + _, _ = s.syncLinkedCampaign(rctx, lc) } }() } @@ -466,7 +460,7 @@ func (s *service) runOrgSyncPass(ctx context.Context, orgID uuid.UUID) { return } for _, lc := range links { - s.syncLinkedCampaign(rctx, lc) + _, _ = s.syncLinkedCampaign(rctx, lc) } } @@ -503,7 +497,8 @@ func (s *service) sweepLinkedCampaigns(ctx context.Context) { return } cctx, cancel := context.WithTimeout(ctx, 30*time.Second) - total += s.syncLinkedCampaign(cctx, lc) + n, _ := s.syncLinkedCampaign(cctx, lc) + total += n cancel() } if total > 0 { diff --git a/internal/app/sequence/handler.go b/internal/app/sequence/handler.go index 02b127c4..547a6e91 100644 --- a/internal/app/sequence/handler.go +++ b/internal/app/sequence/handler.go @@ -2,6 +2,10 @@ package sequence import ( "context" + "fmt" + + "github.com/getsentry/sentry-go" + "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" @@ -29,6 +33,51 @@ func (s *sequenceService) UpdateLayout(ctx context.Context, userID, campaignID s return s.sequenceRepository.UpdateLayout(ctx, userID, campaignID, positions) } +// Delete removes a step. Its attachment rows go with it through the cascade, +// so the objects behind them are listed first and dropped once the delete has +// committed — otherwise the bytes stay in storage against the org's quota with +// no row left to reach them. func (s *sequenceService) Delete(ctx context.Context, userID, campaignID, sequenceID string) *errx.Error { - return s.sequenceRepository.Delete(ctx, userID, campaignID, sequenceID) + keys := s.stepObjectKeys(ctx, campaignID, sequenceID) + + if xerr := s.sequenceRepository.Delete(ctx, userID, campaignID, sequenceID); xerr != nil { + return xerr + } + + for _, key := range keys { + if err := s.storage.Delete(ctx, key); err != nil { + sentry.CaptureException(fmt.Errorf("sequence %s delete: object %s: %w", sequenceID, key, err)) + } + } + return nil +} + +// stepObjectKeys lists the storage keys of the files scoped to one step. Best +// effort: a step still deletes when they cannot be read, it just leaves its +// objects behind rather than refusing the edit. +func (s *sequenceService) stepObjectKeys(ctx context.Context, campaignID, sequenceID string) []string { + if s.attachmentRepo == nil || s.storage == nil { + return nil + } + cID, err := uuid.Parse(campaignID) + if err != nil { + return nil + } + sID, err := uuid.Parse(sequenceID) + if err != nil { + return nil + } + atts, err := s.attachmentRepo.ListForStep(ctx, cID, sID) + if err != nil { + sentry.CaptureException(fmt.Errorf("sequence %s delete: list attachments: %w", sequenceID, err)) + return nil + } + keys := make([]string, 0, len(atts)) + for _, a := range atts { + // ListForStep also returns the campaign-wide files, which outlive the step. + if a.SequenceID != nil && *a.SequenceID == sID { + keys = append(keys, a.S3Key) + } + } + return keys } diff --git a/internal/app/sequence/service.go b/internal/app/sequence/service.go index 4682a4d1..c2eefd0c 100644 --- a/internal/app/sequence/service.go +++ b/internal/app/sequence/service.go @@ -4,6 +4,7 @@ import ( "context" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/infrastructure/storage" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -18,6 +19,8 @@ type SequenceService interface { type sequenceService struct { sequenceRepository repository.SequenceRepository + attachmentRepo repository.AttachmentRepository + storage storage.Store } func NewService(sequenceRepository repository.SequenceRepository) SequenceService { @@ -25,3 +28,15 @@ func NewService(sequenceRepository repository.SequenceRepository) SequenceServic sequenceRepository: sequenceRepository, } } + +// AttachmentAware is implemented by the sequence service so main can hand it +// the attachment repository and object store. Deleting a step cascades its +// attachment rows away, so without these the files scoped to that step leave +// their bytes in storage forever, still counted against the org's quota. +type AttachmentAware interface { + WireAttachments(repo repository.AttachmentRepository, store storage.Store) +} + +func (s *sequenceService) WireAttachments(repo repository.AttachmentRepository, store storage.Store) { + s.attachmentRepo, s.storage = repo, store +} diff --git a/internal/app/stripe/service.go b/internal/app/stripe/service.go index 698e1ccc..b7b6e5e8 100644 --- a/internal/app/stripe/service.go +++ b/internal/app/stripe/service.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "strconv" + "strings" "time" "github.com/getsentry/sentry-go" @@ -116,6 +117,12 @@ type ReferralRewarder interface { InviteeDiscountCode(ctx context.Context, inviteeOrgID uuid.UUID) string } +// OperatorNotifier is the instance-wide operator alert surface, injected +// post-construction so this package needs no import of it. Nil disables it. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + type stripeService struct { cfg *config.StripeConfig subRepo repository.SubscriptionRepository @@ -125,8 +132,12 @@ type stripeService struct { referral ReferralRewarder credits CreditGranter audit AuditLogger + opsNotify OperatorNotifier } +// WireOperatorNotifier attaches the operator alert channel. +func (s *stripeService) WireOperatorNotifier(n OperatorNotifier) { s.opsNotify = n } + func (s *stripeService) WireReferral(r ReferralRewarder) { s.referral = r } func (s *stripeService) WireCredits(g CreditGranter, a AuditLogger) { s.credits = g; s.audit = a } @@ -1161,10 +1172,62 @@ func (s *stripeService) handleChargeRefunded(ctx context.Context, event *stripe. } func (s *stripeService) handleInvoicePaymentFailed(ctx context.Context, event *stripe.Event) *errx.Error { - // Payment failed - subscription status will be updated via subscription.updated event + // The subscription's own status is updated by the subscription.updated + // event; this hook exists so an operator hears about the failure when it + // happens rather than discovering it from a churned customer. + if s.opsNotify != nil { + var inv struct { + CustomerEmail string `json:"customer_email"` + Customer string `json:"customer"` + Number string `json:"number"` + AmountDue int64 `json:"amount_due"` + Currency string `json:"currency"` + } + _ = json.Unmarshal(event.Data.Raw, &inv) + s.opsNotify.NotifyOperator( + "subscription.payment_failed", + "Payment failed", + "Stripe could not collect an invoice, so that workspace's sending is at risk.", + map[string]string{ + "Customer": firstNonEmpty(inv.CustomerEmail, inv.Customer), + "Invoice": inv.Number, + "Amount": formatStripeAmount(inv.AmountDue, inv.Currency), + }, + ) + } return nil } +// zeroDecimalCurrencies have no minor unit, so their amounts are already whole +// units and must not be divided. https://docs.stripe.com/currencies +var zeroDecimalCurrencies = map[string]bool{ + "bif": true, "clp": true, "djf": true, "gnf": true, "jpy": true, "kmf": true, + "krw": true, "mga": true, "pyg": true, "rwf": true, "ugx": true, "vnd": true, + "vuv": true, "xaf": true, "xof": true, "xpf": true, +} + +// formatStripeAmount renders a Stripe amount for a human, honouring the +// currency's exponent. +func formatStripeAmount(amount int64, currency string) string { + code := strings.ToLower(strings.TrimSpace(currency)) + if code == "" { + code = "usd" + } + if zeroDecimalCurrencies[code] { + return fmt.Sprintf("%d %s", amount, strings.ToUpper(code)) + } + return fmt.Sprintf("%.2f %s", float64(amount)/100, strings.ToUpper(code)) +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + func mapStripeStatus(status stripe.SubscriptionStatus) models.SubscriptionStatus { switch status { case stripe.SubscriptionStatusTrialing: diff --git a/internal/app/unsublink/signer.go b/internal/app/unsublink/signer.go new file mode 100644 index 00000000..65faec92 --- /dev/null +++ b/internal/app/unsublink/signer.go @@ -0,0 +1,112 @@ +// Package unsublink mints and verifies the opaque tokens behind recipient +// unsubscribe links. A token names the organization, campaign and contact it +// was minted for and carries its own expiry, all under an HMAC, so a link is +// only ever honoured for the recipient it was sent to and nothing about it can +// be guessed or altered. +package unsublink + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "errors" + "strings" + "time" + + "github.com/google/uuid" +) + +// Validity is how long a minted link keeps working. CASL wants an opt-out to +// work for 60 days and CAN-SPAM for 30, so a year keeps a follow-up sent +// months into a sequence honourable long after the campaign ended. +const Validity = 365 * 24 * time.Hour + +const ( + rawLen = 16 + 16 + 16 + 8 + macLen = 16 +) + +var ( + ErrInvalid = errors.New("unsubscribe token is invalid") + ErrExpired = errors.New("unsubscribe token has expired") +) + +// Claims is what a verified token says. +type Claims struct { + OrgID uuid.UUID + CampaignID uuid.UUID + ContactID uuid.UUID + ExpiresAt time.Time +} + +// Signer mints links under a key derived from the instance auth secret. The +// key is scoped with a purpose string so an unsubscribe token can never be +// replayed as any other signed artefact that shares the secret. +type Signer struct { + key []byte + baseURL string +} + +// New builds a signer. baseURL is the public origin of the API (the process +// that serves /unsubscribe); empty means links cannot be minted and Enabled +// reports false. +func New(secret, baseURL string) *Signer { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte("warmbly:unsubscribe-link:v1")) + return &Signer{key: mac.Sum(nil), baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/")} +} + +// Enabled reports whether the signer knows a public origin to mint links on. +func (s *Signer) Enabled() bool { + return s != nil && s.baseURL != "" && len(s.key) > 0 +} + +// Token mints the bare token for the given recipient. +func (s *Signer) Token(orgID, campaignID, contactID uuid.UUID, now time.Time) string { + raw := make([]byte, 0, rawLen+macLen) + raw = append(raw, orgID[:]...) + raw = append(raw, campaignID[:]...) + raw = append(raw, contactID[:]...) + raw = binary.BigEndian.AppendUint64(raw, uint64(now.Add(Validity).Unix())) + raw = append(raw, s.mac(raw)...) + return base64.RawURLEncoding.EncodeToString(raw) +} + +// URL mints the full link for the given recipient, or "" when disabled. +func (s *Signer) URL(orgID, campaignID, contactID uuid.UUID, now time.Time) string { + if !s.Enabled() { + return "" + } + return s.baseURL + "/unsubscribe/" + s.Token(orgID, campaignID, contactID, now) +} + +// Verify checks the token's signature and expiry and returns its claims. +func (s *Signer) Verify(token string, now time.Time) (Claims, error) { + if s == nil || len(s.key) == 0 { + return Claims{}, ErrInvalid + } + raw, err := base64.RawURLEncoding.DecodeString(strings.TrimSpace(token)) + if err != nil || len(raw) != rawLen+macLen { + return Claims{}, ErrInvalid + } + body, sig := raw[:rawLen], raw[rawLen:] + if !hmac.Equal(sig, s.mac(body)) { + return Claims{}, ErrInvalid + } + var c Claims + copy(c.OrgID[:], body[0:16]) + copy(c.CampaignID[:], body[16:32]) + copy(c.ContactID[:], body[32:48]) + c.ExpiresAt = time.Unix(int64(binary.BigEndian.Uint64(body[48:56])), 0).UTC() + if !now.Before(c.ExpiresAt) { + return c, ErrExpired + } + return c, nil +} + +func (s *Signer) mac(body []byte) []byte { + m := hmac.New(sha256.New, s.key) + m.Write(body) + return m.Sum(nil)[:macLen] +} diff --git a/internal/app/unsublink/signer_test.go b/internal/app/unsublink/signer_test.go new file mode 100644 index 00000000..97926e57 --- /dev/null +++ b/internal/app/unsublink/signer_test.go @@ -0,0 +1,61 @@ +package unsublink + +import ( + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestRoundTrip(t *testing.T) { + s := New("secret", "https://api.example.com/") + org, camp, contact := uuid.New(), uuid.New(), uuid.New() + now := time.Date(2026, 9, 3, 12, 0, 0, 0, time.UTC) + + u := s.URL(org, camp, contact, now) + if !strings.HasPrefix(u, "https://api.example.com/unsubscribe/") { + t.Fatalf("unexpected url %q", u) + } + tok := strings.TrimPrefix(u, "https://api.example.com/unsubscribe/") + c, err := s.Verify(tok, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("verify: %v", err) + } + if c.OrgID != org || c.CampaignID != camp || c.ContactID != contact { + t.Fatalf("claims mismatch: %+v", c) + } + if !c.ExpiresAt.Equal(now.Add(Validity)) { + t.Fatalf("expiry %v", c.ExpiresAt) + } + if _, err := s.Verify(tok, now.Add(Validity)); err != ErrExpired { + t.Fatalf("want expired, got %v", err) + } +} + +func TestTamperAndWrongKey(t *testing.T) { + s := New("secret", "https://api.example.com") + tok := s.Token(uuid.New(), uuid.New(), uuid.New(), time.Now()) + + if _, err := New("other", "https://api.example.com").Verify(tok, time.Now()); err != ErrInvalid { + t.Fatalf("wrong key: want invalid, got %v", err) + } + flipped := []byte(tok) + flipped[3] ^= 1 + if _, err := s.Verify(string(flipped), time.Now()); err != ErrInvalid { + t.Fatalf("tampered: want invalid, got %v", err) + } + if _, err := s.Verify("", time.Now()); err != ErrInvalid { + t.Fatalf("empty: want invalid, got %v", err) + } +} + +func TestDisabledWithoutBase(t *testing.T) { + s := New("secret", "") + if s.Enabled() { + t.Fatal("expected disabled") + } + if s.URL(uuid.New(), uuid.New(), uuid.New(), time.Now()) != "" { + t.Fatal("expected empty url") + } +} diff --git a/internal/app/updates/semver.go b/internal/app/updates/semver.go new file mode 100644 index 00000000..e7ebe970 --- /dev/null +++ b/internal/app/updates/semver.go @@ -0,0 +1,101 @@ +package updates + +import ( + "regexp" + "strconv" + "strings" +) + +// parsed is a version the comparison understands: a release tag (v1.4.0), a +// prerelease (v1.5.0-rc.1) or a git describe string (v1.4.0-3-gabc1234, which +// is three commits past v1.4.0 and therefore newer than it). +type parsed struct { + major, minor, patch int + pre string + ahead int +} + +// describeSuffix is git describe's "--g" tail, with whatever precedes +// it (a prerelease such as rc.1, or nothing) captured first. +var describeSuffix = regexp.MustCompile(`^(?:(.*)-)?(\d+)-g[0-9a-f]+$`) + +func parseVersion(raw string) (parsed, bool) { + s := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(raw), "v")) + s = strings.TrimSuffix(s, "-dirty") + if s == "" { + return parsed{}, false + } + core, rest, _ := strings.Cut(s, "-") + parts := strings.Split(core, ".") + if len(parts) < 2 || len(parts) > 3 { + return parsed{}, false + } + var p parsed + nums := []*int{&p.major, &p.minor, &p.patch} + for i, part := range parts { + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return parsed{}, false + } + *nums[i] = n + } + if rest != "" { + if m := describeSuffix.FindStringSubmatch(rest); m != nil { + ahead, err := strconv.Atoi(m[2]) + if err != nil { + return parsed{}, false + } + p.pre = m[1] + p.ahead = ahead + } else { + p.pre = rest + } + } + return p, true +} + +// compare orders two parsed versions: -1 when a < b, 0 when equal, 1 when a > b. +func compare(a, b parsed) int { + for _, pair := range [][2]int{{a.major, b.major}, {a.minor, b.minor}, {a.patch, b.patch}} { + if pair[0] != pair[1] { + if pair[0] < pair[1] { + return -1 + } + return 1 + } + } + // A prerelease sorts before its release; commits past a tag sort after it. + switch { + case a.pre != "" && b.pre == "": + return -1 + case a.pre == "" && b.pre != "": + return 1 + case a.pre != b.pre: + if a.pre < b.pre { + return -1 + } + return 1 + } + if a.ahead != b.ahead { + if a.ahead < b.ahead { + return -1 + } + return 1 + } + return 0 +} + +// newer reports whether latest is a newer version than running. The second +// result is false when either side cannot be parsed (a "dev" build), in which +// case the caller has to fall back to the checkout's commit distance. +func newer(latest, running string) (isNewer, comparable bool) { + l, ok := parseVersion(latest) + if !ok { + return false, false + } + r, ok := parseVersion(running) + if !ok { + return false, false + } + return compare(l, r) > 0, true +} diff --git a/internal/app/updates/semver_test.go b/internal/app/updates/semver_test.go new file mode 100644 index 00000000..91a5a898 --- /dev/null +++ b/internal/app/updates/semver_test.go @@ -0,0 +1,34 @@ +package updates + +import "testing" + +func TestNewer(t *testing.T) { + cases := []struct { + latest, running string + want, comparable bool + }{ + {"v1.5.0", "v1.4.0", true, true}, + {"v1.4.0", "v1.4.0", false, true}, + {"v1.4.0", "v1.5.0", false, true}, + {"v1.4.1", "v1.4.0-3-gabc1234", true, true}, + {"v1.4.0", "v1.4.0-3-gabc1234", false, true}, + {"v1.5.0", "v1.5.0-rc.1", true, true}, + {"v1.5.0-rc.2", "v1.5.0-rc.1", true, true}, + {"v1.5.0", "v1.5.0-rc.1-2-gabc1234", true, true}, + {"v1.5.0-rc.1-3-gabc1234", "v1.5.0-rc.1-2-gabc1234", true, true}, + {"v1.5.0-rc.1", "v1.5.0-rc.1-2-gabc1234", false, true}, + {"v2.0.0", "1.9.9", true, true}, + {"v1.5.0", "dev", false, false}, + {"v1.5.0", "abc1234", false, false}, + {"", "v1.5.0", false, false}, + {"v1.5.0", "v1.4.0-dirty", true, true}, + // An overflowing commit distance is not a version at all. + {"v1.5.0", "v1.4.0-99999999999999999999-gabc1234", false, false}, + } + for _, c := range cases { + got, ok := newer(c.latest, c.running) + if got != c.want || ok != c.comparable { + t.Errorf("newer(%q, %q) = (%v, %v), want (%v, %v)", c.latest, c.running, got, ok, c.want, c.comparable) + } + } +} diff --git a/internal/app/updates/service.go b/internal/app/updates/service.go new file mode 100644 index 00000000..21f57818 --- /dev/null +++ b/internal/app/updates/service.go @@ -0,0 +1,406 @@ +// Package updates answers "is there a newer Warmbly than the one running, and +// apply it". The check side polls GitHub Releases and the host-side updater on +// an interval, so the admin panel's top bar can show an update the moment one +// exists. The apply side hands the job to the updater (internal/updater) and +// relays its progress; the backend itself never touches git or docker. +package updates + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/warmbly/warmbly/internal/app/releases" + "github.com/warmbly/warmbly/internal/updater" + "github.com/warmbly/warmbly/internal/version" +) + +// Config is read from the environment by cmd/backend. +type Config struct { + // Enabled turns the periodic GitHub check on. The updater status is read + // regardless, because a running job must be visible even with checks off. + Enabled bool + Interval time.Duration + // Channel is stable (releases) or dev (prereleases included). + Channel string + GithubRepo string + GithubToken string + // UpdaterURL is the host-side updater; empty means updates are applied by + // hand and the panel only reports. + UpdaterURL string + UpdaterToken string + HTTPClient *http.Client +} + +// Latest is the newest release on the configured channel. +type Latest struct { + Tag string `json:"tag"` + Name string `json:"name,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + PublishedAt time.Time `json:"published_at,omitempty"` + Channel string `json:"channel"` +} + +// UpdaterView is what the panel knows about the host-side agent. +type UpdaterView struct { + // Configured is whether UPDATER_URL is set at all. + Configured bool `json:"configured"` + // Status is off (not configured), ok, or unreachable. + Status string `json:"status"` + Error string `json:"error,omitempty"` + Mode updater.Mode `json:"mode,omitempty"` + RepoDir string `json:"repo_dir,omitempty"` + // Exactly one of these is set: a checkout in compose and command mode, a + // release in image mode (the clone-free install). + Checkout *updater.Checkout `json:"checkout,omitempty"` + Release *updater.Release `json:"release,omitempty"` + Job *updater.Job `json:"job,omitempty"` + LastJob *updater.Job `json:"last_job,omitempty"` +} + +// State is the whole answer to GET /admin/instance/update. +type State struct { + Running version.Info `json:"running"` + Latest *Latest `json:"latest,omitempty"` + UpdateAvailable bool `json:"update_available"` + // Reason is release (a newer tag exists) or commits (the checkout is + // behind its branch); empty when nothing is pending. + Reason string `json:"reason,omitempty"` + CheckedAt time.Time `json:"checked_at,omitempty"` + CheckError string `json:"check_error,omitempty"` + Enabled bool `json:"enabled"` + Interval string `json:"interval"` + Channel string `json:"channel"` + Repo string `json:"repo"` + Updater UpdaterView `json:"updater"` +} + +var ( + ErrUpdaterNotConfigured = errors.New("no updater is configured on this instance") + ErrNothingToApply = errors.New("this install is pinned to a version and no release is known to move to; run a check first") +) + +type Service struct { + cfg Config + http *http.Client + + mu sync.Mutex + latest *Latest + checkedAt time.Time + checkErr string + + // The updater view is cached briefly so the member-facing version pill, + // the health checks and the admin poll share one read instead of each + // dialling the updater, and a stalled updater cannot slow every caller. + viewMu sync.Mutex + view UpdaterView + viewUntil time.Time +} + +// viewTTL is how long a good updater read is served from cache; viewFailTTL +// how long a failed one is, so an absent updater is dialled rarely. +const ( + viewTTL = 2 * time.Second + viewFailTTL = 30 * time.Second +) + +func New(cfg Config) *Service { + if cfg.HTTPClient == nil { + cfg.HTTPClient = &http.Client{Timeout: 15 * time.Second} + } + if cfg.Interval < 5*time.Minute { + cfg.Interval = 30 * time.Minute + } + if cfg.Channel != "dev" { + cfg.Channel = "stable" + } + cfg.UpdaterURL = strings.TrimRight(strings.TrimSpace(cfg.UpdaterURL), "/") + // Compose substitutes its default for an empty value, so "none" is how a + // .env turns the updater off. + switch strings.ToLower(cfg.UpdaterURL) { + case "none", "off", "false": + cfg.UpdaterURL = "" + } + return &Service{cfg: cfg, http: cfg.HTTPClient} +} + +// Start runs the release check now and on every interval until ctx ends. +func (s *Service) Start(ctx context.Context) { + if !s.cfg.Enabled { + log.Printf("updates: release check disabled (UPDATE_CHECK_ENABLED=false)") + return + } + go func() { + s.checkGitHub(ctx) + t := time.NewTicker(s.cfg.Interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + s.checkGitHub(ctx) + } + } + }() +} + +// Check refreshes both sources now and returns the result. +func (s *Service) Check(ctx context.Context) State { + s.checkGitHub(ctx) + view := s.updaterStatus(ctx, http.MethodPost, "/check") + s.storeView(view) + return s.compose(view, false) +} + +// State returns the cached release check plus the updater's state, read live +// at most every viewTTL. withLog keeps job logs; the top-bar poll drops them. +func (s *Service) State(ctx context.Context, withLog bool) State { + return s.compose(s.cachedView(ctx), !withLog) +} + +func (s *Service) cachedView(ctx context.Context) UpdaterView { + s.viewMu.Lock() + if time.Now().Before(s.viewUntil) { + v := s.view + s.viewMu.Unlock() + return v + } + s.viewMu.Unlock() + view := s.updaterStatus(ctx, http.MethodGet, "/status") + s.storeView(view) + return view +} + +func (s *Service) storeView(view UpdaterView) { + ttl := viewTTL + if view.Status == "unreachable" || (view.Status == "off" && view.Configured) { + ttl = viewFailTTL + } + s.viewMu.Lock() + s.view = view + s.viewUntil = time.Now().Add(ttl) + s.viewMu.Unlock() +} + +// Apply asks the updater to move to target: "latest" picks the tracked branch +// or, on a pinned checkout, the newest release; anything else is a tag. +func (s *Service) Apply(ctx context.Context, target string) (*updater.Job, error) { + if s.cfg.UpdaterURL == "" { + return nil, ErrUpdaterNotConfigured + } + // Every target goes through the same availability check, so a missing + // updater is one clear answer rather than a transport error. + view := s.updaterStatus(ctx, http.MethodGet, "/status") + s.storeView(view) + if view.Status != "ok" { + if view.Error == "" { + return nil, ErrUpdaterNotConfigured + } + return nil, errors.New(view.Error) + } + req := updater.UpdateRequest{} + switch strings.TrimSpace(target) { + case "", "latest", "branch": + // A pinned install has nothing to move to on its own: an image install + // reads its tag from .env and a detached checkout is on a tag, so both + // need the release the check found naming the destination. + pinned := (view.Checkout != nil && view.Checkout.Detached) || + (view.Mode == updater.ModeImage && (view.Release == nil || view.Release.Pinned)) + if pinned { + s.mu.Lock() + latest := s.latest + s.mu.Unlock() + if latest == nil { + return nil, ErrNothingToApply + } + req.Tag = latest.Tag + } + default: + req.Tag = strings.TrimSpace(target) + } + + body, _ := json.Marshal(req) + resp, err := s.call(ctx, http.MethodPost, "/update", bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusAccepted { + var e struct { + Error string `json:"error"` + } + _ = json.Unmarshal(raw, &e) + if e.Error == "" { + e.Error = fmt.Sprintf("updater answered %d", resp.StatusCode) + } + return nil, errors.New(e.Error) + } + var job updater.Job + if err := json.Unmarshal(raw, &job); err != nil { + return nil, fmt.Errorf("decode updater answer: %w", err) + } + return &job, nil +} + +// internals + +func (s *Service) checkGitHub(ctx context.Context) { + cctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + rels, err := releases.FetchReleases(cctx, s.http, s.cfg.GithubRepo, s.cfg.GithubToken) + s.mu.Lock() + defer s.mu.Unlock() + s.checkedAt = time.Now() + if err != nil { + s.checkErr = err.Error() + log.Printf("updates: release check failed: %v", err) + return + } + s.checkErr = "" + stable, dev := releases.PickChannelHeads(rels) + head := stable + if s.cfg.Channel == "dev" && dev != nil { + head = dev + } + if head == nil { + s.latest = nil + return + } + s.latest = &Latest{ + Tag: head.TagName, Name: head.Name, HTMLURL: head.HTMLURL, + PublishedAt: head.PublishedAt, Channel: s.cfg.Channel, + } +} + +func (s *Service) compose(view UpdaterView, dropLogs bool) State { + s.mu.Lock() + st := State{ + Running: version.Current(), + Latest: s.latest, + CheckedAt: s.checkedAt, + CheckError: s.checkErr, + Enabled: s.cfg.Enabled, + Interval: s.cfg.Interval.String(), + Channel: s.cfg.Channel, + Repo: s.cfg.GithubRepo, + Updater: view, + } + s.mu.Unlock() + + if st.Latest != nil { + if isNewer, ok := newer(st.Latest.Tag, st.Running.Version); ok && isNewer { + st.UpdateAvailable = true + st.Reason = "release" + } + } + if !st.UpdateAvailable && view.Checkout != nil && !view.Checkout.Detached && view.Checkout.Behind > 0 { + st.UpdateAvailable = true + st.Reason = "commits" + } + if dropLogs { + if st.Updater.Job != nil { + st.Updater.Job = withoutLog(st.Updater.Job) + } + if st.Updater.LastJob != nil { + st.Updater.LastJob = withoutLog(st.Updater.LastJob) + } + } + return st +} + +func withoutLog(j *updater.Job) *updater.Job { + c := *j + c.Log = nil + return &c +} + +func (s *Service) updaterStatus(ctx context.Context, method, path string) UpdaterView { + if s.cfg.UpdaterURL == "" { + return UpdaterView{Status: "off"} + } + view := UpdaterView{Configured: true} + cctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + resp, err := s.call(cctx, method, path, nil) + if err != nil { + // Under compose the backend always gets UPDATER_URL=http://updater:8095, + // profile or not. The compose service name not resolving is the + // profile being off, which is report-only by choice, not a broken + // updater. Any other host that fails is reported as unreachable. + var dns *net.DNSError + if errors.As(err, &dns) && s.composeServiceHost() { + view.Status = "off" + view.Error = "the updater is not running; enable the updater compose profile (make up does) to update from here" + return view + } + view.Status = "unreachable" + view.Error = describeDialError(err) + return view + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + view.Status = "unreachable" + view.Error = fmt.Sprintf("updater answered %d; check UPDATER_TOKEN matches on both sides", resp.StatusCode) + return view + } + var st updater.Status + if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&st); err != nil { + view.Status = "unreachable" + view.Error = "could not decode the updater's answer" + return view + } + view.Status = "ok" + view.Mode = st.Mode + view.RepoDir = st.RepoDir + view.Checkout = st.Checkout + view.Release = st.Release + view.Job = st.Job + view.LastJob = st.LastJob + return view +} + +func (s *Service) call(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, s.cfg.UpdaterURL+path, body) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+s.cfg.UpdaterToken) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return s.http.Do(req) +} + +// composeServiceHost reports whether UPDATER_URL names the compose service. +func (s *Service) composeServiceHost() bool { + u, err := url.Parse(s.cfg.UpdaterURL) + return err == nil && u.Hostname() == composeUpdaterHost +} + +// composeUpdaterHost is the updater service's name in docker-compose.yml. +const composeUpdaterHost = "updater" + +// describeDialError turns the usual "not running" failures into the sentence +// the panel shows, instead of a raw dial string. +func describeDialError(err error) string { + var dns *net.DNSError + if errors.As(err, &dns) { + return "the updater host does not resolve; check UPDATER_URL" + } + if strings.Contains(err.Error(), "connection refused") { + return "the updater is not accepting connections; it is not running or UPDATER_URL points at the wrong port" + } + return err.Error() +} diff --git a/internal/app/user/onboarding.go b/internal/app/user/onboarding.go index 70d43bb8..18a1a1ea 100644 --- a/internal/app/user/onboarding.go +++ b/internal/app/user/onboarding.go @@ -41,3 +41,16 @@ func (s *userService) UpdateUndoSendSeconds(ctx context.Context, userID uuid.UUI return nil } + +// UpdateAvatar sets (or clears, with nil) the user's avatar URL. It must go +// through the service so the cached /auth/me copy is dropped; writing the +// repository directly leaves the old avatar served until UserTTL expires. +func (s *userService) UpdateAvatar(ctx context.Context, userID uuid.UUID, avatarURL *string) *errx.Error { + if err := s.userRepository.UpdateAvatar(ctx, userID, avatarURL); err != nil { + return errx.InternalError() + } + + s.cache.Del(ctx, getUserKey(userID)) + + return nil +} diff --git a/internal/app/user/service.go b/internal/app/user/service.go index f88cda4a..e6644dcc 100644 --- a/internal/app/user/service.go +++ b/internal/app/user/service.go @@ -16,6 +16,7 @@ type UserService interface { CompleteOnboarding(ctx context.Context, userID uuid.UUID, firstName, lastName, referralSource, role, teamSize string) *errx.Error UpdateProfile(ctx context.Context, userID uuid.UUID, firstName, lastName string) *errx.Error UpdateUndoSendSeconds(ctx context.Context, userID uuid.UUID, seconds int) *errx.Error + UpdateAvatar(ctx context.Context, userID uuid.UUID, avatarURL *string) *errx.Error } type userService struct { diff --git a/internal/app/warmup/service.go b/internal/app/warmup/service.go index 55375363..3f580003 100644 --- a/internal/app/warmup/service.go +++ b/internal/app/warmup/service.go @@ -108,14 +108,24 @@ type Service interface { WireRealtime(r HealthRealtimePublisher, emailRepo repository.EmailRepository) } +// OperatorNotifier is the instance-wide operator alert surface, injected +// post-construction so this package needs no import of it. Nil disables it. +type OperatorNotifier interface { + NotifyOperator(key, title, summary string, fields map[string]string) +} + type service struct { repo repository.WarmupRepository emailRepo repository.EmailRepository webhooks WebhookDispatcher realtime HealthRealtimePublisher + opsNotify OperatorNotifier now func() time.Time } +// WireOperatorNotifier attaches the operator alert channel. +func (s *service) WireOperatorNotifier(n OperatorNotifier) { s.opsNotify = n } + func NewService(repo repository.WarmupRepository) Service { return &service{ repo: repo, @@ -418,6 +428,20 @@ func (s *service) SubmitAppeal(ctx context.Context, userID, accountID uuid.UUID, if err != nil { return uuid.Nil, errx.InternalError() } + + if s.opsNotify != nil { + s.opsNotify.NotifyOperator( + "warmup_appeal.created", + "Warmup ban appealed", + "A blocked mailbox asked to be let back into the warmup pool.", + map[string]string{ + "Mailbox": accountID.String(), + "State": string(health.HealthState), + "Reason": reason, + }, + ) + } + return id, nil } diff --git a/internal/app/warmupcontent/batch.go b/internal/app/warmupcontent/batch.go index b5d87575..b652d43e 100644 --- a/internal/app/warmupcontent/batch.go +++ b/internal/app/warmupcontent/batch.go @@ -143,10 +143,11 @@ func (s *service) GenerateBatch(ctx context.Context, req GenerateRequest) (uuid. return job.ID, nil } -// PollBatches reconciles every in-flight batch job against OpenAI. Completed -// batches are downloaded and ingested (clean, lint, and cache); -// failed/expired/cancelled batches mark the job failed; otherwise the latest -// batch status is persisted so the admin UI reflects progress. +// PollBatches reconciles every in-flight batch job against OpenAI. A batch +// that stopped running is ingested from whatever file it left (clean, lint, +// and cache), with an expired or cancelled reason recorded; one that left +// nothing marks the job failed. Otherwise the latest batch status is +// persisted so the admin UI reflects progress. func (s *service) PollBatches(ctx context.Context) error { if s.gen == nil { return nil @@ -174,34 +175,65 @@ func (s *service) pollBatchJob(ctx context.Context, job *models.WarmupGeneration } job.BatchStatus = state.Status - switch state.Status { - case "completed": - job.BatchOutputFileID = state.OutputFileID - // A batch whose requests all failed completes with no output file; the - // refusals are in the error file, which has the same JSONL shape. Read - // it instead of failing on the empty id, or the reason is never seen. - resultsFileID := state.OutputFileID - if resultsFileID == "" { - resultsFileID = state.ErrorFileID - } - return s.ingestBatch(ctx, job, resultsFileID, state.Counts) - case "failed", "expired", "cancelled": - now := time.Now() - job.Status = "failed" - job.FinishedAt = &now - if job.Error == "" { - // A batch that failed as a whole has no error file; this is the only account of why. - job.Error = fmt.Sprintf("batch %s", state.Status) - if state.FailureReason != "" { - job.Error += ": " + state.FailureReason - } - } - return s.repo.UpdateGenerationJob(ctx, job) - default: + if !batchEnded(state.Status) { // validating | in_progress | finalizing | cancelling | submitted — // still running; persist the latest status for visibility. return s.repo.UpdateGenerationJob(ctx, job) } + + outcome := endedBatchOutcome(state) + if job.Error == "" { + job.Error = outcome.Error + } + if outcome.ResultsFileID != "" { + job.BatchOutputFileID = state.OutputFileID + return s.ingestBatch(ctx, job, outcome.ResultsFileID, state.Counts) + } + + now := time.Now() + job.Status = "failed" + job.FinishedAt = &now + return s.repo.UpdateGenerationJob(ctx, job) +} + +// batchEnded reports whether a batch status is terminal. +func batchEnded(status string) bool { + switch status { + case "completed", "failed", "expired", "cancelled": + return true + } + return false +} + +// batchOutcome is what the poller does with a batch that stopped running. +type batchOutcome struct { + // ResultsFileID is the JSONL to ingest, empty when the batch left none. + ResultsFileID string + // Error is the reason to record, empty when the batch ended cleanly. + Error string +} + +// endedBatchOutcome decides what a terminal batch leaves behind. A window that +// closes early still hands back the requests it did finish, so an expired or +// cancelled batch is ingested rather than discarded, with the reason recorded +// so the shortfall is not read as a clean run. +func endedBatchOutcome(state generation.BatchState) batchOutcome { + // Output holds the successes, the error file the refusals; same JSONL shape. + results := state.OutputFileID + if results == "" { + results = state.ErrorFileID + } + if state.Status == "completed" { + if results == "" { + return batchOutcome{Error: "batch completed with no output file"} + } + return batchOutcome{ResultsFileID: results} + } + reason := fmt.Sprintf("batch %s", state.Status) + if state.FailureReason != "" { + reason += ": " + state.FailureReason + } + return batchOutcome{ResultsFileID: results, Error: reason} } // ingestBatch downloads a completed batch's output, cleans, lints, and caches @@ -339,7 +371,9 @@ func themeForCustomID(customID, pinnedTheme string) string { return defaultThemes[n%len(defaultThemes)] } -// CancelBatch cancels an in-flight batch job both on OpenAI and locally. +// CancelBatch asks OpenAI to cancel an in-flight batch. The job stays running +// so the poller ingests what the batch finished before the cancel landed; the +// admin's reason is recorded now and survives that ingest. func (s *service) CancelBatch(ctx context.Context, jobID uuid.UUID) error { if s.gen == nil { return ErrNotConfigured @@ -357,19 +391,19 @@ func (s *service) CancelBatch(ctx context.Context, jobID uuid.UUID) error { if job.Status == "completed" || job.Status == "failed" { return fmt.Errorf("job already finished") } + if job.BatchStatus == "cancelling" { + return fmt.Errorf("cancellation already requested") + } if err := s.gen.CancelBatch(ctx, job.BatchID); err != nil { return err } - now := time.Now() - job.Status = "failed" - job.BatchStatus = "cancelling" - job.FinishedAt = &now - if job.Error == "" { - job.Error = "cancelled by admin" - } - return s.repo.UpdateGenerationJob(ctx, job) + // A conditional write: if the poller finished the job between the read + // above and now, its terminal state and counts stand and there is nothing + // left to mark. + _, err = s.repo.MarkBatchCancelling(ctx, jobID, "cancelled by admin") + return err } // firstResultError returns the first non-empty per-line error in a batch's diff --git a/internal/app/warmupcontent/batch_test.go b/internal/app/warmupcontent/batch_test.go new file mode 100644 index 00000000..4f98f738 --- /dev/null +++ b/internal/app/warmupcontent/batch_test.go @@ -0,0 +1,88 @@ +package warmupcontent + +import ( + "testing" + + "github.com/warmbly/warmbly/internal/pkg/generation" +) + +func TestEndedBatchOutcome(t *testing.T) { + tests := []struct { + name string + state generation.BatchState + results string + err string + }{ + { + name: "completed reads the output file", + state: generation.BatchState{Status: "completed", OutputFileID: "file-out"}, + results: "file-out", + }, + { + name: "completed with every request refused reads the error file", + state: generation.BatchState{Status: "completed", ErrorFileID: "file-err"}, + results: "file-err", + }, + { + name: "completed with neither file says so", + state: generation.BatchState{Status: "completed"}, + err: "batch completed with no output file", + }, + { + name: "expired ingests what the window did finish", + state: generation.BatchState{Status: "expired", OutputFileID: "file-out", ErrorFileID: "file-err"}, + results: "file-out", + err: "batch expired", + }, + { + name: "expired with nothing finished reads the error file", + state: generation.BatchState{Status: "expired", ErrorFileID: "file-err"}, + results: "file-err", + err: "batch expired", + }, + { + name: "cancelled ingests what was already done", + state: generation.BatchState{Status: "cancelled", OutputFileID: "file-out"}, + results: "file-out", + err: "batch cancelled", + }, + { + name: "failed never ran, so there is nothing to ingest", + state: generation.BatchState{Status: "failed", FailureReason: "quota exceeded"}, + err: "batch failed: quota exceeded", + }, + { + name: "failed without a provider reason still names the status", + state: generation.BatchState{Status: "failed"}, + err: "batch failed", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := endedBatchOutcome(tc.state) + if got.ResultsFileID != tc.results { + t.Errorf("results file = %q, want %q", got.ResultsFileID, tc.results) + } + if got.Error != tc.err { + t.Errorf("error = %q, want %q", got.Error, tc.err) + } + }) + } +} + +func TestBatchEnded(t *testing.T) { + ended := []string{"completed", "failed", "expired", "cancelled"} + running := []string{"validating", "in_progress", "finalizing", "cancelling", "submitted", ""} + + for _, s := range ended { + if !batchEnded(s) { + t.Errorf("batchEnded(%q) = false, want true", s) + } + } + for _, s := range running { + if batchEnded(s) { + t.Errorf("batchEnded(%q) = true, want false", s) + } + } +} diff --git a/internal/app/warmupcontent/service.go b/internal/app/warmupcontent/service.go index d0dea2b3..4b26287f 100644 --- a/internal/app/warmupcontent/service.go +++ b/internal/app/warmupcontent/service.go @@ -84,9 +84,11 @@ type Service interface { // returns the job ID immediately. Results are ingested later by PollBatches. GenerateBatch(ctx context.Context, req GenerateRequest) (uuid.UUID, error) // PollBatches reconciles in-flight batch jobs against OpenAI: it ingests - // completed batches and marks failed/expired/cancelled ones. + // what a finished, expired or cancelled batch produced and marks the + // job failed only when it produced nothing. PollBatches(ctx context.Context) error - // CancelBatch cancels an in-flight batch job (OpenAI + local job row). + // CancelBatch asks OpenAI to cancel an in-flight batch; the job stays + // running until PollBatches ingests what finished before the cancel. CancelBatch(ctx context.Context, jobID uuid.UUID) error // RunScheduled tops every enabled pool/segment up toward its target. RunScheduled(ctx context.Context) error diff --git a/internal/app/webhook/context.go b/internal/app/webhook/context.go new file mode 100644 index 00000000..e9af4c9a --- /dev/null +++ b/internal/app/webhook/context.go @@ -0,0 +1,47 @@ +package webhook + +import "context" + +// AutomationDepthKey is the internal event-data key carrying how many automation +// hops led to an event. Underscore-prefixed so it is stripped from customer +// deliveries and cannot be set by an inbound webhook body. +const AutomationDepthKey = "_automation_depth" + +type automationDepthCtxKey struct{} + +// WithAutomationDepth marks a context as running inside an automation action, +// depth hops down from the original event. Events dispatched under it carry the +// depth so a flow that creates a contact cannot re-trigger itself forever. +func WithAutomationDepth(ctx context.Context, depth int) context.Context { + if depth <= 0 { + return ctx + } + return context.WithValue(ctx, automationDepthCtxKey{}, depth) +} + +// AutomationDepth reports the automation depth carried by ctx, 0 outside one. +func AutomationDepth(ctx context.Context) int { + if d, ok := ctx.Value(automationDepthCtxKey{}).(int); ok && d > 0 { + return d + } + return 0 +} + +// stampAutomationDepth returns a copy of a map payload for the sink, carrying +// the context's automation depth when there is one. Always a copy: the sink +// runs automations on a goroutine that writes into the map while this call +// still marshals the same payload for endpoint delivery. +func stampAutomationDepth(ctx context.Context, data any) any { + m, ok := data.(map[string]any) + if !ok { + return data + } + out := make(map[string]any, len(m)+1) + for k, v := range m { + out[k] = v + } + if depth := AutomationDepth(ctx); depth > 0 { + out[AutomationDepthKey] = float64(depth) + } + return out +} diff --git a/internal/app/webhook/context_test.go b/internal/app/webhook/context_test.go new file mode 100644 index 00000000..0a270816 --- /dev/null +++ b/internal/app/webhook/context_test.go @@ -0,0 +1,43 @@ +package webhook + +import ( + "context" + "testing" +) + +func TestAutomationDepth_RoundTrip(t *testing.T) { + if got := AutomationDepth(context.Background()); got != 0 { + t.Fatalf("depth outside an automation = %d", got) + } + ctx := WithAutomationDepth(context.Background(), 2) + if got := AutomationDepth(ctx); got != 2 { + t.Fatalf("depth = %d, want 2", got) + } + if WithAutomationDepth(context.Background(), 0) != context.Background() { + t.Fatal("a zero depth must not wrap the context") + } +} + +func TestStampAutomationDepth_CopiesMapPayloadsOnly(t *testing.T) { + data := map[string]any{"contact_email": "a@b.co"} + plain := stampAutomationDepth(context.Background(), data).(map[string]any) + if plain[AutomationDepthKey] != nil { + t.Fatal("no depth outside an automation") + } + plain["written_by_sink"] = true + if _, shared := data["written_by_sink"]; shared { + t.Fatal("the sink must get its own copy even without a depth") + } + ctx := WithAutomationDepth(context.Background(), 3) + out, ok := stampAutomationDepth(ctx, data).(map[string]any) + if !ok || out[AutomationDepthKey] != float64(3) { + t.Fatalf("stamped = %v", out) + } + if _, leaked := data[AutomationDepthKey]; leaked { + t.Fatal("the caller's map must stay untouched") + } + type typed struct{ A string } + if got := stampAutomationDepth(ctx, typed{A: "x"}); got != (typed{A: "x"}) { + t.Fatalf("struct payloads pass through unchanged, got %v", got) + } +} diff --git a/internal/app/webhook/service.go b/internal/app/webhook/service.go index 73c69369..749c18fb 100644 --- a/internal/app/webhook/service.go +++ b/internal/app/webhook/service.go @@ -229,7 +229,7 @@ func (s *service) Dispatch(ctx context.Context, orgID uuid.UUID, eventType model // Fan the event to non-webhook subscribers (integration actions) first, // independently of whether any webhook endpoint is configured. if s.sink != nil { - s.sink(ctx, orgID, eventType, data) + s.sink(ctx, orgID, eventType, stampAutomationDepth(ctx, data)) } endpoints, err := s.repo.MatchingEndpoints(ctx, orgID, eventType) diff --git a/internal/app/worker/event_send_email.go b/internal/app/worker/event_send_email.go index 5b253e5f..f8262e5b 100644 --- a/internal/app/worker/event_send_email.go +++ b/internal/app/worker/event_send_email.go @@ -52,7 +52,7 @@ func (w *WorkerService) HandleSendEmail(ctx context.Context, sendEmail models.Se } // Fetch email body from S3 (attachment refs ride inside the emsg blob). - bodyPlain, bodyHTML, attachmentRefs, err := w.fetchEmailBody(ctx, sendEmail.OrgID, sendEmail.BodyS3Key) + bodyPlain, bodyHTML, attachmentRefs, fromName, err := w.fetchEmailBody(ctx, sendEmail.OrgID, sendEmail.BodyS3Key) if err != nil { log.Error().Err(err).Str("s3_key", sendEmail.BodyS3Key).Msg("Failed to fetch email body from S3") return w.failSend(ctx, sendEmail, fmt.Sprintf("failed to fetch email body: %v", err), true) @@ -84,6 +84,7 @@ func (w *WorkerService) HandleSendEmail(ctx context.Context, sendEmail models.Se WarmupToken: sendEmail.WarmupToken, UnsubscribeURL: sendEmail.UnsubscribeURL, Attachments: attachments, + FromName: fromName, }) w.recordSendLatency(time.Since(sendStart)) w.recordSendOutcome(result) @@ -126,29 +127,30 @@ func (w *WorkerService) deleteTransportEmailBody(ctx context.Context, taskID uui } // fetchEmailBody fetches and decodes the email body from S3, returning the -// decrypted plain/HTML bodies and the attachment refs carried inside the blob. -func (w *WorkerService) fetchEmailBody(ctx context.Context, orgID uuid.UUID, s3Key string) (string, string, []emsg.Attachment, error) { +// decrypted plain/HTML bodies, the attachment refs and the sender display name +// carried inside the blob (empty when the publisher predates it). +func (w *WorkerService) fetchEmailBody(ctx context.Context, orgID uuid.UUID, s3Key string) (string, string, []emsg.Attachment, string, error) { if w.Storage == nil { - return "", "", nil, fmt.Errorf("storage client not configured") + return "", "", nil, "", fmt.Errorf("storage client not configured") } // Get object from storage body, err := w.Storage.Get(ctx, s3Key) if err != nil { - return "", "", nil, fmt.Errorf("failed to get S3 object: %w", err) + return "", "", nil, "", fmt.Errorf("failed to get S3 object: %w", err) } defer body.Close() // Read the body data, err := io.ReadAll(body) if err != nil { - return "", "", nil, fmt.Errorf("failed to read S3 object: %w", err) + return "", "", nil, "", fmt.Errorf("failed to read S3 object: %w", err) } // Decode using emsg blob, err := emsg.DecodeBinary(bytes.NewReader(data)) if err != nil { - return "", "", nil, fmt.Errorf("failed to decode emsg blob: %w", err) + return "", "", nil, "", fmt.Errorf("failed to decode emsg blob: %w", err) } bodyPlain := string(blob.PlainText) @@ -169,7 +171,7 @@ func (w *WorkerService) fetchEmailBody(ctx context.Context, orgID uuid.UUID, s3K } } - return bodyPlain, bodyHTML, blob.Attachments, nil + return bodyPlain, bodyHTML, blob.Attachments, blob.FromName, nil } // fetchAttachments downloads each attachment's bytes from object storage by diff --git a/internal/app/worker/wmail/send.go b/internal/app/worker/wmail/send.go index ad8d56ff..a8c64873 100644 --- a/internal/app/worker/wmail/send.go +++ b/internal/app/worker/wmail/send.go @@ -72,6 +72,9 @@ type SendRequest struct { // Attachments, when present, are encoded as multipart/mixed parts after the // multipart/alternative text body. Warmup sends never carry attachments. Attachments []Attachment + // FromName is the display name the control plane holds for the mailbox at + // publish time. Empty falls back to the name cached from ADD_EMAIL. + FromName string } // buildSendHeaders assembles the outbound custom headers: the warmup @@ -190,6 +193,7 @@ func (w *WMail) sendViaGmail(ctx context.Context, req *SendRequest, bodyHTML str // Send via Gmail API gmailMsg, err := w.GoogleData.Client.SendMessage( ctx, + req.FromName, req.To, req.Cc, req.Bcc, @@ -260,6 +264,7 @@ func (w *WMail) sendViaGraph(ctx context.Context, req *SendRequest, bodyHTML str sentMessageID, err := w.GraphData.Client.SendMessage( ctx, + req.FromName, req.To, req.Cc, req.Bcc, @@ -324,6 +329,7 @@ func (w *WMail) sendViaSMTP(ctx context.Context, req *SendRequest, bodyHTML stri // empty list still selects the same code path. raw, merr := w.SmtpImapData.SmtpClient.Send( ctx, + req.FromName, req.To, req.Cc, req.Bcc, diff --git a/internal/cli/api/client.go b/internal/cli/api/client.go new file mode 100644 index 00000000..f025ba41 --- /dev/null +++ b/internal/cli/api/client.go @@ -0,0 +1,348 @@ +// Package api is the CLI's HTTP client for the public Warmbly REST API. +// +// It exists so every command speaks to the API the same way: one place that +// knows the /v1 prefix, the bearer header, the idempotency header, the error +// envelope and how to walk a cursor. Nothing here is Warmbly-specific beyond +// those; the typed commands are a table on top of it. +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// Client is one host's API surface. +type Client struct { + BaseURL string + Token string + UserAgent string + HTTP *http.Client + // Debug prints the request line to stderr, which is the first thing + // anyone wants when a call goes somewhere unexpected. + Debug io.Writer +} + +func New(baseURL, token, userAgent string) *Client { + return &Client{ + BaseURL: strings.TrimRight(baseURL, "/"), + Token: token, + UserAgent: userAgent, + HTTP: &http.Client{Timeout: 120 * time.Second}, + } +} + +// Error is a non-2xx response. Every field comes from the API's stable +// envelope, so a script can branch on Code without reading the prose. +type Error struct { + Status int + Code string + Message string + RequestID string + RetryAfter string + Method string + Path string + Body string +} + +func (e *Error) Error() string { + msg := e.Message + if msg == "" { + msg = strings.TrimSpace(e.Body) + } + if msg == "" { + msg = http.StatusText(e.Status) + } + out := fmt.Sprintf("%s %s failed (HTTP %d", e.Method, e.Path, e.Status) + if e.Code != "" { + out += " " + e.Code + } + out += "): " + msg + if e.RequestID != "" { + out += " (request " + e.RequestID + ")" + } + if e.Status == http.StatusTooManyRequests && e.RetryAfter != "" { + out += ". Rate limited; retry after " + e.RetryAfter + "s." + } + return out +} + +// IsNotFound and IsUnauthorized are what command code branches on. +func (e *Error) IsNotFound() bool { return e.Status == http.StatusNotFound } +func (e *Error) IsUnauthorized() bool { return e.Status == http.StatusUnauthorized } + +// StatusOf returns the HTTP status of an API error, or 0. +func StatusOf(err error) int { + var apiErr *Error + if errors.As(err, &apiErr) { + return apiErr.Status + } + return 0 +} + +// Request is one call. Path is relative to /v1 unless it already names a +// version, which is what makes `warmbly api get /campaigns` work. +type Request struct { + Method string + Path string + Query url.Values + Body []byte + Headers map[string]string + // IdempotencyKey rides the documented header, for retryable writes. + IdempotencyKey string + // Anonymous skips the bearer header. Only the sign-in handshake uses it: + // the CLI has no credential yet, which is the whole point of the flow. + Anonymous bool +} + +// Response is a completed call. Body is the raw payload: JSON for every +// documented endpoint, but a few stream files, so it is not parsed here. +type Response struct { + Status int + Header http.Header + Body []byte + Request *Request +} + +// NormalizePath applies the /v1 rule. Exported because `warmbly api` prints +// the path it is about to call. +func NormalizePath(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if strings.HasPrefix(path, "/v") && len(path) > 2 && path[2] >= '0' && path[2] <= '9' { + return path + } + return "/v1" + path +} + +func (c *Client) Do(ctx context.Context, req Request) (*Response, error) { + if c.Token == "" && !req.Anonymous { + return nil, errors.New("no API token. Run `warmbly auth login`, or set WARMBLY_TOKEN.") + } + path := NormalizePath(req.Path) + full := c.BaseURL + path + if len(req.Query) > 0 { + full += "?" + req.Query.Encode() + } + + var reader io.Reader + if len(req.Body) > 0 { + reader = bytes.NewReader(req.Body) + } + httpReq, err := http.NewRequestWithContext(ctx, strings.ToUpper(req.Method), full, reader) + if err != nil { + return nil, err + } + if !req.Anonymous { + httpReq.Header.Set("Authorization", "Bearer "+c.Token) + } + httpReq.Header.Set("Accept", "application/json") + if c.UserAgent != "" { + httpReq.Header.Set("User-Agent", c.UserAgent) + } + if len(req.Body) > 0 && httpReq.Header.Get("Content-Type") == "" { + httpReq.Header.Set("Content-Type", "application/json") + } + if req.IdempotencyKey != "" { + httpReq.Header.Set("Idempotency-Key", req.IdempotencyKey) + } + for k, v := range req.Headers { + httpReq.Header.Set(k, v) + } + + if c.Debug != nil { + fmt.Fprintf(c.Debug, "* %s %s\n", httpReq.Method, full) + } + + resp, err := c.HTTP.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("could not reach the API at %s: %w\nCheck the host with `warmbly auth status`, or set WARMBLY_API_URL.", c.BaseURL, err) + } + defer resp.Body.Close() + + payload, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20)) + if err != nil { + return nil, fmt.Errorf("reading the API response: %w", err) + } + if c.Debug != nil { + fmt.Fprintf(c.Debug, "* HTTP %d (%d bytes)\n", resp.StatusCode, len(payload)) + } + + out := &Response{Status: resp.StatusCode, Header: resp.Header, Body: payload, Request: &req} + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return out, nil + } + + apiErr := &Error{ + Status: resp.StatusCode, + Method: strings.ToUpper(req.Method), + Path: path, + Body: string(payload), + RetryAfter: resp.Header.Get("Retry-After"), + } + var envelope struct { + Error string `json:"error"` + Message string `json:"message"` + Code string `json:"code"` + RequestID string `json:"request_id"` + } + if json.Unmarshal(payload, &envelope) == nil { + apiErr.Code = envelope.Code + apiErr.RequestID = envelope.RequestID + apiErr.Message = envelope.Message + if apiErr.Message == "" { + apiErr.Message = envelope.Error + } + } + return out, apiErr +} + +// JSON runs a request and decodes the body into v. +func (c *Client) JSON(ctx context.Context, req Request, v any) error { + resp, err := c.Do(ctx, req) + if err != nil { + return err + } + if v == nil || len(bytes.TrimSpace(resp.Body)) == 0 { + return nil + } + if err := json.Unmarshal(resp.Body, v); err != nil { + return fmt.Errorf("the API returned something that is not JSON: %w", err) + } + return nil +} + +// paginateRetries is how many rate-limited pages one walk will wait out +// before giving up. Three covers a walk that crosses a minute boundary or two; +// beyond that the budget is the problem, not the timing. +const paginateRetries = 3 + +// listEnvelope is the documented list shape: data plus pagination. +type listEnvelope struct { + Data json.RawMessage `json:"data"` + Pagination struct { + NextCursor *string `json:"next_cursor"` + HasMore bool `json:"has_more"` + } `json:"pagination"` +} + +// Paginate walks every page of a list endpoint and returns one merged +// envelope: data holds every row, pagination reports no more pages. Endpoints +// that do not use the cursor envelope come back unchanged after one call. +func (c *Client) Paginate(ctx context.Context, req Request, maxPages int) ([]byte, error) { + if maxPages <= 0 { + maxPages = 100 + } + var merged []json.RawMessage + // The cursor the walk stopped on, empty when it reached the end. It is + // what tells the caller a --max-pages cut the list short rather than the + // data running out. + nextCursor := "" + query := url.Values{} + for k, v := range req.Query { + query[k] = v + } + + // Retries are budgeted across the whole walk, not per page: a limit that + // never clears has to end as an error rather than looping until maxPages. + retriesLeft := paginateRetries + + for page := 0; page < maxPages; page++ { + req.Query = query + resp, err := c.Do(ctx, req) + if err != nil { + // A long walk will meet the per-key minute budget. The response + // says how long to wait, so waiting is strictly better than + // handing back a partial list the caller cannot tell from a + // complete one. + wait, ok := retryAfter(err) + if !ok || retriesLeft == 0 { + return nil, err + } + retriesLeft-- + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + page-- + continue + } + var env listEnvelope + if jerr := json.Unmarshal(resp.Body, &env); jerr != nil || env.Data == nil { + // Not a cursor list. One page is the whole answer. + if page == 0 { + return resp.Body, nil + } + break + } + var rows []json.RawMessage + if err := json.Unmarshal(env.Data, &rows); err != nil { + if page == 0 { + return resp.Body, nil + } + break + } + merged = append(merged, rows...) + if !env.Pagination.HasMore || env.Pagination.NextCursor == nil || *env.Pagination.NextCursor == "" { + nextCursor = "" + break + } + nextCursor = *env.Pagination.NextCursor + query = cloneValues(query) + query.Set("cursor", *env.Pagination.NextCursor) + } + + envelope := struct { + Data []json.RawMessage `json:"data"` + Pagination struct { + Total int `json:"total"` + NextCursor *string `json:"next_cursor"` + HasMore bool `json:"has_more"` + } `json:"pagination"` + }{Data: merged} + if envelope.Data == nil { + envelope.Data = []json.RawMessage{} + } + envelope.Pagination.Total = len(merged) + if nextCursor != "" { + envelope.Pagination.HasMore = true + envelope.Pagination.NextCursor = &nextCursor + } + return json.Marshal(envelope) +} + +// retryAfter reports how long a rate-limited response asked the caller to +// wait. The wait is capped so a hostile or misconfigured Retry-After cannot +// park the CLI for an hour. +func retryAfter(err error) (time.Duration, bool) { + var apiErr *Error + if !errors.As(err, &apiErr) || apiErr.Status != http.StatusTooManyRequests { + return 0, false + } + seconds, perr := strconv.Atoi(strings.TrimSpace(apiErr.RetryAfter)) + if perr != nil || seconds <= 0 { + seconds = 5 + } + if seconds > 120 { + seconds = 120 + } + return time.Duration(seconds) * time.Second, true +} + +func cloneValues(in url.Values) url.Values { + out := url.Values{} + for k, v := range in { + out[k] = append([]string(nil), v...) + } + return out +} diff --git a/internal/cli/api/client_test.go b/internal/cli/api/client_test.go new file mode 100644 index 00000000..6a829a14 --- /dev/null +++ b/internal/cli/api/client_test.go @@ -0,0 +1,232 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNormalizePath(t *testing.T) { + cases := map[string]string{ + "/campaigns": "/v1/campaigns", + "campaigns": "/v1/campaigns", + "/v1/campaigns": "/v1/campaigns", + "/v2/campaigns": "/v2/campaigns", + // "/verify" starts with /v but is not a version, so it must be + // prefixed rather than treated as v-something. + "/verify": "/v1/verify", + } + for in, want := range cases { + if got := NormalizePath(in); got != want { + t.Errorf("NormalizePath(%q) = %q, want %q", in, got, want) + } + } +} + +func TestErrorEnvelopeSurvives(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"error":"forbidden","message":"missing scope","code":"insufficient_scope","request_id":"req_123"}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + _, err := c.Do(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}) + var apiErr *Error + if !errors.As(err, &apiErr) { + t.Fatalf("got %T, want *api.Error", err) + } + if apiErr.Code != "insufficient_scope" || apiErr.RequestID != "req_123" { + t.Errorf("the machine-readable fields were lost: %+v", apiErr) + } + if !strings.Contains(apiErr.Error(), "req_123") { + t.Errorf("the request id must reach the message: %s", apiErr.Error()) + } + if StatusOf(err) != http.StatusForbidden { + t.Errorf("StatusOf = %d", StatusOf(err)) + } +} + +func TestAnonymousRequestSendsNoBearer(t *testing.T) { + var sawAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sawAuth = r.Header.Get("Authorization") + fmt.Fprint(w, `{}`) + })) + defer srv.Close() + + c := New(srv.URL, "", "test") + if _, err := c.Do(context.Background(), Request{Method: http.MethodPost, Path: "/auth/cli/code", Anonymous: true}); err != nil { + t.Fatalf("anonymous request failed: %v", err) + } + if sawAuth != "" { + t.Errorf("an anonymous request carried %q", sawAuth) + } + + // A normal request with no token must fail before it reaches the network. + if _, err := c.Do(context.Background(), Request{Method: http.MethodGet, Path: "/me"}); err == nil { + t.Error("a request with no token should not be attempted") + } +} + +func TestPaginateFollowsTheCursor(t *testing.T) { + pages := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cursor := r.URL.Query().Get("cursor") + switch cursor { + case "": + pages++ + fmt.Fprint(w, `{"data":[{"id":"a"},{"id":"b"}],"pagination":{"has_more":true,"next_cursor":"c2"}}`) + case "c2": + pages++ + fmt.Fprint(w, `{"data":[{"id":"c"}],"pagination":{"has_more":false,"next_cursor":null}}`) + default: + t.Errorf("unexpected cursor %q", cursor) + } + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + merged, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}, 10) + if err != nil { + t.Fatalf("paginate: %v", err) + } + if pages != 2 { + t.Errorf("fetched %d pages, want 2", pages) + } + var doc struct { + Data []map[string]string `json:"data"` + Pagination struct { + HasMore bool `json:"has_more"` + } `json:"pagination"` + } + if err := json.Unmarshal(merged, &doc); err != nil { + t.Fatalf("merged payload is not JSON: %v", err) + } + if len(doc.Data) != 3 { + t.Errorf("merged %d rows, want 3", len(doc.Data)) + } + if doc.Pagination.HasMore { + t.Error("the merged envelope must report no more pages") + } +} + +// An endpoint that does not use the cursor envelope comes back untouched +// rather than being flattened into an empty list. +func TestPaginateLeavesNonListPayloadsAlone(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"count":7}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + out, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/unibox/count"}, 10) + if err != nil { + t.Fatalf("paginate: %v", err) + } + if !strings.Contains(string(out), `"count":7`) { + t.Errorf("payload was rewritten: %s", out) + } +} + +// A long --all walk will meet the per-key minute budget. The client waits the +// Retry-After rather than returning a partial list the caller cannot +// distinguish from a complete one. +func TestPaginateWaitsOutARateLimit(t *testing.T) { + limited := true + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if limited { + limited = false + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + fmt.Fprint(w, `{"code":"rate_limit_exceeded","message":"slow down"}`) + return + } + fmt.Fprint(w, `{"data":[{"id":"a"}],"pagination":{"has_more":false,"next_cursor":null}}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + merged, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}, 5) + if err != nil { + t.Fatalf("paginate should have waited and retried: %v", err) + } + if !strings.Contains(string(merged), `"id":"a"`) { + t.Errorf("the retried page was lost: %s", merged) + } +} + +// A rate limit that never clears must still end, rather than looping until +// max-pages with the caller none the wiser. +func TestPaginateGivesUpOnAPermanentRateLimit(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + fmt.Fprint(w, `{"code":"rate_limit_exceeded"}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + if _, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}, 2); err == nil { + t.Fatal("a permanent rate limit must surface as an error") + } +} + +// A walk cut short by --max-pages has to say so. A caller that cannot tell a +// truncated list from a complete one will act on a partial answer. +func TestPaginateReportsTruncation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Always another page. + fmt.Fprint(w, `{"data":[{"id":"x"}],"pagination":{"has_more":true,"next_cursor":"more"}}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + merged, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}, 2) + if err != nil { + t.Fatalf("paginate: %v", err) + } + var doc struct { + Data []map[string]string `json:"data"` + Pagination struct { + HasMore bool `json:"has_more"` + NextCursor *string `json:"next_cursor"` + Total int `json:"total"` + } `json:"pagination"` + } + if err := json.Unmarshal(merged, &doc); err != nil { + t.Fatalf("merged payload: %v", err) + } + if len(doc.Data) != 2 { + t.Errorf("collected %d rows over 2 pages, want 2", len(doc.Data)) + } + if !doc.Pagination.HasMore { + t.Error("a walk stopped by max-pages must report has_more") + } + if doc.Pagination.NextCursor == nil || *doc.Pagination.NextCursor != "more" { + t.Errorf("the cursor to resume from was lost: %+v", doc.Pagination.NextCursor) + } +} + +// And a walk that genuinely ran out still reports the end. +func TestPaginateReportsCompletion(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"data":[{"id":"x"}],"pagination":{"has_more":false,"next_cursor":null}}`) + })) + defer srv.Close() + + c := New(srv.URL, "wmbly_x", "test") + merged, err := c.Paginate(context.Background(), Request{Method: http.MethodGet, Path: "/campaigns"}, 10) + if err != nil { + t.Fatalf("paginate: %v", err) + } + if !strings.Contains(string(merged), `"has_more":false`) || !strings.Contains(string(merged), `"next_cursor":null`) { + t.Errorf("a completed walk must report the end: %s", merged) + } +} diff --git a/internal/cli/config/config.go b/internal/cli/config/config.go new file mode 100644 index 00000000..cf6c04a0 --- /dev/null +++ b/internal/cli/config/config.go @@ -0,0 +1,245 @@ +// Package config is where the `warmbly` CLI remembers who you are. +// +// Two files, the split gh made conventional: config.yml holds preferences and +// aliases and is safe to read; hosts.yml holds one credential per host and is +// written 0600. Environment variables override both and are never written +// back, so CI sets WARMBLY_TOKEN and never runs a login. +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// DefaultHost is the hosted service. A bare `warmbly auth login` means this. +const DefaultHost = "warmbly.com" + +// Env vars, all documented. TokenEnv is checked first; APIKeyEnv is the name +// warmblyctl already taught people and keeps working. +const ( + DirEnv = "WARMBLY_CONFIG_DIR" + TokenEnv = "WARMBLY_TOKEN" + APIKeyEnv = "WARMBLY_API_KEY" + HostEnv = "WARMBLY_HOST" + APIURLEnv = "WARMBLY_API_URL" +) + +// Host is one signed-in instance. +type Host struct { + // APIURL is the base the CLI calls, without a trailing slash and without + // the /v1 prefix. Resolved once at login so no later command has to guess. + APIURL string `yaml:"api_url"` + // AppURL is the dashboard origin, as the instance reports it. Stored at + // sign-in so `warmbly browse` opens the right page rather than guessing + // from the hostname, which is wrong on any non-default layout. + AppURL string `yaml:"app_url,omitempty"` + Token string `yaml:"token,omitempty"` + + User string `yaml:"user,omitempty"` + UserID string `yaml:"user_id,omitempty"` + Organization string `yaml:"organization,omitempty"` + OrganizationID string `yaml:"organization_id,omitempty"` + Scopes []string `yaml:"scopes,omitempty"` + // APIKeyID is what `warmbly auth logout` revokes. + APIKeyID string `yaml:"api_key_id,omitempty"` + AddedAt time.Time `yaml:"added_at,omitempty"` +} + +// Config is the preference file. +type Config struct { + // ActiveHost is what commands use when no --host is given. + ActiveHost string `yaml:"active_host,omitempty"` + // Output is the default renderer: table or json. + Output string `yaml:"output,omitempty"` + // Confirm is "always" or "sends". "sends" (the default) prompts only for + // commands that put real mail on the wire. + Confirm string `yaml:"confirm,omitempty"` + // Pager is the command long output is piped through, or "cat" to disable. + Pager string `yaml:"pager,omitempty"` + // Browser overrides the command used to open a URL. + Browser string `yaml:"browser,omitempty"` + Aliases map[string]string `yaml:"aliases,omitempty"` +} + +// Keys are the settable config fields, with what each one does. `warmbly +// config set` refuses anything not in here so a typo is not silently stored. +var Keys = []struct { + Name, Help, Default string +}{ + {"active_host", "Which signed-in host commands use by default", DefaultHost}, + {"output", "Default output format: table or json", "table"}, + {"confirm", "When to prompt before a command that sends: sends or always", "sends"}, + {"pager", "Program to page long output through; cat disables paging", "$PAGER"}, + {"browser", "Program used to open a URL", "$BROWSER"}, +} + +// Dir is where both files live: WARMBLY_CONFIG_DIR, then XDG, then ~/.config. +func Dir() string { + if v := strings.TrimSpace(os.Getenv(DirEnv)); v != "" { + return v + } + if v := strings.TrimSpace(os.Getenv("XDG_CONFIG_HOME")); v != "" { + return filepath.Join(v, "warmbly") + } + home, err := os.UserHomeDir() + if err != nil { + return ".warmbly" + } + return filepath.Join(home, ".config", "warmbly") +} + +func configPath() string { return filepath.Join(Dir(), "config.yml") } +func hostsPath() string { return filepath.Join(Dir(), "hosts.yml") } + +// Load reads config.yml. A missing file is an empty config, not an error: the +// CLI has to work on a machine that has never run it. +func Load() (*Config, error) { + c := &Config{} + raw, err := os.ReadFile(configPath()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return c, nil + } + return c, fmt.Errorf("reading %s: %w", configPath(), err) + } + if err := yaml.Unmarshal(raw, c); err != nil { + return c, fmt.Errorf("%s is not valid YAML: %w", configPath(), err) + } + return c, nil +} + +func (c *Config) Save() error { + raw, err := yaml.Marshal(c) + if err != nil { + return err + } + return writeFile(configPath(), raw, 0o600) +} + +// Get reads one settable key, falling back to its default. +func (c *Config) Get(key string) string { + switch key { + case "active_host": + return c.ActiveHost + case "output": + if c.Output == "" { + return "table" + } + return c.Output + case "confirm": + if c.Confirm == "" { + return "sends" + } + return c.Confirm + case "pager": + return c.Pager + case "browser": + return c.Browser + } + return "" +} + +// Set writes one settable key. Values are validated here rather than at use, +// so a bad value is rejected while the user is still looking at it. +func (c *Config) Set(key, value string) error { + value = strings.TrimSpace(value) + switch key { + case "active_host": + c.ActiveHost = value + case "output": + if value != "table" && value != "json" { + return fmt.Errorf("output must be table or json, not %q", value) + } + c.Output = value + case "confirm": + if value != "sends" && value != "always" { + return fmt.Errorf("confirm must be sends or always, not %q", value) + } + c.Confirm = value + case "pager": + c.Pager = value + case "browser": + c.Browser = value + default: + return fmt.Errorf("unknown config key %q. Run `warmbly config list` for the settable keys.", key) + } + return nil +} + +// Hosts is hosts.yml: host name to credential. +type Hosts map[string]*Host + +func LoadHosts() (Hosts, error) { + h := Hosts{} + raw, err := os.ReadFile(hostsPath()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return h, nil + } + return h, fmt.Errorf("reading %s: %w", hostsPath(), err) + } + if err := yaml.Unmarshal(raw, &h); err != nil { + return h, fmt.Errorf("%s is not valid YAML: %w", hostsPath(), err) + } + for name, entry := range h { + if entry == nil { + delete(h, name) + } + } + return h, nil +} + +// SaveHosts writes the credential file at 0600. An empty map removes the file +// rather than leaving a stub, so `auth logout` leaves nothing behind. +func (h Hosts) Save() error { + if len(h) == 0 { + if err := os.Remove(hostsPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + raw, err := yaml.Marshal(map[string]*Host(h)) + if err != nil { + return err + } + return writeFile(hostsPath(), raw, 0o600) +} + +// Names returns the configured hosts in a stable order. +func (h Hosts) Names() []string { + out := make([]string, 0, len(h)) + for name := range h { + out = append(out, name) + } + sort.Strings(out) + return out +} + +func writeFile(path string, data []byte, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + // Write-then-rename so an interrupted save cannot truncate a working file + // and lock someone out of their own CLI. + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, mode); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("writing %s: %w", path, err) + } + return os.Chmod(path, mode) +} + +// HostsPath and ConfigPath are exported for `auth status`, which tells people +// where their credentials actually live. +func HostsPath() string { return hostsPath() } +func ConfigPath() string { return configPath() } diff --git a/internal/cli/config/config_test.go b/internal/cli/config/config_test.go new file mode 100644 index 00000000..11bc507b --- /dev/null +++ b/internal/cli/config/config_test.go @@ -0,0 +1,180 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNormalizeHost(t *testing.T) { + cases := map[string]string{ + "warmbly.com": "warmbly.com", + "WARMBLY.COM": "warmbly.com", + "https://app.warmbly.com": "warmbly.com", + "api.warmbly.com": "warmbly.com", + "https://api.acme.dev/": "acme.dev", + "warmbly.acme.com": "warmbly.acme.com", + "localhost:8080": "localhost:8080", + "": DefaultHost, + // A two-label host is not a subdomain of anything, so "api.dev" must + // not be stripped down to "dev". + "api.dev": "api.dev", + } + for in, want := range cases { + if got := NormalizeHost(in); got != want { + t.Errorf("NormalizeHost(%q) = %q, want %q", in, got, want) + } + } +} + +func TestDefaultAPIURL(t *testing.T) { + cases := map[string]string{ + "warmbly.com": "https://api.warmbly.com", + "warmbly.acme.com": "https://api.warmbly.acme.com", + "localhost:8080": "http://localhost:8080", + } + for in, want := range cases { + if got := DefaultAPIURL(in); got != want { + t.Errorf("DefaultAPIURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCandidateAPIURLs(t *testing.T) { + got := CandidateAPIURLs("acme.dev") + want := []string{"https://api.acme.dev", "https://acme.dev", "https://acme.dev/api"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } + // A host with a port is an exact address; inventing api.:port would + // probe something that cannot exist. + if urls := CandidateAPIURLs("localhost:8080"); len(urls) != 1 || urls[0] != "http://localhost:8080" { + t.Errorf("port host candidates = %v", urls) + } +} + +func TestResolvePrefersEnvironmentAndSaysSo(t *testing.T) { + t.Setenv(TokenEnv, "wmbly_from_env") + hosts := Hosts{"warmbly.com": {APIURL: "https://api.warmbly.com", Token: "wmbly_from_file"}} + r, err := Resolve(&Config{ActiveHost: "warmbly.com"}, hosts, "") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if r.Token != "wmbly_from_env" { + t.Errorf("token = %q, want the environment's", r.Token) + } + if r.Source != TokenEnv { + t.Errorf("source = %q, want %q so a surprising result is traceable", r.Source, TokenEnv) + } +} + +func TestResolveWithNoCredential(t *testing.T) { + t.Setenv(TokenEnv, "") + t.Setenv(APIKeyEnv, "") + _, err := Resolve(&Config{}, Hosts{}, "") + var missing *ErrNoToken + if err == nil { + t.Fatal("expected an error when nothing is signed in") + } + if !asErrNoToken(err, &missing) { + t.Fatalf("got %T, want *ErrNoToken", err) + } +} + +func asErrNoToken(err error, target **ErrNoToken) bool { + e, ok := err.(*ErrNoToken) + if ok { + *target = e + } + return ok +} + +func TestResolveUsesTheOnlyHostWhenNoneIsActive(t *testing.T) { + t.Setenv(TokenEnv, "") + t.Setenv(APIKeyEnv, "") + hosts := Hosts{"warmbly.acme.com": {APIURL: "https://api.warmbly.acme.com", Token: "wmbly_x"}} + r, err := Resolve(&Config{}, hosts, "") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if r.Host != "warmbly.acme.com" { + t.Errorf("host = %q, want the only signed-in one", r.Host) + } +} + +// The credential file must never be group or world readable. +func TestHostsSaveIsPrivate(t *testing.T) { + dir := t.TempDir() + t.Setenv(DirEnv, dir) + + hosts := Hosts{"warmbly.com": {APIURL: "https://api.warmbly.com", Token: "wmbly_secret"}} + if err := hosts.Save(); err != nil { + t.Fatalf("save: %v", err) + } + info, err := os.Stat(filepath.Join(dir, "hosts.yml")) + if err != nil { + t.Fatalf("stat: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("hosts.yml is %o, want 600", perm) + } + + loaded, err := LoadHosts() + if err != nil { + t.Fatalf("load: %v", err) + } + if loaded["warmbly.com"].Token != "wmbly_secret" { + t.Errorf("round trip lost the token") + } + + // Signing out of the last host leaves no file behind. + delete(loaded, "warmbly.com") + if err := loaded.Save(); err != nil { + t.Fatalf("save empty: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "hosts.yml")); !os.IsNotExist(err) { + t.Errorf("hosts.yml survived an empty save") + } +} + +func TestConfigSetRejectsUnknownAndInvalid(t *testing.T) { + c := &Config{} + if err := c.Set("nonsense", "x"); err == nil { + t.Error("an unknown key must be rejected, not silently stored") + } + if err := c.Set("output", "yaml"); err == nil { + t.Error("an invalid output format must be rejected at set time") + } + if err := c.Set("output", "json"); err != nil { + t.Errorf("valid value rejected: %v", err) + } + if c.Get("confirm") != "sends" { + t.Errorf("confirm default = %q, want sends", c.Get("confirm")) + } +} + +// Every request carries a bearer token, so only this machine may be reached +// over plaintext HTTP. A remote host that merely names a port must not be +// downgraded, and the two URL helpers must not disagree about it. +func TestRemoteHostWithAPortStaysHTTPS(t *testing.T) { + if got := DefaultAPIURL("acme.dev:8443"); got != "https://acme.dev:8443" { + t.Errorf("DefaultAPIURL = %q, want https", got) + } + if urls := CandidateAPIURLs("acme.dev:8443"); len(urls) != 1 || urls[0] != "https://acme.dev:8443" { + t.Errorf("CandidateAPIURLs = %v, want one https entry", urls) + } + for _, local := range []string{"localhost:8080", "127.0.0.1:8080", "localhost"} { + if got := DefaultAPIURL(local); got[:5] != "http:" { + t.Errorf("DefaultAPIURL(%q) = %q, want plaintext for the local machine", local, got) + } + } + // A host that merely starts with "localhost" is somebody else's domain. + if got := DefaultAPIURL("localhost.evil.example"); got[:6] != "https:" { + t.Errorf("DefaultAPIURL(localhost.evil.example) = %q, want https", got) + } +} diff --git a/internal/cli/config/resolve.go b/internal/cli/config/resolve.go new file mode 100644 index 00000000..dada84ce --- /dev/null +++ b/internal/cli/config/resolve.go @@ -0,0 +1,149 @@ +package config + +import ( + "fmt" + "net/url" + "os" + "strings" +) + +// Resolved is the answer to "who am I and where am I pointed", worked out once +// per invocation from flags, environment and the two files, in that order. +type Resolved struct { + Host string + APIURL string + Token string + // Source says where the token came from, because "it worked yesterday" is + // almost always an environment variable nobody remembers exporting. + Source string + Entry *Host +} + +// ErrNoToken is returned when nothing is signed in. Callers turn it into the +// one message worth printing: how to sign in. +type ErrNoToken struct{ Host string } + +func (e *ErrNoToken) Error() string { + return fmt.Sprintf("not signed in to %s.\nRun `warmbly auth login` to sign in, or set WARMBLY_TOKEN to an API key (wmbly_...).", e.Host) +} + +// Resolve works out the active host and token. hostFlag is --host, empty when +// not given. +func Resolve(cfg *Config, hosts Hosts, hostFlag string) (*Resolved, error) { + host := strings.TrimSpace(hostFlag) + if host == "" { + host = strings.TrimSpace(os.Getenv(HostEnv)) + } + if host == "" { + host = strings.TrimSpace(cfg.ActiveHost) + } + if host == "" { + // One signed-in host and no preference is not ambiguous. + if names := hosts.Names(); len(names) == 1 { + host = names[0] + } + } + if host == "" { + host = DefaultHost + } + host = NormalizeHost(host) + + r := &Resolved{Host: host, Entry: hosts[host]} + if r.Entry != nil { + r.APIURL = r.Entry.APIURL + r.Token = r.Entry.Token + r.Source = HostsPath() + } + + // The environment wins, and says so, so a surprising result is traceable. + if v := strings.TrimSpace(os.Getenv(TokenEnv)); v != "" { + r.Token, r.Source = v, TokenEnv + } else if v := strings.TrimSpace(os.Getenv(APIKeyEnv)); v != "" { + r.Token, r.Source = v, APIKeyEnv + } + if v := strings.TrimSpace(os.Getenv(APIURLEnv)); v != "" { + r.APIURL = strings.TrimRight(v, "/") + } + if r.APIURL == "" { + r.APIURL = DefaultAPIURL(host) + } + if r.Token == "" { + return r, &ErrNoToken{Host: host} + } + return r, nil +} + +// NormalizeHost turns whatever someone typed into the key hosts.yml uses: a +// bare host, no scheme, no path, no trailing slash. +func NormalizeHost(raw string) string { + h := strings.TrimSpace(strings.ToLower(raw)) + h = strings.TrimSuffix(h, "/") + if h == "" { + return DefaultHost + } + if strings.Contains(h, "://") { + if u, err := url.Parse(h); err == nil && u.Host != "" { + h = u.Host + } + } + // A bare "app.warmbly.com" or "api.warmbly.com" means the same account as + // "warmbly.com"; storing three entries for one instance helps nobody. The + // dot count keeps a real two-label host like "api.dev" intact. + for _, prefix := range []string{"app.", "api.", "www."} { + if strings.HasPrefix(h, prefix) && strings.Count(h, ".") > 1 { + return strings.TrimPrefix(h, prefix) + } + } + return h +} + +// isLoopback reports whether a host is this machine. That is the only case +// where plaintext HTTP is acceptable, because every request carries a bearer +// token: a remote host that merely names a port must still get https. +func isLoopback(host string) bool { + name, _, found := strings.Cut(host, ":") + if !found { + name = host + } + return name == "localhost" || name == "127.0.0.1" || name == "[::1]" || name == "::1" +} + +// DefaultAPIURL is the base URL to try first for a host. The hosted service is +// known; a self-hosted host follows the layout the installer writes. +func DefaultAPIURL(host string) string { + host = NormalizeHost(host) + if host == DefaultHost { + return "https://api." + DefaultHost + } + if isLoopback(host) { + return "http://" + host + } + // A host that names a port is already an exact address, so no subdomain is + // invented for it; the scheme stays https. + if strings.Contains(host, ":") { + return "https://" + host + } + return "https://api." + host +} + +// CandidateAPIURLs are the bases `auth login` probes for a self-hosted host, +// most likely first. The installer's three shapes produce the first three. +func CandidateAPIURLs(host string) []string { + host = NormalizeHost(host) + if strings.Contains(host, "://") { + return []string{strings.TrimRight(host, "/")} + } + scheme := "https" + if isLoopback(host) { + scheme = "http" + } + // A host with a port is already an exact address; do not invent subdomains. + if strings.Contains(host, ":") { + return []string{scheme + "://" + host} + } + return []string{ + scheme + "://api." + host, + scheme + "://" + host, + scheme + "://" + host + "/api", + } +} diff --git a/internal/cli/config/state.go b/internal/cli/config/state.go new file mode 100644 index 00000000..3f78734f --- /dev/null +++ b/internal/cli/config/state.go @@ -0,0 +1,56 @@ +package config + +import ( + "errors" + "os" + "path/filepath" + "time" + + "gopkg.in/yaml.v3" +) + +// State is the CLI's own bookkeeping: nothing the user sets, nothing secret. +// Kept apart from config.yml so a hand-edited config is never fighting with +// something the tool rewrites on its own. +type State struct { + // LastUpdateCheck is when the release check last ran. It is what keeps the + // check to once a day rather than once a command. + LastUpdateCheck time.Time `yaml:"last_update_check,omitempty"` + // LatestVersion is the newest release seen, so the reminder can be printed + // without a network call on every run. + LatestVersion string `yaml:"latest_version,omitempty"` +} + +func statePath() string { return filepath.Join(Dir(), "state.yml") } + +// LoadState never fails in a way a caller has to handle: a missing or corrupt +// state file means "we know nothing", which is always a safe answer. +func LoadState() *State { + s := &State{} + raw, err := os.ReadFile(statePath()) + if err != nil { + return s + } + if err := yaml.Unmarshal(raw, s); err != nil { + return &State{} + } + return s +} + +// Save is best effort on purpose. This file holds a timestamp and a version +// string; a read-only home directory or a container with no writable HOME must +// cost the user a redundant version check, not a failed command. +func (s *State) Save() error { + raw, err := yaml.Marshal(s) + if err != nil { + return err + } + if err := writeFile(statePath(), raw, 0o600); err != nil && !errors.Is(err, os.ErrPermission) { + return err + } + return nil +} + +// StatePath is exported for `warmbly config list`, which shows where every +// file the CLI owns lives. +func StatePath() string { return statePath() } diff --git a/internal/cli/iostreams/iostreams.go b/internal/cli/iostreams/iostreams.go new file mode 100644 index 00000000..ba432e4c --- /dev/null +++ b/internal/cli/iostreams/iostreams.go @@ -0,0 +1,327 @@ +// Package iostreams is the CLI's terminal: where output goes, whether anyone +// is watching, and how to ask a question. +// +// The rule the whole CLI depends on: nothing prompts when stdin is not a +// terminal. A command that would need an answer fails with the flag that +// supplies it instead, which is what makes the surface scriptable. +package iostreams + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "golang.org/x/term" +) + +type IOStreams struct { + In io.Reader + Out io.Writer + ErrOut io.Writer + + stdinTTY bool + stdoutTTY bool + color bool + width int +} + +// System builds the streams from the real process, reading the environment +// conventions everyone already expects: NO_COLOR off, FORCE_COLOR on. +func System() *IOStreams { + s := &IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} + s.stdinTTY = term.IsTerminal(int(os.Stdin.Fd())) + s.stdoutTTY = term.IsTerminal(int(os.Stdout.Fd())) + s.width = 80 + if s.stdoutTTY { + if w, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil && w > 20 { + s.width = w + } + } + s.color = s.stdoutTTY && os.Getenv("NO_COLOR") == "" && os.Getenv("TERM") != "dumb" + if os.Getenv("FORCE_COLOR") != "" { + s.color = true + } + return s +} + +func (s *IOStreams) IsStdinTTY() bool { return s.stdinTTY } +func (s *IOStreams) IsStdoutTTY() bool { return s.stdoutTTY } +func (s *IOStreams) ColorEnabled() bool { + return s.color +} +func (s *IOStreams) TerminalWidth() int { return s.width } + +// SetColor forces colour on or off (--no-color, or a test). +func (s *IOStreams) SetColor(on bool) { s.color = on } + +func (s *IOStreams) Printf(format string, a ...any) { fmt.Fprintf(s.Out, format, a...) } +func (s *IOStreams) Println(a ...any) { fmt.Fprintln(s.Out, a...) } +func (s *IOStreams) Errorf(format string, a ...any) { fmt.Fprintf(s.ErrOut, format, a...) } +func (s *IOStreams) Errorln(a ...any) { fmt.Fprintln(s.ErrOut, a...) } + +// Colour helpers. Each one is a no-op when colour is off, so call sites never +// branch and piped output never carries escape codes. +func (s *IOStreams) paint(code, text string) string { + if !s.color { + return text + } + return "\033[" + code + "m" + text + "\033[0m" +} + +func (s *IOStreams) Bold(t string) string { return s.paint("1", t) } +func (s *IOStreams) Dim(t string) string { return s.paint("2", t) } +func (s *IOStreams) Red(t string) string { return s.paint("31", t) } +func (s *IOStreams) Green(t string) string { return s.paint("32", t) } +func (s *IOStreams) Yellow(t string) string { return s.paint("33", t) } +func (s *IOStreams) Blue(t string) string { return s.paint("34", t) } +func (s *IOStreams) Magenta(t string) string { return s.paint("35", t) } +func (s *IOStreams) Cyan(t string) string { return s.paint("36", t) } +func (s *IOStreams) Gray(t string) string { return s.paint("90", t) } + +// Icons that degrade to ASCII, because a Windows console or a CI log should +// not render boxes. +func (s *IOStreams) Tick() string { + if s.color { + return s.Green("✓") + } + return "ok" +} + +func (s *IOStreams) Cross() string { + if s.color { + return s.Red("✗") + } + return "x" +} + +// ErrNoTTY is what every prompt returns when nobody is there to answer. +type ErrNoTTY struct{ Need string } + +func (e *ErrNoTTY) Error() string { + return "this needs an answer and there is no terminal to ask on. " + e.Need +} + +// Confirm asks a yes/no question. def is the answer a bare Enter gives. +func (s *IOStreams) Confirm(question string, def bool) (bool, error) { + if !s.stdinTTY { + return false, &ErrNoTTY{Need: "Pass --yes to answer yes without being asked."} + } + suffix := " [y/N] " + if def { + suffix = " [Y/n] " + } + reader := bufio.NewReader(s.In) + for { + fmt.Fprint(s.ErrOut, question+suffix) + line, err := reader.ReadString('\n') + if err != nil { + return false, err + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "": + return def, nil + case "y", "yes": + return true, nil + case "n", "no": + return false, nil + } + fmt.Fprintln(s.ErrOut, "Please answer y or n.") + } +} + +// Input asks for a line of text, offering def when the answer is empty. +func (s *IOStreams) Input(question, def string) (string, error) { + if !s.stdinTTY { + return "", &ErrNoTTY{Need: "Supply it with a flag instead."} + } + prompt := question + if def != "" { + prompt += " (" + def + ")" + } + fmt.Fprint(s.ErrOut, prompt+": ") + line, err := bufio.NewReader(s.In).ReadString('\n') + if err != nil { + return "", err + } + line = strings.TrimSpace(line) + if line == "" { + return def, nil + } + return line, nil +} + +// Secret reads a value without echoing it. Used for pasting an API key. +func (s *IOStreams) Secret(question string) (string, error) { + if !s.stdinTTY { + // A piped secret is the documented CI path, so read it plainly. + // ReadString returns io.EOF alongside the final line when the input + // ends without a newline, which `printf '%s' "$KEY" |` always does; + // treating that as a failure would reject the exact form CI uses. + line, err := bufio.NewReader(s.In).ReadString('\n') + if errors.Is(err, io.EOF) { + err = nil + } + return strings.TrimSpace(line), err + } + fmt.Fprint(s.ErrOut, question+": ") + raw, err := term.ReadPassword(int(os.Stdin.Fd())) + fmt.Fprintln(s.ErrOut) + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +// Select asks the user to pick one of a list. On a terminal it draws an +// arrow-key menu; anywhere else it is a numbered prompt, which is also the +// fallback when raw mode is unavailable. +func (s *IOStreams) Select(question string, options []string) (int, error) { + if len(options) == 0 { + return 0, fmt.Errorf("nothing to choose from") + } + if len(options) == 1 { + return 0, nil + } + if !s.stdinTTY { + return 0, &ErrNoTTY{Need: "Supply the choice with a flag instead."} + } + if idx, err := s.selectInteractive(question, options); err == nil { + return idx, nil + } + return s.selectNumbered(question, options) +} + +func (s *IOStreams) selectNumbered(question string, options []string) (int, error) { + fmt.Fprintln(s.ErrOut, question) + for i, o := range options { + fmt.Fprintf(s.ErrOut, " %d) %s\n", i+1, o) + } + reader := bufio.NewReader(s.In) + for { + fmt.Fprintf(s.ErrOut, "Choose 1-%d [1]: ", len(options)) + line, err := reader.ReadString('\n') + if err != nil { + return 0, err + } + line = strings.TrimSpace(line) + if line == "" { + return 0, nil + } + n, err := strconv.Atoi(line) + if err == nil && n >= 1 && n <= len(options) { + return n - 1, nil + } + fmt.Fprintln(s.ErrOut, "Not one of the options.") + } +} + +// selectInteractive is the arrow-key menu. Every redraw rewinds exactly as +// many lines as it printed, so a wrapped line would draw over the screen +// above it: each option is clipped to the terminal width first. +func (s *IOStreams) selectInteractive(question string, options []string) (int, error) { + fd := int(os.Stdin.Fd()) + state, err := term.MakeRaw(fd) + if err != nil { + return 0, err + } + defer func() { _ = term.Restore(fd, state) }() + + cursor := 0 + draw := func(first bool) { + if !first { + fmt.Fprintf(s.ErrOut, "\033[%dA", len(options)) + } + for i, o := range options { + prefix := " " + line := o + if i == cursor { + prefix = s.Cyan("> ") + line = s.Bold(o) + } + fmt.Fprintf(s.ErrOut, "\r\033[K%s%s\r\n", prefix, s.clip(line, 2)) + } + } + + fmt.Fprintf(s.ErrOut, "%s %s\r\n", question, s.Dim("(arrows or j/k, enter to choose)")) + draw(true) + + buf := make([]byte, 3) + for { + n, err := os.Stdin.Read(buf) + if err != nil { + return 0, err + } + switch { + case n == 1 && (buf[0] == '\r' || buf[0] == '\n'): + return cursor, nil + case n == 1 && (buf[0] == 3 || buf[0] == 27): // ctrl-c, esc + return 0, fmt.Errorf("cancelled") + case n == 1 && (buf[0] == 'j' || buf[0] == 'J'): + cursor = (cursor + 1) % len(options) + case n == 1 && (buf[0] == 'k' || buf[0] == 'K'): + cursor = (cursor - 1 + len(options)) % len(options) + case n >= 3 && buf[0] == 27 && buf[1] == '[': + switch buf[2] { + case 'B': + cursor = (cursor + 1) % len(options) + case 'A': + cursor = (cursor - 1 + len(options)) % len(options) + } + default: + // Digits pick directly, which is faster than arrowing down a list. + if n == 1 && buf[0] >= '1' && buf[0] <= '9' { + if idx := int(buf[0] - '1'); idx < len(options) { + cursor = idx + return cursor, nil + } + } + continue + } + draw(false) + } +} + +// clip truncates to the terminal width, counting a prefix the caller already +// printed. Colour codes are invisible, so they are measured out first. +func (s *IOStreams) clip(text string, used int) string { + limit := s.width - used - 1 + if limit < 10 { + limit = 10 + } + if visibleLen(text) <= limit { + return text + } + // Truncating a coloured string mid-escape would leak codes; strip first. + plain := StripANSI(text) + if len(plain) <= limit { + return plain + } + return plain[:limit-1] + "…" +} + +func visibleLen(s string) int { return len([]rune(StripANSI(s))) } + +// StripANSI removes SGR escape sequences, so widths can be measured and +// redirected output never carries colour. +func StripANSI(s string) string { + var b strings.Builder + for i := 0; i < len(s); { + if s[i] == 0x1b && i+1 < len(s) && s[i+1] == '[' { + j := i + 2 + for j < len(s) && s[j] != 'm' { + j++ + } + if j < len(s) { + i = j + 1 + continue + } + } + b.WriteByte(s[i]) + i++ + } + return b.String() +} diff --git a/internal/cli/output/output.go b/internal/cli/output/output.go new file mode 100644 index 00000000..4330abf0 --- /dev/null +++ b/internal/cli/output/output.go @@ -0,0 +1,383 @@ +// Package output renders an API response for whoever is reading it. +// +// A terminal gets a table, a pipe gets the JSON the API sent, and --template +// gets a Go template. The default flips on whether stdout is a terminal, so +// `warmbly campaign list` is readable and `warmbly campaign list > f.json` is +// parseable without anyone passing a flag. +package output + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "text/template" + "time" + + "github.com/warmbly/warmbly/internal/cli/iostreams" +) + +// Column is one table column: a header and where to read it from. +type Column struct { + Header string + // Path is dotted, so "organization.name" and "counts.sent" both work. + Path string + // Format names a renderer: time, date, bool, status, bytes, or empty for + // the value as it stands. + Format string + // Truncate caps the rendered width; 0 means no cap. + Truncate int +} + +// Table describes how one endpoint's payload becomes rows. +type Table struct { + // Root is the dotted path to the array of rows. Empty means the payload is + // itself the array, or a single object rendered as one row. + Root string + Columns []Column + // Empty is what to say when there are no rows, phrased for the resource. + Empty string +} + +// Printer holds the choice of renderer for one invocation. +type Printer struct { + IO *iostreams.IOStreams + JSON bool + Template string + // Fields narrows a table to named columns (--fields id,name). + Fields []string +} + +// Print renders payload. table may be empty, in which case JSON is the only +// honest rendering and is used regardless of the terminal. +func (p *Printer) Print(payload []byte, table Table) error { + if p.Template != "" { + return p.renderTemplate(payload) + } + if p.JSON || len(table.Columns) == 0 || !p.IO.IsStdoutTTY() { + return p.renderJSON(payload) + } + return p.renderTable(payload, table) +} + +func (p *Printer) renderJSON(payload []byte) error { + trimmed := bytes.TrimSpace(payload) + if len(trimmed) == 0 { + fmt.Fprintln(p.IO.Out, "{}") + return nil + } + var buf bytes.Buffer + if err := json.Indent(&buf, trimmed, "", " "); err != nil { + // Some endpoints stream a file. Pass it through untouched. + _, werr := p.IO.Out.Write(payload) + return werr + } + fmt.Fprintln(p.IO.Out, buf.String()) + return nil +} + +func (p *Printer) renderTemplate(payload []byte) error { + tmpl, err := template.New("out").Funcs(templateFuncs).Parse(p.Template) + if err != nil { + return fmt.Errorf("the --template is not a valid Go template: %w", err) + } + var data any + if err := json.Unmarshal(bytes.TrimSpace(payload), &data); err != nil { + return fmt.Errorf("the response is not JSON, so --template has nothing to walk: %w", err) + } + if err := tmpl.Execute(p.IO.Out, data); err != nil { + return err + } + fmt.Fprintln(p.IO.Out) + return nil +} + +var templateFuncs = template.FuncMap{ + "join": func(sep string, in []any) string { + parts := make([]string, 0, len(in)) + for _, v := range in { + parts = append(parts, fmt.Sprint(v)) + } + return strings.Join(parts, sep) + }, + "pluck": func(field string, rows []any) []any { + out := make([]any, 0, len(rows)) + for _, r := range rows { + if m, ok := r.(map[string]any); ok { + out = append(out, m[field]) + } + } + return out + }, + "timeago": func(v any) string { return relative(fmt.Sprint(v)) }, +} + +func (p *Printer) renderTable(payload []byte, table Table) error { + var doc any + if err := json.Unmarshal(bytes.TrimSpace(payload), &doc); err != nil { + return p.renderJSON(payload) + } + + node := doc + if table.Root != "" { + node = dig(doc, table.Root) + } + var rows []any + switch v := node.(type) { + case []any: + rows = v + case map[string]any: + rows = []any{v} + case nil: + rows = nil + default: + return p.renderJSON(payload) + } + + columns := table.Columns + if len(p.Fields) > 0 { + columns = filterColumns(columns, p.Fields) + if len(columns) == 0 { + return fmt.Errorf("none of the requested fields exist here. Available: %s", strings.Join(headerNames(table.Columns), ", ")) + } + } + + if len(rows) == 0 { + empty := table.Empty + if empty == "" { + empty = "Nothing here yet." + } + fmt.Fprintln(p.IO.Out, p.IO.Gray(empty)) + return nil + } + + cells := make([][]string, 0, len(rows)+1) + header := make([]string, len(columns)) + for i, c := range columns { + header[i] = strings.ToUpper(c.Header) + } + cells = append(cells, header) + for _, r := range rows { + row := make([]string, len(columns)) + for i, c := range columns { + row[i] = render(dig(r, c.Path), c) + } + cells = append(cells, row) + } + + p.writeTable(cells) + return nil +} + +// writeTable pads to the widest cell per column, then drops trailing columns +// that no longer fit rather than wrapping: a wrapped table is unreadable and +// the JSON is one flag away. +func (p *Printer) writeTable(cells [][]string) { + if len(cells) == 0 { + return + } + cols := len(cells[0]) + widths := make([]int, cols) + for _, row := range cells { + for i, cell := range row { + if n := len([]rune(iostreams.StripANSI(cell))); n > widths[i] { + widths[i] = n + } + } + } + + limit := p.IO.TerminalWidth() + keep := cols + used := 0 + for i := 0; i < cols; i++ { + next := used + widths[i] + 2 + if i > 0 && next > limit { + keep = i + break + } + used = next + } + if keep < 1 { + keep = 1 + } + + for r, row := range cells { + var line strings.Builder + for i := 0; i < keep; i++ { + cell := row[i] + if r == 0 { + cell = p.IO.Gray(cell) + } + line.WriteString(cell) + if i < keep-1 { + pad := widths[i] - len([]rune(iostreams.StripANSI(row[i]))) + 2 + line.WriteString(strings.Repeat(" ", pad)) + } + } + fmt.Fprintln(p.IO.Out, strings.TrimRight(line.String(), " ")) + } +} + +func filterColumns(cols []Column, want []string) []Column { + keep := make(map[string]bool, len(want)) + for _, w := range want { + keep[strings.ToLower(strings.TrimSpace(w))] = true + } + out := make([]Column, 0, len(cols)) + for _, c := range cols { + if keep[strings.ToLower(c.Header)] || keep[strings.ToLower(c.Path)] { + out = append(out, c) + } + } + return out +} + +func headerNames(cols []Column) []string { + out := make([]string, 0, len(cols)) + for _, c := range cols { + out = append(out, strings.ToLower(c.Header)) + } + return out +} + +// dig walks a dotted path through decoded JSON. A numeric segment indexes an +// array, so "data.0.name" works the way anyone would expect it to. +func dig(node any, path string) any { + if path == "" { + return node + } + cur := node + for _, seg := range strings.Split(path, ".") { + if cur == nil { + return nil + } + switch v := cur.(type) { + case map[string]any: + cur = v[seg] + case []any: + idx, err := strconv.Atoi(seg) + if err != nil || idx < 0 || idx >= len(v) { + return nil + } + cur = v[idx] + default: + return nil + } + } + return cur +} + +func render(v any, c Column) string { + s := stringify(v) + switch c.Format { + case "time": + s = relative(s) + case "date": + s = shortDate(s) + case "bool": + if b, ok := v.(bool); ok { + if b { + return "yes" + } + return "no" + } + case "int": + if f, ok := v.(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + } + if c.Truncate > 0 && len([]rune(s)) > c.Truncate { + s = string([]rune(s)[:c.Truncate-1]) + "…" + } + return s +} + +func stringify(v any) string { + switch t := v.(type) { + case nil: + return "-" + case string: + if t == "" { + return "-" + } + return t + case bool: + if t { + return "yes" + } + return "no" + case float64: + if t == float64(int64(t)) { + return strconv.FormatInt(int64(t), 10) + } + return strconv.FormatFloat(t, 'f', 2, 64) + case []any: + parts := make([]string, 0, len(t)) + for _, item := range t { + parts = append(parts, stringify(item)) + } + if len(parts) == 0 { + return "-" + } + return strings.Join(parts, ",") + case map[string]any: + // A nested object in a table cell is noise; name it if it has a name. + for _, key := range []string{"name", "email", "title", "id"} { + if s, ok := t[key].(string); ok && s != "" { + return s + } + } + return "{…}" + default: + return fmt.Sprint(t) + } +} + +// relative turns a timestamp into "3h ago", which is what a person reading a +// list actually wants to know. +func relative(raw string) string { + if raw == "" || raw == "-" { + return "-" + } + ts, err := time.Parse(time.RFC3339, raw) + if err != nil { + return raw + } + d := time.Since(ts) + future := "" + if d < 0 { + d = -d + future = "in " + } + suffix := " ago" + if future != "" { + suffix = "" + } + switch { + case d < time.Minute: + if future != "" { + return "in a moment" + } + return "just now" + case d < time.Hour: + return fmt.Sprintf("%s%dm%s", future, int(d.Minutes()), suffix) + case d < 24*time.Hour: + return fmt.Sprintf("%s%dh%s", future, int(d.Hours()), suffix) + case d < 30*24*time.Hour: + return fmt.Sprintf("%s%dd%s", future, int(d.Hours()/24), suffix) + default: + return ts.Local().Format("2 Jan 2006") + } +} + +func shortDate(raw string) string { + if raw == "" || raw == "-" { + return "-" + } + ts, err := time.Parse(time.RFC3339, raw) + if err != nil { + return raw + } + return ts.Local().Format("2006-01-02 15:04") +} diff --git a/internal/cli/output/output_test.go b/internal/cli/output/output_test.go new file mode 100644 index 00000000..39f1659d --- /dev/null +++ b/internal/cli/output/output_test.go @@ -0,0 +1,118 @@ +package output + +import ( + "bytes" + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/cli/iostreams" +) + +func testPrinter() (*Printer, *bytes.Buffer) { + buf := &bytes.Buffer{} + io := iostreams.System() + io.Out = buf + io.SetColor(false) + return &Printer{IO: io}, buf +} + +func TestDigWalksObjectsAndArrays(t *testing.T) { + doc := map[string]any{ + "data": []any{ + map[string]any{"name": "first", "counts": map[string]any{"sent": 4.0}}, + }, + } + if got := dig(doc, "data.0.name"); got != "first" { + t.Errorf("dig name = %v", got) + } + if got := dig(doc, "data.0.counts.sent"); got != 4.0 { + t.Errorf("dig nested = %v", got) + } + if got := dig(doc, "data.9.name"); got != nil { + t.Errorf("out of range should be nil, got %v", got) + } + if got := dig(doc, "missing.key"); got != nil { + t.Errorf("missing path should be nil, got %v", got) + } +} + +func TestTableRendersRowsAndHeaders(t *testing.T) { + p, buf := testPrinter() + // A table is only rendered for a terminal; force it directly instead. + err := p.renderTable([]byte(`{"data":[{"id":"1","name":"Alpha","status":"active"},{"id":"2","name":"Beta","status":"draft"}]}`), + Table{Root: "data", Columns: []Column{{Header: "ID", Path: "id"}, {Header: "NAME", Path: "name"}, {Header: "STATUS", Path: "status"}}}) + if err != nil { + t.Fatalf("render: %v", err) + } + out := buf.String() + for _, want := range []string{"ID", "NAME", "STATUS", "Alpha", "Beta", "draft"} { + if !strings.Contains(out, want) { + t.Errorf("table is missing %q:\n%s", want, out) + } + } + if lines := strings.Count(strings.TrimSpace(out), "\n"); lines != 2 { + t.Errorf("want a header and two rows, got:\n%s", out) + } +} + +func TestEmptyListSaysSomethingUseful(t *testing.T) { + p, buf := testPrinter() + if err := p.renderTable([]byte(`{"data":[]}`), Table{Root: "data", Columns: []Column{{Header: "ID", Path: "id"}}, Empty: "No campaigns yet."}); err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(buf.String(), "No campaigns yet.") { + t.Errorf("an empty list should explain itself, got %q", buf.String()) + } +} + +func TestFieldsNarrowTheTable(t *testing.T) { + p, buf := testPrinter() + p.Fields = []string{"name"} + if err := p.renderTable([]byte(`{"data":[{"id":"1","name":"Alpha"}]}`), + Table{Root: "data", Columns: []Column{{Header: "ID", Path: "id"}, {Header: "NAME", Path: "name"}}}); err != nil { + t.Fatalf("render: %v", err) + } + out := buf.String() + if strings.Contains(out, "ID") { + t.Errorf("--fields name should drop the ID column:\n%s", out) + } + if !strings.Contains(out, "Alpha") { + t.Errorf("--fields name dropped the row:\n%s", out) + } +} + +func TestJSONIsPassedThroughWhenNotAnObject(t *testing.T) { + p, buf := testPrinter() + if err := p.renderJSON([]byte("not json at all")); err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(buf.String(), "not json at all") { + t.Errorf("a non-JSON body must pass through untouched, got %q", buf.String()) + } +} + +func TestTemplateRendering(t *testing.T) { + p, buf := testPrinter() + p.Template = "{{range .data}}{{.name}} {{end}}" + if err := p.Print([]byte(`{"data":[{"name":"a"},{"name":"b"}]}`), Table{}); err != nil { + t.Fatalf("template: %v", err) + } + if strings.TrimSpace(buf.String()) != "a b" { + t.Errorf("template output = %q", buf.String()) + } +} + +func TestStringifyKeepsCellsReadable(t *testing.T) { + if got := stringify(nil); got != "-" { + t.Errorf("nil = %q, want -", got) + } + if got := stringify([]any{"a@x.com", "b@x.com"}); got != "a@x.com,b@x.com" { + t.Errorf("list = %q", got) + } + if got := stringify(map[string]any{"id": "x", "name": "Jane"}); got != "Jane" { + t.Errorf("object cell = %q, want the name", got) + } + if got := stringify(4.0); got != "4" { + t.Errorf("whole float = %q, want 4", got) + } +} diff --git a/internal/cli/update/update.go b/internal/cli/update/update.go new file mode 100644 index 00000000..a69dcbc0 --- /dev/null +++ b/internal/cli/update/update.go @@ -0,0 +1,305 @@ +// Package update keeps an installed CLI current. +// +// Two jobs: telling someone a newer release exists without getting in their +// way, and replacing the binary when they ask for it. +// +// The version lookup deliberately does not use the GitHub API. The +// unauthenticated API is rate limited per IP, which on a shared CI runner or +// behind a corporate NAT means the check fails for everyone at once; the +// releases/latest redirect is a plain HTTP redirect with no such limit. +package update + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +const ( + repo = "warmbly/warmbly" + // LatestURL redirects to the newest release's tag page. + LatestURL = "https://github.com/" + repo + "/releases/latest" + // DownloadBase is where the release assets live. Names carry no version, + // so "latest" resolves without knowing the tag first. + DownloadBase = "https://github.com/" + repo + "/releases/latest/download" +) + +// CheckInterval is how often the background nudge looks for a new release. +// Once a day: often enough to matter, rare enough that nobody notices it. +const CheckInterval = 24 * time.Hour + +// LatestVersion resolves the newest published release tag by following the +// latest-release redirect and reading the tag out of the final URL. +func LatestVersion(ctx context.Context, timeout time.Duration) (string, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + client := &http.Client{ + // Stop at the redirect: the tag is in the Location header, and + // following it would download an HTML page for nothing. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } + req, err := http.NewRequestWithContext(ctx, http.MethodHead, LatestURL, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + location := resp.Header.Get("Location") + if location == "" { + return "", errors.New("no release redirect") + } + tag := location[strings.LastIndex(location, "/")+1:] + if !strings.HasPrefix(tag, "v") { + return "", fmt.Errorf("unexpected release tag %q", tag) + } + return tag, nil +} + +// IsNewer reports whether candidate is a later release than current. Both are +// vX.Y.Z. A current version that is not a clean release tag (a dev build, a +// git describe string) returns false: someone running their own build does not +// want to be told to download ours. +func IsNewer(current, candidate string) bool { + cur, ok := parseVersion(current) + if !ok { + return false + } + next, ok := parseVersion(candidate) + if !ok { + return false + } + for i := 0; i < 3; i++ { + if next[i] != cur[i] { + return next[i] > cur[i] + } + } + return false +} + +func parseVersion(v string) ([3]int, bool) { + var out [3]int + v = strings.TrimPrefix(strings.TrimSpace(v), "v") + // A git describe string (1.2.3-4-gabc1234) or a prerelease is not a plain + // release, so it never compares. + if v == "" || strings.ContainsAny(v, "-+ ") { + return out, false + } + parts := strings.Split(v, ".") + if len(parts) != 3 { + return out, false + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 { + return out, false + } + out[i] = n + } + return out, true +} + +// Method is how this binary got here, which decides how it should be replaced. +type Method int + +const ( + // MethodBinary is a plain binary we can overwrite ourselves. + MethodBinary Method = iota + MethodHomebrew + MethodScoop + MethodGoInstall + MethodPackage +) + +// UpgradeCommand is what to tell the user to run when we must not replace the +// binary ourselves. Empty when a self-replace is the right answer. +func (m Method) UpgradeCommand() string { + switch m { + case MethodHomebrew: + return "brew upgrade warmbly" + case MethodScoop: + return "scoop update warmbly" + case MethodGoInstall: + return "go install github.com/" + repo + "/cmd/cli@latest" + case MethodPackage: + return "your package manager" + default: + return "" + } +} + +// DetectMethod works out how this binary was installed from where it sits. +// Fighting a package manager by overwriting the file it owns produces a +// version that reverts on the next upgrade, so this is what stops that. +func DetectMethod(executable string) Method { + path, err := filepath.EvalSymlinks(executable) + if err != nil { + path = executable + } + // Backslashes are normalised explicitly rather than with filepath.ToSlash, + // which is a no-op off Windows: the detection then behaves the same + // wherever it runs, including in a test. + lower := strings.ToLower(strings.ReplaceAll(path, `\`, "/")) + + switch { + case strings.Contains(lower, "/cellar/"), strings.Contains(lower, "/homebrew/"), + strings.Contains(lower, "/linuxbrew/"): + return MethodHomebrew + case strings.Contains(lower, "/scoop/"): + return MethodScoop + case strings.Contains(lower, "/go/bin/"), strings.HasSuffix(lower, "/gopath/bin/warmbly"): + return MethodGoInstall + case strings.HasPrefix(lower, "/usr/bin/"), strings.HasPrefix(lower, "/opt/"), + strings.HasPrefix(lower, "/snap/"), strings.HasPrefix(lower, "/nix/"): + return MethodPackage + default: + return MethodBinary + } +} + +// AssetName is the archive published for the running platform. +func AssetName() string { + if runtime.GOOS == "windows" { + return fmt.Sprintf("warmbly_%s_%s.zip", runtime.GOOS, runtime.GOARCH) + } + return fmt.Sprintf("warmbly_%s_%s.tar.gz", runtime.GOOS, runtime.GOARCH) +} + +// Replace downloads the newest build for this platform, verifies it against +// the published checksums, and swaps it in for the running binary. +// +// The swap is a rename, which is atomic: an interrupted upgrade leaves either +// the old binary or the new one, never half of either. +func Replace(ctx context.Context, executable string, progress func(string)) error { + if runtime.GOOS == "windows" { + return errors.New("self-upgrade is not supported on Windows because a running .exe cannot be replaced.\nRun the installer again instead:\n irm https://warmbly.com/cli.ps1 | iex") + } + + asset := AssetName() + progress("downloading " + asset) + archive, err := download(ctx, DownloadBase+"/"+asset) + if err != nil { + return err + } + + progress("verifying checksum") + sums, err := download(ctx, DownloadBase+"/checksums.txt") + if err != nil { + return fmt.Errorf("could not fetch checksums.txt, so the download was not verified: %w", err) + } + want := checksumFor(string(sums), asset) + if want == "" { + return fmt.Errorf("checksums.txt has no entry for %s", asset) + } + sum := sha256.Sum256(archive) + if got := hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("checksum mismatch for %s.\n expected %s\n got %s\nNothing was changed", asset, want, got) + } + + progress("unpacking") + binary, err := extractBinary(archive) + if err != nil { + return err + } + + target, err := filepath.EvalSymlinks(executable) + if err != nil { + target = executable + } + dir := filepath.Dir(target) + tmp, err := os.CreateTemp(dir, ".warmbly-upgrade-*") + if err != nil { + return fmt.Errorf("cannot write to %s: %w\nIf it is system-owned, re-run the installer instead:\n curl -fsSL https://warmbly.com/cli.sh | sh", dir, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(binary); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmpName, 0o755); err != nil { + return err + } + if err := os.Rename(tmpName, target); err != nil { + return fmt.Errorf("could not replace %s: %w", target, err) + } + return nil +} + +func download(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: 5 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("could not download %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("could not download %s: HTTP %d", url, resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, 200<<20)) +} + +func checksumFor(sums, asset string) string { + for _, line := range strings.Split(sums, "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && strings.TrimPrefix(fields[1], "*") == asset { + return fields[0] + } + } + return "" +} + +// extractBinary pulls just the warmbly executable out of the release archive. +func extractBinary(archive []byte) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + return nil, fmt.Errorf("the archive is not readable: %w", err) + } + defer gz.Close() + + reader := tar.NewReader(gz) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + if filepath.Base(header.Name) != "warmbly" { + continue + } + return io.ReadAll(io.LimitReader(reader, 200<<20)) + } + return nil, errors.New("the archive did not contain a warmbly binary") +} diff --git a/internal/cli/update/update_test.go b/internal/cli/update/update_test.go new file mode 100644 index 00000000..5188e5aa --- /dev/null +++ b/internal/cli/update/update_test.go @@ -0,0 +1,97 @@ +package update + +import ( + "runtime" + "strings" + "testing" +) + +func TestIsNewer(t *testing.T) { + cases := []struct { + current, candidate string + want bool + }{ + {"v1.2.3", "v1.2.4", true}, + {"v1.2.3", "v1.3.0", true}, + {"v1.2.3", "v2.0.0", true}, + {"v1.2.3", "v1.2.3", false}, + {"v1.2.3", "v1.2.2", false}, + {"v1.10.0", "v1.9.0", false}, + {"v1.9.0", "v1.10.0", true}, + // Someone running their own build is not behind ours, and must not be + // told to download something. + {"dev", "v1.2.3", false}, + {"v1.2.3-4-gabc1234", "v1.2.4", false}, + {"", "v1.2.3", false}, + // A malformed tag on the far side is ignored rather than trusted. + {"v1.2.3", "not-a-version", false}, + {"v1.2.3", "v1.2", false}, + } + for _, c := range cases { + if got := IsNewer(c.current, c.candidate); got != c.want { + t.Errorf("IsNewer(%q, %q) = %v, want %v", c.current, c.candidate, got, c.want) + } + } +} + +func TestDetectMethod(t *testing.T) { + cases := map[string]Method{ + "/home/jane/.local/bin/warmbly": MethodBinary, + "/opt/homebrew/bin/warmbly": MethodHomebrew, + "/usr/local/Cellar/warmbly/1.0/bin/warmbly": MethodHomebrew, + "/home/linuxbrew/.linuxbrew/bin/warmbly": MethodHomebrew, + "C:\\Users\\jane\\scoop\\shims\\warmbly.exe": MethodScoop, + "/home/jane/go/bin/warmbly": MethodGoInstall, + "/usr/bin/warmbly": MethodPackage, + "/nix/store/abc-warmbly/bin/warmbly": MethodPackage, + } + for path, want := range cases { + if got := DetectMethod(path); got != want { + t.Errorf("DetectMethod(%q) = %v, want %v", path, got, want) + } + } +} + +// Replacing a file a package manager owns produces a version that silently +// reverts on its next upgrade, so each of these has to name a command. +func TestPackageMethodsNameACommand(t *testing.T) { + for _, m := range []Method{MethodHomebrew, MethodScoop, MethodGoInstall, MethodPackage} { + if m.UpgradeCommand() == "" { + t.Errorf("method %v self-replaces; it must tell the user what to run instead", m) + } + } + if MethodBinary.UpgradeCommand() != "" { + t.Error("a plain binary should be replaced in place, not delegated") + } +} + +func TestAssetNameMatchesWhatWePublish(t *testing.T) { + name := AssetName() + if !strings.HasPrefix(name, "warmbly_"+runtime.GOOS+"_"+runtime.GOARCH) { + t.Errorf("asset name %q does not name this platform", name) + } + if runtime.GOOS == "windows" { + if !strings.HasSuffix(name, ".zip") { + t.Errorf("windows asset %q should be a zip", name) + } + } else if !strings.HasSuffix(name, ".tar.gz") { + t.Errorf("unix asset %q should be a tar.gz", name) + } +} + +func TestChecksumFor(t *testing.T) { + sums := `abc123 warmbly_linux_amd64.tar.gz +def456 warmbly_darwin_arm64.tar.gz +` + if got := checksumFor(sums, "warmbly_linux_amd64.tar.gz"); got != "abc123" { + t.Errorf("got %q", got) + } + if got := checksumFor(sums, "warmbly_windows_amd64.zip"); got != "" { + t.Errorf("an absent asset must report no checksum, got %q", got) + } + // The BSD-style "*name" form has to resolve too, or verification silently + // degrades to a warning on machines whose sha tool writes it. + if got := checksumFor("abc123 *warmbly_linux_amd64.tar.gz", "warmbly_linux_amd64.tar.gz"); got != "abc123" { + t.Errorf("star-prefixed name not matched, got %q", got) + } +} diff --git a/internal/client/goog/helper.go b/internal/client/goog/helper.go index 39e6c811..00103a96 100644 --- a/internal/client/goog/helper.go +++ b/internal/client/goog/helper.go @@ -10,7 +10,16 @@ import ( // 8-bit name for "Renée", and silently splits the address into two recipients // for "Doe, Jane". Both matter now that every send builds its own headers. func (c *Client) GetAddress() string { - name := strings.TrimSpace(c.FirstName + " " + c.LastName) + return c.FromAddress("") +} + +// FromAddress is GetAddress with a per-send display name; empty falls back to +// the name the client was built with. +func (c *Client) FromAddress(name string) string { + name = strings.TrimSpace(name) + if name == "" { + name = strings.TrimSpace(c.FirstName + " " + c.LastName) + } addr := mail.Address{Name: name, Address: c.Email} return addr.String() } diff --git a/internal/client/goog/send.go b/internal/client/goog/send.go index 8ca3af93..a9bc1f1b 100644 --- a/internal/client/goog/send.go +++ b/internal/client/goog/send.go @@ -27,6 +27,7 @@ type Attachment struct { func (c *Client) SendMessage( ctx context.Context, + fromName string, to, cc, bcc []string, messageID, subject, bodyPlain, bodyHTML string, @@ -39,12 +40,13 @@ func (c *Client) SendMessage( // returns when you READ a parsed message; submitting one to Send is // rejected outright with "'raw' RFC822 payload message string or uploading // message via /upload/* URL required". Every send goes through raw. - return c.sendRaw(to, cc, bcc, messageID, subject, bodyPlain, bodyHTML, parent, attachments, customHeaders...) + return c.sendRaw(fromName, to, cc, bcc, messageID, subject, bodyPlain, bodyHTML, parent, attachments, customHeaders...) } // sendRaw builds an RFC 5322 message and submits it as base64url-encoded Raw, // which is the only body Gmail's Send endpoint accepts. func (c *Client) sendRaw( + fromName string, to, cc, bcc []string, messageID, subject, bodyPlain, bodyHTML string, @@ -54,7 +56,7 @@ func (c *Client) sendRaw( ) (*gmail.Message, error) { var hdrs []header hdrs = append(hdrs, - header{"From", c.GetAddress()}, + header{"From", c.FromAddress(fromName)}, header{"To", mailhdr.AddressList(to)}, // We now own header encoding, so a non-ASCII subject (or recipient // display name) has to be RFC 2047-encoded here. Encoding is a no-op diff --git a/internal/client/msgraph/helper.go b/internal/client/msgraph/helper.go index c5e9859c..9c4e8f65 100644 --- a/internal/client/msgraph/helper.go +++ b/internal/client/msgraph/helper.go @@ -11,6 +11,16 @@ import ( // RFC 2047-encoding a non-ASCII display name so it does not reach the // recipient as mojibake. func (c *Client) GetAddress() string { - addr := mail.Address{Name: strings.TrimSpace(c.FirstName + " " + c.LastName), Address: c.Email} + return c.FromAddress("") +} + +// FromAddress is GetAddress with a per-send display name; empty falls back to +// the name the client was built with. +func (c *Client) FromAddress(name string) string { + name = strings.TrimSpace(name) + if name == "" { + name = strings.TrimSpace(c.FirstName + " " + c.LastName) + } + addr := mail.Address{Name: name, Address: c.Email} return mailhdr.Address(addr.String()) } diff --git a/internal/client/msgraph/send.go b/internal/client/msgraph/send.go index 581bbc53..e5c1b6bc 100644 --- a/internal/client/msgraph/send.go +++ b/internal/client/msgraph/send.go @@ -33,6 +33,7 @@ import ( // customHeaders is variadic to mirror goog.Client.SendMessage. func (c *Client) SendMessage( ctx context.Context, + fromName string, to, cc, bcc []string, messageID, subject, bodyPlain, bodyHTML string, @@ -40,7 +41,7 @@ func (c *Client) SendMessage( attachments []Attachment, customHeaders ...map[string]string, ) (string, error) { - raw, err := buildMIME(sendHeaders(c.GetAddress(), to, cc, bcc, "", subject, parent, customHeaders...), bodyPlain, bodyHTML, attachments) + raw, err := buildMIME(sendHeaders(c.FromAddress(fromName), to, cc, bcc, "", subject, parent, customHeaders...), bodyPlain, bodyHTML, attachments) if err != nil { return "", fmt.Errorf("build mime: %w", err) } @@ -51,7 +52,7 @@ func (c *Client) SendMessage( // single-shot path with our own Message-ID. The send still lands; only // the id we learn is lost. log.Warn().Err(err).Str("email", c.Email).Msg("graph draft creation failed; sending without a readable message id") - fallback, berr := buildMIME(sendHeaders(c.GetAddress(), to, cc, bcc, messageID, subject, parent, customHeaders...), bodyPlain, bodyHTML, attachments) + fallback, berr := buildMIME(sendHeaders(c.FromAddress(fromName), to, cc, bcc, messageID, subject, parent, customHeaders...), bodyPlain, bodyHTML, attachments) if berr != nil { return "", fmt.Errorf("build mime: %w", berr) } diff --git a/internal/client/msgraph/send_test.go b/internal/client/msgraph/send_test.go index 2f7355ce..4ab87e0c 100644 --- a/internal/client/msgraph/send_test.go +++ b/internal/client/msgraph/send_test.go @@ -86,6 +86,7 @@ func newSendClient(rt *sendRT) *Client { func send(c *Client, headers map[string]string, parent *models.EmailMessageData) (string, error) { return c.SendMessage( context.Background(), + "", []string{"partner@example.com"}, nil, nil, "", "quick learning question", diff --git a/internal/client/msgraph/token_error_test.go b/internal/client/msgraph/token_error_test.go index dc86c53e..5a562e15 100644 --- a/internal/client/msgraph/token_error_test.go +++ b/internal/client/msgraph/token_error_test.go @@ -110,7 +110,7 @@ func TestRevokedGrantIsAnAuthenticationErrorOnEveryCallPath(t *testing.T) { t.Run("send", func(t *testing.T) { c, _ := tokenRefusal(t, http.StatusBadRequest, invalidGrantBody) - _, err := c.SendMessage(ctx, []string{"partner@example.com"}, nil, nil, + _, err := c.SendMessage(ctx, "", []string{"partner@example.com"}, nil, nil, "", "subject", "body", "", nil, nil, nil) if got := mailErrorOf(t, err).Code; got != errx.MailErrorCodeAuthenticationFailed { t.Errorf("code = %s, want %s", got, errx.MailErrorCodeAuthenticationFailed) diff --git a/internal/client/smtpimap/smtp/client.go b/internal/client/smtpimap/smtp/client.go index a1553cda..90849a4e 100644 --- a/internal/client/smtpimap/smtp/client.go +++ b/internal/client/smtpimap/smtp/client.go @@ -54,6 +54,7 @@ type Attachment struct { // caller that ignores them is unaffected. func (c *Client) Send( ctx context.Context, + fromName string, to, cc, bcc []string, messageID, subject, bodyPlain, bodyHTML, @@ -61,7 +62,13 @@ func (c *Client) Send( attachments []Attachment, customHeaders ...map[string]string, ) ([]byte, *errx.MailError) { - from := mail.Address{Address: c.Email, Name: fmt.Sprintf("%s %s", c.FirstName, c.LastName)} + // A per-send name (the mailbox as renamed in the dashboard) wins over the + // one cached at load time. + fromName = strings.TrimSpace(fromName) + if fromName == "" { + fromName = strings.TrimSpace(c.FirstName + " " + c.LastName) + } + from := mail.Address{Address: c.Email, Name: fromName} // ----- Headers ----- headers := map[string]string{ diff --git a/internal/config/constants.go b/internal/config/constants.go index 9033b24f..3912bce3 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -2,7 +2,6 @@ package config const ( DefaultColor = "#c4c8cf" - Domain = "warmbly.com" // LimitMin/LimitMax bound every per-mailbox and per-campaign daily send // cap the API will store. 5000 covers real provider ceilings (Google // Workspace 2000/day, M365 10000 recipients/day); the safe cold band @@ -69,9 +68,9 @@ const ( SyncFloodPerHour = 5_000 // new live messages observed in one hour that mark a mailbox as flooding SyncThrottleEscalationDays = 3 // throttled UTC days out of the last 7 that deactivate a mailbox - // Forms. Funnel events feed analytics ranges up to 90 days; the fixed - // retention window keeps double coverage without a per-org setting. - FormEventsRetentionDays = 180 + // Forms. Funnel events feed analytics ranges up to 90 days, so the default + // window keeps double coverage. Operator-editable under Instance settings. + FormEventsRetentionDaysDefault = 180 // Sequences. Empty by default so the editor shows a smart, position-based // label (e.g. "Email 1") until the user names the step themselves. @@ -133,6 +132,38 @@ const ( // it must stay well clear of a slow provider handshake. CampaignSendReclaimAfterMinutes = 30 + // TrackingMachineWindowSeconds is how soon after a step was dispatched an + // open or click is treated as automated rather than a person. The clock + // starts when the send is handed to the worker, before the provider has + // even accepted the message, so a person cannot plausibly have read and + // acted on it inside this window; security gateways that detonate every + // link at delivery time routinely do. + TrackingMachineWindowSeconds = 10 + + // TrackingClickBurstSeconds is the window inside which clicks on two + // different links of the same email from the same source are treated as + // a scanner walking the message. A person follows one link at a time. + TrackingClickBurstSeconds = 5 + + // EngagementEventRetentionDaysDefault is how long the per-event open and + // click logs (client, device, location) are kept. The summary on the + // progress row outlives them, so counts and routing never change. + // Operator-editable under Instance settings. + EngagementEventRetentionDaysDefault = 365 + + // AuditLogRetentionDaysDefault is how long the audit trail is kept. The + // trail carries IP addresses, user agents and change payloads, so this + // window is also how long that PII is held; a privacy-conscious operator + // shortens it under Instance settings. + AuditLogRetentionDaysDefault = 90 + + // Ten years is the ceiling every retention window shares. It is not a + // recommendation: it is the point past which "keep it" and "keep it + // forever" stop differing, and it bounds a typo. One day is the floor, so + // there is always a window in which an event can be read. + RetentionDaysMin = 1 + RetentionDaysMax = 3650 + // CampaignSendStampAttempts is how many times the control plane retries the // sent_at stamp after a send is already on the bus. The reservation is what // keeps the step from being re-sent, so a lost stamp is a pacing problem, @@ -207,10 +238,9 @@ const ( WarmupVerifyHeader = "X-Mailtrace-Verify" // Product-level hard caps. These are the backstop for plans that - // advertise "unlimited" — marketing can keep saying unlimited, but - // the runtime never grants truly unbounded usage. Each cap is the - // floor that GetEffectiveLimits falls back to when both the - // per-org override and the plan column are unset. + // advertise "unlimited" on campaigns, seats, contacts and daily sends. + // Each cap is the floor that GetEffectiveLimits falls back to when both + // the per-org override and the plan column are unset. // // Admins can grant strictly larger caps per-org through the // override flow when there is a legitimate business reason. Growth @@ -219,15 +249,31 @@ const ( // acknowledging the new ceiling. // // These numbers are deliberately generous enough that ordinary use - // never trips them, and conservative enough that "I want to spin up - // 5,000 mailboxes overnight" can't happen without explicit approval. - HardCapMailboxes = 200 // total connected mailboxes per org + // never trips them. Mailboxes are not in this list: see + // FairUseSendsPerMailbox below. HardCapCampaignsTotal = 500 // total campaigns ever created HardCapCampaignsActive = 50 // simultaneously active campaigns HardCapTeamMembers = 100 // seats per org HardCapContacts = 1_000_000 // contacts per org HardCapDailyCampaignSends = 1000 // campaign emails per org per day + // Mailboxes have no hard cap. A paid workspace's allowance is fair use + // derived from the daily sends its plan includes: one mailbox for every + // FairUseSendsPerMailbox sends a day. At 1 a 15,000/day plan holds 15,000 + // mailboxes, one per daily send, which is deliberately far more than safe + // sending ever needs: the allowance must never be the reason a customer + // runs a mailbox hotter. A plan with no daily send cap holds unlimited + // mailboxes, and an approved limit-increase request raises the allowance + // for one workspace. + FairUseSendsPerMailbox = 1 + + // Bulk connect: how many SMTP/IMAP rows one request may carry, and how + // many of them are validated against a worker at the same time. The + // dashboard streams a CSV through batches of this size so a 3,000 row + // file shows live progress instead of one request that times out. + MailboxBulkBatchMax = 50 + MailboxBulkConcurrency = 8 + // Daily creation throttles. The total caps above stop "you have // 5000 campaigns on this org" — the throttles below stop "you // created 1000 campaigns today on a fresh unlimited account." @@ -239,7 +285,6 @@ const ( // because the per-day shape protects abuse posture rather than // product utility. DailyThrottleNewCampaigns = 20 // new campaigns per org per day - DailyThrottleNewMailboxes = 5 // newly connected mailboxes per org per day // Pool link: mailboxes a self-hosted instance may enroll in the hosted // warmup pool without a paid pool plan, and the handshake lifetimes. @@ -251,6 +296,11 @@ const ( WarmupPoolFallbackMinAgeDays = 3 // other-tier mailboxes must be this old before they fill in DailyThrottleNewOrgs = 3 // new workspaces per owner per day + // CLI sign-in handshake (`warmbly auth login`). Shorter-lived than the pool + // link handshake because a person is watching the terminal while it runs. + CLIAuthCodeTTLMinutes = 10 + CLIAuthPollIntervalSeconds = 3 + // DailyThrottleNewScheduledSends caps how many NEW scheduled-send // schedules a single user can create in a rolling 24h window. The // real defense against burst abuse — someone writing a loop that diff --git a/internal/config/endpoints.go b/internal/config/endpoints.go index de68563d..c2d3f844 100644 --- a/internal/config/endpoints.go +++ b/internal/config/endpoints.go @@ -23,6 +23,30 @@ func AppBaseURL() string { return "https://app.warmbly.com" } +// WebsocketURL is the realtime gateway clients connect to. It is deployment +// configuration rather than a secret, which is why GET /v1/auth/config serves +// it: a CLI or a developer client cannot otherwise find the socket on a +// self-hosted instance, where the host layout is whatever the operator chose. +func WebsocketURL() string { + v := strings.TrimRight(strings.TrimSpace(os.Getenv("WEBSOCKET_URL")), "/") + if v == "" { + return "" + } + // The variable is written three ways in the wild: a bare host, the Phoenix + // socket mount (".../socket"), and the full transport endpoint. Clients + // dial what this returns, so all three normalise to the last one. Matching + // on a "/socket" substring instead of the suffix left ".../socket" + // untouched, which is not a websocket endpoint. + switch { + case strings.HasSuffix(v, "/socket/websocket"): + case strings.HasSuffix(v, "/socket"): + v += "/websocket" + default: + v += "/socket/websocket" + } + return v +} + func GetPasswordResetURL(sessionToken string) string { return AppBaseURL() + "/auth/reset-password/confirm?session=" + url.QueryEscape(sessionToken) } diff --git a/internal/config/endpoints_test.go b/internal/config/endpoints_test.go new file mode 100644 index 00000000..ca9c7e4d --- /dev/null +++ b/internal/config/endpoints_test.go @@ -0,0 +1,23 @@ +package config + +import "testing" + +// Clients dial whatever GET /v1/auth/config advertises, so every form an +// operator plausibly writes has to normalise to the Phoenix transport path. +func TestWebsocketURLNormalisation(t *testing.T) { + cases := map[string]string{ + "wss://ws.example.com": "wss://ws.example.com/socket/websocket", + "wss://ws.example.com/": "wss://ws.example.com/socket/websocket", + "wss://ws.example.com/socket": "wss://ws.example.com/socket/websocket", + "wss://ws.example.com/socket/": "wss://ws.example.com/socket/websocket", + "wss://ws.example.com/socket/websocket": "wss://ws.example.com/socket/websocket", + "ws://localhost:4000/socket/websocket": "ws://localhost:4000/socket/websocket", + "": "", + } + for in, want := range cases { + t.Setenv("WEBSOCKET_URL", in) + if got := WebsocketURL(); got != want { + t.Errorf("WebsocketURL(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/errx/common.go b/internal/errx/common.go index eef701fc..28bebd90 100644 --- a/internal/errx/common.go +++ b/internal/errx/common.go @@ -119,8 +119,6 @@ var ( ErrEmailOnboardUserInfo = New(BadRequest, "Could not read account details from the provider.") ErrEmailOnboardAlreadyExists = New(Conflict, "This email account is already connected.") ErrEmailOnboardNoWorker = New(ServiceUnavailable, "No mailbox workers are available right now. Please try again shortly.") - ErrEmailOnboardInboxLimit = New(Forbidden, "A free workspace holds up to 10 mailboxes. Subscribe to add more.") - ErrEmailOnboardTrialExpired = New(Forbidden, "A free workspace holds up to 10 mailboxes. Subscribe to add more.") ErrEmailReauthProvider = New(BadRequest, "This mailbox connects with SMTP/IMAP credentials. Update its credentials instead of re-authorizing.") ErrEmailReauthOAuthOnly = New(BadRequest, "This mailbox signs in with OAuth. Re-authorize it instead of entering credentials.") ErrEmailReauthWrongAccount = New(Conflict, "The account you signed in with is not this mailbox's address. Sign in with the mailbox's own account and try again.") @@ -193,3 +191,24 @@ var ( ErrAdvisorFixForbidden = New(Forbidden, "You can see this recommendation but don't have permission to apply the change it makes.") ErrAdvisorNoAgentFix = New(BadRequest, "This recommendation needs a person: there's no change an agent can safely make for it.") ) + +// MailboxAllowanceReached is the refusal every connect path returns when the +// workspace holds its whole allowance. The identifier is stable so the +// dashboard can open the request-more flow instead of showing the text. +func MailboxAllowanceReached(used, allowance int, paid bool) *Error { + if !paid { + return NewWithIdentifier(Forbidden, "mailbox_allowance_reached", + fmt.Sprintf("A free workspace holds up to %d mailboxes. Choose a plan to add more.", allowance)) + } + return NewWithIdentifier(Forbidden, "mailbox_allowance_reached", + fmt.Sprintf("This workspace holds %d of its %d mailboxes. Request an increase, or move to a plan with more daily sends.", used, allowance)) +} + +// StorageLimitReached is the refusal for an attachment upload or copy that +// would take the workspace past its storage quota. +func StorageLimitReached(usedBytes, limitBytes, addingBytes int64) *Error { + const mb = 1024 * 1024 + return NewWithIdentifier(BadRequest, "storage_limit_reached", + fmt.Sprintf("Storage limit reached: %d MB of %d MB used, %d MB to add. Remove attachments or upgrade your plan.", + usedBytes/mb, limitBytes/mb, addingBytes/mb)) +} diff --git a/internal/errx/errx.go b/internal/errx/errx.go index a3b9cda8..6c777c90 100644 --- a/internal/errx/errx.go +++ b/internal/errx/errx.go @@ -44,6 +44,10 @@ func (e *Error) identifier() string { return codeToIdentifier[e.Code] } +// ResponseCode is the machine-readable `code` this error answers with, for +// callers that embed errors in a body of their own (per-row results). +func (e *Error) ResponseCode() string { return e.identifier() } + // --- Predefined errors (exported) --- var ( ErrUnauthorized = New(Unauthorized, "Token not found.") diff --git a/internal/events/publisher.go b/internal/events/publisher.go index d865af7f..1f83df55 100644 --- a/internal/events/publisher.go +++ b/internal/events/publisher.go @@ -66,6 +66,10 @@ type SendEmailParams struct { // object (reached by the worker via BodyS3Key). They are deliberately NOT // added to models.SendEmail / the Avro event — the Kafka contract is fixed. Attachments []models.AttachmentRef + // FromName is the mailbox display name at publish time. It travels in the + // emsg blob for the same reason attachments do, and lets a renamed mailbox + // send under its new name without a worker reload. + FromName string } type publisher struct { @@ -100,7 +104,7 @@ func (p *publisher) PublishSendEmail(ctx context.Context, workerID uuid.UUID, pa // would be published body-less and fail there. return fmt.Errorf("object storage not configured; cannot hand send %s to a worker", params.TaskID) } - s3Key, err := p.storeEmailBody(ctx, params.TaskID, params.OrgID, params.BodyPlain, params.BodyHTML, params.Attachments) + s3Key, err := p.storeEmailBody(ctx, params.TaskID, params.OrgID, params.BodyPlain, params.BodyHTML, params.Attachments, params.FromName) if err != nil { return fmt.Errorf("failed to store email body: %w", err) } @@ -163,14 +167,15 @@ func (p *publisher) PublishSendEmail(ctx context.Context, workerID uuid.UUID, pa // StoreEmailBody stores email body in S3 and returns the S3 key. It is the // interface method; the attachment-aware path goes through storeEmailBody. func (p *publisher) StoreEmailBody(ctx context.Context, taskID, orgID uuid.UUID, plainText, htmlBody string) (string, error) { - return p.storeEmailBody(ctx, taskID, orgID, plainText, htmlBody, nil) + return p.storeEmailBody(ctx, taskID, orgID, plainText, htmlBody, nil, "") } // storeEmailBody encodes the email body plus attachment refs into the emsg blob // and uploads it to object storage, returning the S3 key. Bodies are encrypted -// with the organization DEK before encoding; attachment refs are plaintext metadata (the bytes -// they point to are stored separately and the worker fetches them by key). -func (p *publisher) storeEmailBody(ctx context.Context, taskID, orgID uuid.UUID, plainText, htmlBody string, attachments []models.AttachmentRef) (string, error) { +// with the organization DEK before encoding; attachment refs and the from name +// are plaintext metadata (the bytes refs point to are stored separately and the +// worker fetches them by key). +func (p *publisher) storeEmailBody(ctx context.Context, taskID, orgID uuid.UUID, plainText, htmlBody string, attachments []models.AttachmentRef, fromName string) (string, error) { if p.storageClient == nil { return "", nil } @@ -201,6 +206,7 @@ func (p *publisher) storeEmailBody(ctx context.Context, taskID, orgID uuid.UUID, blob := &emsg.EmailBlob{ PlainText: []byte(encPlainText), HTMLBody: []byte(encHTMLBody), + FromName: fromName, } for _, a := range attachments { blob.Attachments = append(blob.Attachments, emsg.Attachment{ diff --git a/internal/events/schemas.go b/internal/events/schemas.go index 1aeb1cd4..728e6a10 100644 --- a/internal/events/schemas.go +++ b/internal/events/schemas.go @@ -53,7 +53,14 @@ type TrackingEvent struct { EventType string `json:"event_type" avro:"event_type"` // EMAIL_OPENED or EMAIL_CLICKED TaskID string `json:"task_id" avro:"task_id"` // UUID string OriginalURL *string `json:"original_url" avro:"original_url"` // For click events only (nullable) + LinkID *string `json:"link_id" avro:"link_id"` // Click ticket id (nullable; absent from older tracking builds) Timestamp string `json:"timestamp" avro:"timestamp"` // ISO8601 timestamp UserAgent *string `json:"user_agent" avro:"user_agent"` // Browser user agent (nullable) IPHash *string `json:"ip_hash" avro:"ip_hash"` // Hashed IP for privacy (nullable) + // ClientIP is the source network, not the address: the edge zeroes the + // last IPv4 octet (or everything past the first 48 IPv6 bits) before + // publishing, so what the bus retains cannot single out a host. The + // consumer resolves it to a location and does not store it. Nullable and + // absent from events written before the field existed. + ClientIP *string `json:"client_ip" avro:"client_ip"` } diff --git a/internal/infrastructure/db/migrations/000122_campaign_kind.down.sql b/internal/infrastructure/db/migrations/000122_campaign_kind.down.sql new file mode 100644 index 00000000..ac22b091 --- /dev/null +++ b/internal/infrastructure/db/migrations/000122_campaign_kind.down.sql @@ -0,0 +1 @@ +ALTER TABLE campaigns DROP COLUMN IF EXISTS kind; diff --git a/internal/infrastructure/db/migrations/000122_campaign_kind.up.sql b/internal/infrastructure/db/migrations/000122_campaign_kind.up.sql new file mode 100644 index 00000000..1983dbe3 --- /dev/null +++ b/internal/infrastructure/db/migrations/000122_campaign_kind.up.sql @@ -0,0 +1,5 @@ +-- A campaign is either a multi-step sequence or a one-time email: one message +-- to an audience, no follow-ups. Both send through the same pacer; the kind +-- only changes what the editor allows and how status reads in the list. +ALTER TABLE campaigns ADD COLUMN kind text NOT NULL DEFAULT 'sequence' + CHECK (kind IN ('sequence', 'one_time')); diff --git a/internal/infrastructure/db/migrations/000123_link_clicks.down.sql b/internal/infrastructure/db/migrations/000123_link_clicks.down.sql new file mode 100644 index 00000000..7f43a2ef --- /dev/null +++ b/internal/infrastructure/db/migrations/000123_link_clicks.down.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS email_link_clicks; + +ALTER TABLE tracked_links + DROP COLUMN IF EXISTS label; + +ALTER TABLE campaigns + DROP COLUMN IF EXISTS utm_tracking, + DROP COLUMN IF EXISTS utm_source, + DROP COLUMN IF EXISTS utm_medium, + DROP COLUMN IF EXISTS utm_campaign; diff --git a/internal/infrastructure/db/migrations/000123_link_clicks.up.sql b/internal/infrastructure/db/migrations/000123_link_clicks.up.sql new file mode 100644 index 00000000..0280b20f --- /dev/null +++ b/internal/infrastructure/db/migrations/000123_link_clicks.up.sql @@ -0,0 +1,38 @@ +-- Per-link click attribution and automatic UTM tagging. +-- +-- Every click on a tracked link is logged as its own row so the contact +-- timeline can say WHICH link was clicked, not only that one was. A click +-- that looks automated (a security gateway following every link seconds +-- after delivery) is kept for the record but flagged, and it never stamps +-- campaign_contact_progress.clicked_at, so "clicked" keeps meaning a person. + +ALTER TABLE campaigns + ADD COLUMN IF NOT EXISTS utm_tracking boolean NOT NULL DEFAULT false, + ADD COLUMN IF NOT EXISTS utm_source text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS utm_medium text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS utm_campaign text NOT NULL DEFAULT ''; + +-- The anchor text the link was minted from ("Pricing"), so a click can be +-- named without re-parsing the email. +ALTER TABLE tracked_links + ADD COLUMN IF NOT EXISTS label text NOT NULL DEFAULT ''; + +CREATE TABLE IF NOT EXISTS email_link_clicks ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + tracked_link_id uuid REFERENCES tracked_links(id) ON DELETE SET NULL, + task_id uuid NOT NULL, + campaign_id uuid NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE, + contact_id uuid NOT NULL REFERENCES contacts(id) ON DELETE CASCADE, + sequence_id uuid NOT NULL REFERENCES sequences(id) ON DELETE CASCADE, + destination text NOT NULL, + label text NOT NULL DEFAULT '', + user_agent text NOT NULL DEFAULT '', + ip_hash text NOT NULL DEFAULT '', + machine boolean NOT NULL DEFAULT false, + machine_reason text NOT NULL DEFAULT '' CHECK (machine_reason IN ('', 'prefetch', 'instant', 'burst')), + clicked_at timestamp with time zone NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_email_link_clicks_contact ON email_link_clicks (contact_id, clicked_at DESC); +CREATE INDEX IF NOT EXISTS idx_email_link_clicks_task ON email_link_clicks (task_id, clicked_at DESC); +CREATE INDEX IF NOT EXISTS idx_email_link_clicks_step ON email_link_clicks (campaign_id, contact_id, sequence_id); diff --git a/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.down.sql b/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.down.sql new file mode 100644 index 00000000..8b1593e4 --- /dev/null +++ b/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.down.sql @@ -0,0 +1,6 @@ +DROP FUNCTION IF EXISTS recipient_suppressed(uuid, text); +DELETE FROM suppressed_recipients WHERE kind = 'domain'; +ALTER TABLE suppressed_recipients DROP CONSTRAINT IF EXISTS suppressed_recipients_kind_check; +ALTER TABLE suppressed_recipients DROP COLUMN IF EXISTS kind; +ALTER TABLE campaigns DROP CONSTRAINT IF EXISTS campaigns_unsubscribe_mode_check; +ALTER TABLE campaigns DROP COLUMN IF EXISTS unsubscribe_mode; diff --git a/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.up.sql b/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.up.sql new file mode 100644 index 00000000..64539d45 --- /dev/null +++ b/internal/infrastructure/db/migrations/000124_unsubscribe_opt_out.up.sql @@ -0,0 +1,43 @@ +-- A campaign can override the workspace's in-body opt-out (text line, link, +-- or none); 'inherit' follows Settings > Sending. +-- The new column's default satisfies the check on every existing row, so the +-- constraints are added NOT VALID: they bind every new write without the +-- table scan under ACCESS EXCLUSIVE that a validating add would take. +ALTER TABLE campaigns ADD COLUMN unsubscribe_mode text NOT NULL DEFAULT 'inherit'; +ALTER TABLE campaigns ADD CONSTRAINT campaigns_unsubscribe_mode_check + CHECK (unsubscribe_mode IN ('inherit', 'text', 'link', 'off')) NOT VALID; + +-- The suppression list takes whole domains as well as addresses. A domain +-- row keeps the bare host in email ("acme.com") so the existing unique key +-- and index apply unchanged. +ALTER TABLE suppressed_recipients ADD COLUMN kind text NOT NULL DEFAULT 'email'; +ALTER TABLE suppressed_recipients ADD CONSTRAINT suppressed_recipients_kind_check + CHECK (kind IN ('email', 'domain')) NOT VALID; + +-- Every write lowercases the value, so the stored email is the identity and +-- the existing (organization_id, email) unique key and index serve every +-- lookup as an equality. Rows written before that rule was strict are folded +-- here: of two entries that differ only by case, the newer one stays. +DELETE FROM suppressed_recipients a +USING suppressed_recipients b +WHERE a.organization_id = b.organization_id + AND lower(a.email) = lower(b.email) + AND a.id <> b.id + AND (a.updated_at, a.id) < (b.updated_at, b.id); +UPDATE suppressed_recipients SET email = lower(email) WHERE email <> lower(email); + +-- One predicate for every send gate, count and filter, so a domain entry is +-- honoured everywhere an address entry is. Written as a single SELECT so the +-- planner inlines it like the subqueries it replaces. +CREATE OR REPLACE FUNCTION recipient_suppressed(org uuid, addr text) RETURNS boolean +LANGUAGE sql STABLE AS $$ + SELECT EXISTS ( + SELECT 1 FROM suppressed_recipients sr + WHERE sr.organization_id = org + AND (sr.expires_at IS NULL OR sr.expires_at > now()) + AND ( + (sr.kind = 'email' AND sr.email = lower(addr)) + OR (sr.kind = 'domain' AND sr.email = split_part(lower(addr), '@', 2)) + ) + ) +$$; diff --git a/internal/infrastructure/db/migrations/000125_engagement_origin.down.sql b/internal/infrastructure/db/migrations/000125_engagement_origin.down.sql new file mode 100644 index 00000000..288652a3 --- /dev/null +++ b/internal/infrastructure/db/migrations/000125_engagement_origin.down.sql @@ -0,0 +1,12 @@ +DROP TABLE email_opens; +ALTER TABLE email_link_clicks + DROP COLUMN announce_claimed_at, + DROP COLUMN announce_pending, + DROP COLUMN client, + DROP COLUMN device_type, + DROP COLUMN os, + DROP COLUMN browser, + DROP COLUMN browser_version, + DROP COLUMN country_code, + DROP COLUMN region, + DROP COLUMN city; diff --git a/internal/infrastructure/db/migrations/000125_engagement_origin.up.sql b/internal/infrastructure/db/migrations/000125_engagement_origin.up.sql new file mode 100644 index 00000000..dfbf97a3 --- /dev/null +++ b/internal/infrastructure/db/migrations/000125_engagement_origin.up.sql @@ -0,0 +1,47 @@ +-- Where an open or click came from: the mail client or image proxy, the +-- browser, device and operating system parsed from the user agent, and the +-- country, region and city resolved from the source network. Clicks gain +-- these columns on the per-link log. Opens get their own log, because the +-- progress row keeps only the first open per step and a second open from +-- another device is worth seeing. +ALTER TABLE email_link_clicks + ADD COLUMN client text NOT NULL DEFAULT '', + ADD COLUMN device_type text NOT NULL DEFAULT '', + ADD COLUMN os text NOT NULL DEFAULT '', + ADD COLUMN browser text NOT NULL DEFAULT '', + ADD COLUMN browser_version text NOT NULL DEFAULT '', + ADD COLUMN country_code text NOT NULL DEFAULT '', + ADD COLUMN region text NOT NULL DEFAULT '', + ADD COLUMN city text NOT NULL DEFAULT '', + -- A person's click waits out the burst window before its effects fire; + -- the row is the durable record of that pending work, so a consumer + -- restart inside the window loses nothing. The flag clears only once + -- the effects ran; a claim leases the row for the attempt, and a lease + -- that expires without completion is retried. + ADD COLUMN announce_pending boolean NOT NULL DEFAULT false, + ADD COLUMN announce_claimed_at timestamp with time zone; + +CREATE TABLE email_opens ( + id uuid PRIMARY KEY, + task_id uuid NOT NULL, + campaign_id uuid NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE, + contact_id uuid NOT NULL REFERENCES contacts(id) ON DELETE CASCADE, + sequence_id uuid NOT NULL REFERENCES sequences(id) ON DELETE CASCADE, + opened_at timestamp with time zone NOT NULL, + machine boolean NOT NULL DEFAULT false, + machine_reason text NOT NULL DEFAULT '' CHECK (machine_reason IN ('', 'prefetch', 'instant')), + user_agent text NOT NULL DEFAULT '', + ip_hash text NOT NULL DEFAULT '', + client text NOT NULL DEFAULT '', + device_type text NOT NULL DEFAULT '', + os text NOT NULL DEFAULT '', + browser text NOT NULL DEFAULT '', + browser_version text NOT NULL DEFAULT '', + country_code text NOT NULL DEFAULT '', + region text NOT NULL DEFAULT '', + city text NOT NULL DEFAULT '' +); + +CREATE INDEX idx_email_opens_contact ON email_opens (contact_id, opened_at DESC); +CREATE INDEX idx_email_opens_campaign ON email_opens (campaign_id, opened_at DESC); +CREATE INDEX idx_email_opens_step ON email_opens (campaign_id, contact_id, sequence_id); diff --git a/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.down.sql b/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.down.sql new file mode 100644 index 00000000..0b87fd56 --- /dev/null +++ b/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_email_link_clicks_pending; diff --git a/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.up.sql b/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.up.sql new file mode 100644 index 00000000..b709119b --- /dev/null +++ b/internal/infrastructure/db/migrations/000126_link_clicks_pending_index.up.sql @@ -0,0 +1,4 @@ +-- The sweep for held-back click announcements reads only pending rows, so +-- the index is partial. Built concurrently, on its own, because the click +-- log is a live table and a plain CREATE INDEX would block writes to it. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_link_clicks_pending ON email_link_clicks (clicked_at) WHERE announce_pending; diff --git a/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.down.sql b/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.down.sql new file mode 100644 index 00000000..8b091c01 --- /dev/null +++ b/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_campaign_progress_dispatch_task; diff --git a/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.up.sql b/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.up.sql new file mode 100644 index 00000000..937591b4 --- /dev/null +++ b/internal/infrastructure/db/migrations/000127_campaign_progress_dispatch_task_index.up.sql @@ -0,0 +1,8 @@ +-- A campaign task counts as a send only when it holds a step's reservation +-- (issue #306): the chain's wake-ups complete without sending and must not +-- spend the mailbox's daily budget. The counters look the reservation up by +-- its task, so that lookup needs an index. Built concurrently, on its own, +-- because progress is a live table and a plain CREATE INDEX would block it. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_campaign_progress_dispatch_task + ON campaign_contact_progress (dispatch_task_id) + WHERE dispatch_task_id IS NOT NULL; diff --git a/internal/infrastructure/db/migrations/000128_cli_auth.down.sql b/internal/infrastructure/db/migrations/000128_cli_auth.down.sql new file mode 100644 index 00000000..fe55061e --- /dev/null +++ b/internal/infrastructure/db/migrations/000128_cli_auth.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cli_auth_codes; diff --git a/internal/infrastructure/db/migrations/000128_cli_auth.up.sql b/internal/infrastructure/db/migrations/000128_cli_auth.up.sql new file mode 100644 index 00000000..35e772c2 --- /dev/null +++ b/internal/infrastructure/db/migrations/000128_cli_auth.up.sql @@ -0,0 +1,28 @@ +-- CLI sign-in: the `warmbly` CLI has no credential of its own, so it opens a +-- device-code handshake, a signed-in member approves it in the browser, and the +-- approval mints an ordinary API key. The key is the credential; this table +-- only carries the handshake and is empty within minutes. +-- +-- Same shape as pool_link_codes, one row per `warmbly auth login`. + +CREATE TABLE IF NOT EXISTS cli_auth_codes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + device_code_hash text NOT NULL UNIQUE, + user_code text NOT NULL UNIQUE, + -- What the CLI asked for, shown on the approval screen. + client_name text NOT NULL DEFAULT '', + hostname text NOT NULL DEFAULT '', + cli_version text NOT NULL DEFAULT '', + scopes bigint NOT NULL DEFAULT 0, + status text NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'approved', 'claimed', 'denied')), + organization_id uuid REFERENCES organizations (id) ON DELETE CASCADE, + approved_by uuid REFERENCES users (id) ON DELETE SET NULL, + api_key_id uuid REFERENCES api_keys (id) ON DELETE SET NULL, + -- The minted secret, held only between approval and the CLI's next poll. + api_key_secret text, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_cli_auth_codes_expires ON cli_auth_codes (expires_at); diff --git a/internal/infrastructure/db/migrations/000129_contact_source_automation.down.sql b/internal/infrastructure/db/migrations/000129_contact_source_automation.down.sql new file mode 100644 index 00000000..24219c55 --- /dev/null +++ b/internal/infrastructure/db/migrations/000129_contact_source_automation.down.sql @@ -0,0 +1,8 @@ +-- Rows stamped 'automation' fall back to 'unknown' so the narrower CHECK holds. +UPDATE public.contacts SET source = 'unknown' WHERE source = 'automation'; +-- NOT VALID skips the table scan under the migration's lock; the rows already +-- satisfy the set, so nothing needs validating. +ALTER TABLE public.contacts DROP CONSTRAINT contacts_source_check; +ALTER TABLE public.contacts + ADD CONSTRAINT contacts_source_check + CHECK (source IN ('unknown', 'manual', 'campaign', 'import', 'sheet_sync', 'api', 'ai_assistant', 'form')) NOT VALID; diff --git a/internal/infrastructure/db/migrations/000129_contact_source_automation.up.sql b/internal/infrastructure/db/migrations/000129_contact_source_automation.up.sql new file mode 100644 index 00000000..743587b8 --- /dev/null +++ b/internal/infrastructure/db/migrations/000129_contact_source_automation.up.sql @@ -0,0 +1,9 @@ +-- A contact created by an automation's "create or update contact" action is a +-- first-touch origin of its own, so an inbound lead (a Zapier, Make or n8n +-- push, a lead-ads form) reads as what it is instead of "unknown". +-- NOT VALID skips the table scan under the migration's lock; the rows already +-- satisfy the set, so nothing needs validating. +ALTER TABLE public.contacts DROP CONSTRAINT contacts_source_check; +ALTER TABLE public.contacts + ADD CONSTRAINT contacts_source_check + CHECK (source IN ('unknown', 'manual', 'campaign', 'import', 'sheet_sync', 'api', 'ai_assistant', 'form', 'automation')) NOT VALID; diff --git a/internal/infrastructure/db/migrations/000130_campaign_continuous.down.sql b/internal/infrastructure/db/migrations/000130_campaign_continuous.down.sql new file mode 100644 index 00000000..cd33b1d1 --- /dev/null +++ b/internal/infrastructure/db/migrations/000130_campaign_continuous.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE public.campaigns + DROP COLUMN IF EXISTS idle_since, + DROP COLUMN IF EXISTS continuous; diff --git a/internal/infrastructure/db/migrations/000130_campaign_continuous.up.sql b/internal/infrastructure/db/migrations/000130_campaign_continuous.up.sql new file mode 100644 index 00000000..40c5a75e --- /dev/null +++ b/internal/infrastructure/db/migrations/000130_campaign_continuous.up.sql @@ -0,0 +1,9 @@ +-- A continuous campaign stays active when it runs out of leads and waits for +-- more instead of finishing (issue #336). idle_since marks that wait. +ALTER TABLE public.campaigns + ADD COLUMN continuous BOOLEAN NOT NULL DEFAULT false, + ADD COLUMN idle_since TIMESTAMPTZ; + +-- A campaign fed by a linked segment is exactly the case this exists for. +UPDATE public.campaigns SET continuous = true +WHERE id IN (SELECT campaign_id FROM public.campaign_segments); diff --git a/internal/infrastructure/pubsub/events.go b/internal/infrastructure/pubsub/events.go index 1d895b82..1aca48af 100644 --- a/internal/infrastructure/pubsub/events.go +++ b/internal/infrastructure/pubsub/events.go @@ -35,6 +35,9 @@ const ( EventCampaignStarted EventType = "CAMPAIGN_STARTED" EventCampaignPaused EventType = "CAMPAIGN_PAUSED" EventCampaignCompleted EventType = "CAMPAIGN_COMPLETED" + // EventCampaignIdle: a continuous campaign ran out of leads and stays + // active waiting for more (Status stays "active"). + EventCampaignIdle EventType = "CAMPAIGN_IDLE" // Email account events EventAccountConnected EventType = "ACCOUNT_CONNECTED" @@ -217,9 +220,19 @@ type TrackingEventPayload struct { ContactEmail string `json:"contact_email,omitempty"` SequenceID string `json:"step_id,omitempty"` OriginalURL string `json:"original_url,omitempty"` // For click events - // Machine marks an automated open (Apple MPP prefetch, UA-less fetcher) - // so live views can badge it instead of presenting it as a human open. + LinkLabel string `json:"link_label,omitempty"` // Anchor text of the clicked link + // Machine marks an automated open or click (Apple MPP prefetch, UA-less + // fetcher, a security gateway walking the links) so live views can badge + // it instead of presenting it as a person's. Machine bool `json:"machine,omitempty"` + // OccurredAt is when the tracking service saw the open or click; the + // base timestamp is when this event was published. + OccurredAt time.Time `json:"occurred_at,omitempty"` + // Where and on what, from the engagement logs, for live feeds. + Client string `json:"client,omitempty"` + DeviceType string `json:"device_type,omitempty"` + CountryCode string `json:"country_code,omitempty"` + City string `json:"city,omitempty"` } // PageHitEvent is a website page view tied to a contact. diff --git a/internal/jobs/audit_retention.go b/internal/jobs/audit_retention.go index a56222d1..9e129d1f 100644 --- a/internal/jobs/audit_retention.go +++ b/internal/jobs/audit_retention.go @@ -5,33 +5,42 @@ import ( "time" "github.com/getsentry/sentry-go" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/repository" ) // AuditRetentionJob deletes audit-log entries older than the retention window. // Bounding the trail's age also bounds how long PII (IP addresses, user agents, -// change payloads) is retained, which is a privacy-positive property. +// change payloads) is retained, which is a privacy-positive property. The +// window is an instance setting, read on every pass. type AuditRetentionJob struct { repo repository.AuditRepository - retention time.Duration + retention RetentionSource } -// NewAuditRetentionJob creates a retention job that prunes entries older than -// the given retention window. -func NewAuditRetentionJob(repo repository.AuditRepository, retention time.Duration) *AuditRetentionJob { - return &AuditRetentionJob{ - repo: repo, - retention: retention, - } +// NewAuditRetentionJob creates a retention job for the audit trail. Without a +// wired settings source it prunes at the compiled default window. +func NewAuditRetentionJob(repo repository.AuditRepository) *AuditRetentionJob { + return &AuditRetentionJob{repo: repo} +} + +// WireRetention attaches the instance settings the window is read from. +func (j *AuditRetentionJob) WireRetention(src RetentionSource) *AuditRetentionJob { + j.retention = src + return j } // Run executes one pruning pass. func (j *AuditRetentionJob) Run(ctx context.Context) error { - if j.repo == nil || j.retention <= 0 { + if j.repo == nil { return nil } + days := config.AuditLogRetentionDaysDefault + if j.retention != nil { + days = j.retention.RetentionWindows(ctx).AuditLogDays + } - cutoff := time.Now().Add(-j.retention) + cutoff := time.Now().AddDate(0, 0, -days) if _, err := j.repo.PruneOlderThan(ctx, cutoff); err != nil { sentry.CaptureException(err) return err diff --git a/internal/jobs/form_events_retention.go b/internal/jobs/form_events_retention.go index d8aab24c..9d75f525 100644 --- a/internal/jobs/form_events_retention.go +++ b/internal/jobs/form_events_retention.go @@ -8,22 +8,34 @@ import ( "github.com/warmbly/warmbly/internal/repository" ) -// FormEventsRetentionJob prunes funnel events past the platform window; the -// forms analytics ranges top out at 90 days, so the fixed window keeps double -// coverage without a per-org setting. +// FormEventsRetentionJob prunes funnel events past the retention window. The +// window is an instance setting, read on every pass, so an operator who +// shortens it in the admin panel sees the next sweep honour it. type FormEventsRetentionJob struct { - repo repository.FormEventRepository + repo repository.FormEventRepository + retention RetentionSource } func NewFormEventsRetentionJob(repo repository.FormEventRepository) *FormEventsRetentionJob { return &FormEventsRetentionJob{repo: repo} } +// WireRetention attaches the instance settings the window is read from. Unset +// keeps the compiled default. +func (j *FormEventsRetentionJob) WireRetention(src RetentionSource) *FormEventsRetentionJob { + j.retention = src + return j +} + func (j *FormEventsRetentionJob) Run(ctx context.Context) error { if j.repo == nil { return nil } - before := time.Now().AddDate(0, 0, -config.FormEventsRetentionDays) + days := config.FormEventsRetentionDaysDefault + if j.retention != nil { + days = j.retention.RetentionWindows(ctx).FormEventDays + } + before := time.Now().AddDate(0, 0, -days) if _, xerr := j.repo.PruneBefore(ctx, before); xerr != nil { return xerr } diff --git a/internal/jobs/retention.go b/internal/jobs/retention.go new file mode 100644 index 00000000..7f08ad1f --- /dev/null +++ b/internal/jobs/retention.go @@ -0,0 +1,15 @@ +package jobs + +import ( + "context" + + "github.com/warmbly/warmbly/internal/app/instancesettings" +) + +// RetentionSource is the operator-editable retention section, satisfied by +// instancesettings.Service. The retention jobs read it on every pass rather +// than capturing a window at construction, so shortening one in the admin +// panel takes effect on the next sweep instead of at the next restart. +type RetentionSource interface { + RetentionWindows(ctx context.Context) instancesettings.Retention +} diff --git a/internal/jobs/warmup_batch_poller.go b/internal/jobs/warmup_batch_poller.go index 4e364f0a..e57fe89a 100644 --- a/internal/jobs/warmup_batch_poller.go +++ b/internal/jobs/warmup_batch_poller.go @@ -10,8 +10,8 @@ import ( ) // WarmupBatchPoller reconciles in-flight OpenAI Batch API warmup-generation jobs: -// it polls each active batch, ingests completed ones into the content bank, and -// marks failed/expired/cancelled ones. It is a thin scheduler around +// it polls each active batch, ingests what each finished one produced into the +// content bank, and marks empty ones failed. It is a thin scheduler around // warmupcontent.Service.PollBatches; all policy lives in the service. Batches run // async (up to a 24h window) so a coarse 5-minute tick is plenty. type WarmupBatchPoller struct { diff --git a/internal/models/advanced_outreach.go b/internal/models/advanced_outreach.go index 4cb38d51..5400e287 100644 --- a/internal/models/advanced_outreach.go +++ b/internal/models/advanced_outreach.go @@ -2,6 +2,7 @@ package models import ( "math" + "strings" "time" "github.com/google/uuid" @@ -77,6 +78,86 @@ func (s *AdvancedOutreachSettings) Normalize() { if s.Preflight.MinContentScore < 1 { s.Preflight.MinContentScore = 1 } + if !ValidUnsubscribeMode(string(s.Unsubscribe.Mode)) || s.Unsubscribe.Mode == UnsubscribeModeInherit { + s.Unsubscribe.Mode = UnsubscribeModeText + } + s.Unsubscribe.Text = clampLine(s.Unsubscribe.Text) + s.Unsubscribe.LinkIntro = clampLine(s.Unsubscribe.LinkIntro) + s.Unsubscribe.LinkText = clampLine(s.Unsubscribe.LinkText) +} + +// clampLine trims a one-line copy field and caps it; the email footer is not +// the place for a paragraph or for line breaks. +func clampLine(v string) string { + v = strings.Join(strings.Fields(v), " ") + if r := []rune(v); len(r) > UnsubscribeCopyMaxLen { + v = string(r[:UnsubscribeCopyMaxLen]) + } + return v +} + +// UnsubscribeMode is how a campaign email carries its opt-out. "text" appends +// a plain sentence inviting a reply (the default: it reads as a personal +// email and the reply is honoured automatically), "link" appends a sentence +// with a real unsubscribe link, "off" appends nothing. A campaign's own +// column may also hold "inherit", which follows the workspace setting. +type UnsubscribeMode string + +const ( + UnsubscribeModeInherit UnsubscribeMode = "inherit" + UnsubscribeModeText UnsubscribeMode = "text" + UnsubscribeModeLink UnsubscribeMode = "link" + UnsubscribeModeOff UnsubscribeMode = "off" +) + +// ValidUnsubscribeMode reports whether m is a value a campaign may store. +func ValidUnsubscribeMode(m string) bool { + switch UnsubscribeMode(m) { + case UnsubscribeModeInherit, UnsubscribeModeText, UnsubscribeModeLink, UnsubscribeModeOff: + return true + } + return false +} + +const ( + DefaultUnsubscribeText = "If this isn't relevant, just reply and let me know and I won't email you again." + DefaultUnsubscribeLinkIntro = "Not the right person, or not interested?" + DefaultUnsubscribeLinkText = "Unsubscribe" + UnsubscribeCopyMaxLen = 300 +) + +// UnsubscribeSettings is the workspace default for the in-body opt-out. The +// List-Unsubscribe header is a per-campaign flag and is not part of this. +type UnsubscribeSettings struct { + Mode UnsubscribeMode `json:"mode"` + // Text is the sentence appended in "text" mode. + Text string `json:"text"` + // LinkIntro and LinkText make up the "link" mode line: " text". + LinkIntro string `json:"link_intro"` + LinkText string `json:"link_text"` +} + +// Effective resolves a campaign's stored mode against the workspace default +// and fills any blank copy with the defaults, so the send path never has to +// think about settings written before this block existed. +func (u UnsubscribeSettings) Effective(campaignMode string) UnsubscribeSettings { + out := u + if m := UnsubscribeMode(campaignMode); m != "" && m != UnsubscribeModeInherit && ValidUnsubscribeMode(campaignMode) { + out.Mode = m + } + if !ValidUnsubscribeMode(string(out.Mode)) || out.Mode == UnsubscribeModeInherit { + out.Mode = UnsubscribeModeText + } + if strings.TrimSpace(out.Text) == "" { + out.Text = DefaultUnsubscribeText + } + if strings.TrimSpace(out.LinkIntro) == "" { + out.LinkIntro = DefaultUnsubscribeLinkIntro + } + if strings.TrimSpace(out.LinkText) == "" { + out.LinkText = DefaultUnsubscribeLinkText + } + return out } type DeliverabilityDashboardSettings struct { @@ -94,6 +175,7 @@ type AdvancedOutreachSettings struct { SendTimeOptimization SendTimeOptimizationSettings `json:"send_time_optimization"` Preflight PreflightValidationSettings `json:"preflight"` Dashboard DeliverabilityDashboardSettings `json:"dashboard"` + Unsubscribe UnsubscribeSettings `json:"unsubscribe"` Custom map[string]interface{} `json:"custom,omitempty"` } @@ -116,8 +198,48 @@ const ( DeliverabilityEventOpen DeliverabilityEventType = "open" DeliverabilityEventClick DeliverabilityEventType = "click" DeliverabilityEventReply DeliverabilityEventType = "reply" + + // Suppression-only sources: never ingested as deliverability events, but + // they share the column so one list explains why every entry is there. + SuppressionSourceManual DeliverabilityEventType = "manual" + SuppressionSourceImport DeliverabilityEventType = "import" ) +// SuppressionKind says what a suppression row matches: one address, or every +// address at a domain. +type SuppressionKind string + +const ( + SuppressionKindEmail SuppressionKind = "email" + SuppressionKindDomain SuppressionKind = "domain" +) + +// SuppressionListResult is the GET /suppressions page. +type SuppressionListResult struct { + Data []SuppressedRecipient `json:"data"` + Pagination CPagination `json:"pagination"` +} + +// AddSuppressionsRequest adds addresses and domains by hand or from a pasted +// list. A value without "@" (or with a leading "@") is a domain. +type AddSuppressionsRequest struct { + Entries []SuppressionEntry `json:"entries"` + // Reason applies to every entry that does not carry its own. + Reason string `json:"reason"` +} + +type SuppressionEntry struct { + Value string `json:"value"` + Reason string `json:"reason,omitempty"` +} + +// AddSuppressionsResult reports what the request did. Skipped lists the +// values that were neither a valid address nor a valid domain. +type AddSuppressionsResult struct { + Added int `json:"added"` + Skipped []string `json:"skipped"` +} + type DeliverabilityEvent struct { ID uuid.UUID `json:"id"` OrganizationID uuid.UUID `json:"organization_id"` @@ -146,16 +268,18 @@ type IngestDeliverabilityEventRequest struct { } type SuppressedRecipient struct { - ID uuid.UUID `json:"id"` - OrganizationID uuid.UUID `json:"organization_id"` - Email string `json:"email"` - Reason string `json:"reason"` - Source DeliverabilityEventType `json:"source"` - CampaignID *uuid.UUID `json:"campaign_id,omitempty"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + // Email holds the address, or the bare domain when Kind is "domain". + Email string `json:"email"` + Kind SuppressionKind `json:"kind"` + Reason string `json:"reason"` + Source DeliverabilityEventType `json:"source"` + CampaignID *uuid.UUID `json:"campaign_id,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type CampaignABVariant struct { @@ -491,6 +615,15 @@ func DefaultAdvancedOutreachSettings() AdvancedOutreachSettings { ShowIntentSummary: true, ShowDLQStats: true, }, + // A plain reply-to-opt-out sentence by default: it satisfies CAN-SPAM, + // CASL and the Spam Act (all accept a reply mechanism), and it reads + // as a personal email where a formal link reads as bulk mail. + Unsubscribe: UnsubscribeSettings{ + Mode: UnsubscribeModeText, + Text: DefaultUnsubscribeText, + LinkIntro: DefaultUnsubscribeLinkIntro, + LinkText: DefaultUnsubscribeLinkText, + }, Custom: map[string]interface{}{}, } } diff --git a/internal/models/analytics.go b/internal/models/analytics.go index e8f446df..db5c9190 100644 --- a/internal/models/analytics.go +++ b/internal/models/analytics.go @@ -47,6 +47,27 @@ type CampaignAnalytics struct { Summary CampaignSummary `json:"summary"` Sequences []SequenceStats `json:"steps"` DailyStats []CampaignDailyStats `json:"daily_stats,omitempty"` + // Engagement is where and on what people opened and clicked, from the + // per-event logs. Human events only. + Engagement *CampaignEngagementBreakdown `json:"engagement,omitempty"` +} + +// EngagementBucket is one slice of a breakdown: how many distinct contacts +// opened and clicked from that country, client, or device. +type EngagementBucket struct { + Key string `json:"key"` + Opens int `json:"opens"` + Clicks int `json:"clicks"` +} + +// CampaignEngagementBreakdown is the "where from, on what" view of a +// campaign's human opens and clicks. Buckets are ordered by activity, capped, +// and keyed by ISO country code, client or browser name, and device type. +// Unknown is the empty key. +type CampaignEngagementBreakdown struct { + Countries []EngagementBucket `json:"countries"` + Clients []EngagementBucket `json:"clients"` + Devices []EngagementBucket `json:"devices"` } type CampaignSummary struct { @@ -58,9 +79,13 @@ type CampaignSummary struct { // (Apple MPP prefetch, UA-less clients). Human opens = unique - machine. MachineOpens int `json:"machine_opens"` UniqueClicks int `json:"unique_clicks"` - Replies int `json:"replies"` - Bounces int `json:"bounces"` - Unsubscribes int `json:"unsubscribes"` + // MachineClicks counts steps whose only clicks came from automated + // fetchers (security gateways walking the links). They are not part of + // UniqueClicks, which only ever counts a person's click. + MachineClicks int `json:"machine_clicks"` + Replies int `json:"replies"` + Bounces int `json:"bounces"` + Unsubscribes int `json:"unsubscribes"` OpenRate float64 `json:"open_rate"` // percentage ClickRate float64 `json:"click_rate"` // percentage @@ -249,8 +274,11 @@ type DashboardOverallStats struct { TotalEmailsSent int `json:"total_emails_sent"` TotalOpens int `json:"total_opens"` // MachineOpens is the subset of TotalOpens from automated fetchers. - MachineOpens int `json:"machine_opens"` - TotalClicks int `json:"total_clicks"` + MachineOpens int `json:"machine_opens"` + TotalClicks int `json:"total_clicks"` + // MachineClicks counts steps clicked only by automated fetchers; they are + // not part of TotalClicks. + MachineClicks int `json:"machine_clicks"` TotalReplies int `json:"total_replies"` TotalBounces int `json:"total_bounces"` OpenRate float64 `json:"open_rate"` diff --git a/internal/models/audit.go b/internal/models/audit.go index 28b487ba..1b310aa9 100644 --- a/internal/models/audit.go +++ b/internal/models/audit.go @@ -65,6 +65,8 @@ const ( AuditEntityAWSCredentials AuditEntityType = "aws_credentials" AuditEntityWorkerProfile AuditEntityType = "worker_profile" AuditEntityRelease AuditEntityType = "release" + // AuditEntityInstance is the deployment itself: settings and updates. + AuditEntityInstance AuditEntityType = "instance" // Org-scoped configuration & governance entities AuditEntityOrganizationMember AuditEntityType = "organization_member" @@ -80,6 +82,7 @@ const ( AuditEntityForm AuditEntityType = "form" AuditEntitySubscription AuditEntityType = "subscription" AuditEntitySettings AuditEntityType = "settings" + AuditEntitySuppression AuditEntityType = "suppression" // CRM entities AuditEntityCRMPipeline AuditEntityType = "crm_pipeline" diff --git a/internal/models/campaign.go b/internal/models/campaign.go index bf841a2a..a508dfc4 100644 --- a/internal/models/campaign.go +++ b/internal/models/campaign.go @@ -86,6 +86,19 @@ func (w *ScheduleWindows) Scan(src any) error { return nil } +// Campaign kinds. A sequence is the multi-step default; a one-time email is +// one message to an audience with no follow-ups. Both send through the same +// pacer and caps; the kind is fixed at creation. +const ( + CampaignKindSequence = "sequence" + CampaignKindOneTime = "one_time" +) + +// ValidCampaignKind reports whether k is a known campaign kind. +func ValidCampaignKind(k string) bool { + return k == CampaignKindSequence || k == CampaignKindOneTime +} + type Campaign struct { ID uuid.UUID `json:"id"` UserID string `json:"user_id"` @@ -94,6 +107,7 @@ type Campaign struct { Name string `json:"name"` Description string `json:"description"` Status string `json:"status"` + Kind string `json:"kind"` StopOnReply bool `json:"stop_on_reply"` OpenTracking bool `json:"open_tracking"` @@ -102,6 +116,8 @@ type Campaign struct { DailyLimit int `json:"daily_limit"` UnsubscribeHeader bool `json:"unsubscribe_header"` RiskyEmails bool `json:"risky_emails"` + // UnsubscribeMode is the in-body opt-out: inherit | text | link | off. + UnsubscribeMode string `json:"unsubscribe_mode"` CC []string `json:"cc"` BCC []string `json:"bcc"` @@ -148,6 +164,11 @@ type Campaign struct { MaxNewLeadsPerDay int `json:"max_new_leads_per_day"` PrioritizeNewLeads bool `json:"prioritize_new_leads"` + // Continuous keeps the campaign active when it runs out of leads: it waits + // for more instead of finishing. IdleSince is set while it waits. + Continuous bool `json:"continuous"` + IdleSince *time.Time `json:"idle_since,omitempty"` + // Auto-pause guardrails. Rates are evaluated over a rolling window and the // campaign is paused the moment a band is breached, rather than waiting for // a mailbox provider to react first. A rate threshold of 0 disables that @@ -171,6 +192,14 @@ type Campaign struct { TrackingDomainVerified bool `json:"tracking_domain_verified"` TrackingDomainVerifiedAt *time.Time `json:"tracking_domain_verified_at,omitempty"` + // Automatic UTM tagging of every link in the email body. Empty source, + // medium and campaign values mean the defaults ("warmbly", "email", the + // campaign name); utm_content is always the link's own text. + UTMTracking bool `json:"utm_tracking"` + UTMSource string `json:"utm_source"` + UTMMedium string `json:"utm_medium"` + UTMCampaign string `json:"utm_campaign"` + LastStatusChangeAt *time.Time `json:"last_status_change_at,omitempty"` UpdatedAt time.Time `json:"updated_at"` @@ -211,9 +240,36 @@ type CampaignsOverview struct { Paused int64 `json:"paused"` Draft int64 `json:"draft"` Completed int64 `json:"completed"` + OneTime int64 `json:"one_time"` Folders []CampaignFolderCount `json:"folders"` } +// CampaignEstimate is the request for POST /campaigns-estimate: how many +// contacts a set of segments resolves to and how long a sender pool needs +// to reach them under the per-mailbox caps. Nothing is written. +type CampaignEstimate struct { + SegmentIDs []string `json:"segment_ids"` + EmailTagIDs []string `json:"email_tag_ids,omitempty"` + DailyLimit *int `json:"daily_limit,omitempty"` + Days *uint8 `json:"days,omitempty"` + Timezone *string `json:"timezone,omitempty"` + StartDate *time.Time `json:"start_date,omitempty"` +} + +// CampaignEstimateResult is the projection. DailyCapacity is the pool's +// per-day ceiling under the campaign limit; RemainingToday subtracts what the +// mailboxes already sent today. SendingDays is how many sending days the +// audience needs and EstimatedFinishAt the calendar day the last send lands +// on, both nil when the pool has no capacity. +type CampaignEstimateResult struct { + Recipients int `json:"recipients"` + Mailboxes int `json:"mailboxes"` + DailyCapacity int `json:"daily_capacity"` + RemainingToday int `json:"remaining_today"` + SendingDays *int `json:"sending_days"` + EstimatedFinishAt *time.Time `json:"estimated_finish_at"` +} + type CampaignFolderCount struct { FolderID uuid.UUID `json:"folder_id"` Total int64 `json:"total"` @@ -224,13 +280,14 @@ type UpdateCampaign struct { Description *string `json:"description"` Status *string `json:"status,omitempty"` - StopOnReply *bool `json:"stop_on_reply"` - OpenTracking *bool `json:"open_tracking"` - LinkTracking *bool `json:"link_tracking"` - TextOnly *bool `json:"text_only"` - DailyLimit *int `json:"daily_limit"` - UnsubscribeHeader *bool `json:"unsubscribe_header"` - RiskyEmails *bool `json:"risky_emails"` + StopOnReply *bool `json:"stop_on_reply"` + OpenTracking *bool `json:"open_tracking"` + LinkTracking *bool `json:"link_tracking"` + TextOnly *bool `json:"text_only"` + DailyLimit *int `json:"daily_limit"` + UnsubscribeHeader *bool `json:"unsubscribe_header"` + RiskyEmails *bool `json:"risky_emails"` + UnsubscribeMode *string `json:"unsubscribe_mode"` CC []string `json:"cc"` BCC []string `json:"bcc"` @@ -267,8 +324,14 @@ type UpdateCampaign struct { ESPMatchMode *string `json:"esp_match_mode,omitempty"` MaxNewLeadsPerDay *int `json:"max_new_leads_per_day,omitempty"` PrioritizeNewLeads *bool `json:"prioritize_new_leads,omitempty"` + Continuous *bool `json:"continuous,omitempty"` TrackingDomain *string `json:"tracking_domain,omitempty"` + UTMTracking *bool `json:"utm_tracking,omitempty"` + UTMSource *string `json:"utm_source,omitempty"` + UTMMedium *string `json:"utm_medium,omitempty"` + UTMCampaign *string `json:"utm_campaign,omitempty"` + // Auto-pause guardrails. GuardrailTrippedAt/Reason are server-owned and // are cleared when the campaign is started again, so they are not settable // here. @@ -294,15 +357,19 @@ func (u *UpdateCampaign) TouchesSchedule() bool { type CreateCampaign struct { Name string `json:"name"` Description string `json:"description"` + // Kind defaults to "sequence". A "one_time" campaign accepts a single + // email step here and refuses further ones later. + Kind *string `json:"kind,omitempty"` // Sending rules / tracking - StopOnReply *bool `json:"stop_on_reply,omitempty"` - OpenTracking *bool `json:"open_tracking,omitempty"` - LinkTracking *bool `json:"link_tracking,omitempty"` - TextOnly *bool `json:"text_only,omitempty"` - DailyLimit *int `json:"daily_limit,omitempty"` - UnsubscribeHeader *bool `json:"unsubscribe_header,omitempty"` - RiskyEmails *bool `json:"risky_emails,omitempty"` + StopOnReply *bool `json:"stop_on_reply,omitempty"` + OpenTracking *bool `json:"open_tracking,omitempty"` + LinkTracking *bool `json:"link_tracking,omitempty"` + TextOnly *bool `json:"text_only,omitempty"` + DailyLimit *int `json:"daily_limit,omitempty"` + UnsubscribeHeader *bool `json:"unsubscribe_header,omitempty"` + RiskyEmails *bool `json:"risky_emails,omitempty"` + UnsubscribeMode *string `json:"unsubscribe_mode,omitempty"` CC []string `json:"cc,omitempty"` BCC []string `json:"bcc,omitempty"` @@ -315,6 +382,9 @@ type CreateCampaign struct { StartTime *string `json:"start_time,omitempty"` EndTime *string `json:"end_time,omitempty"` + // Authoritative per-day schedule. When sent, supersedes Days/StartTime/EndTime. + ScheduleWindows *ScheduleWindows `json:"schedule_windows,omitempty"` + // Sender pool — accepts UUIDs already created by the user. EmailTagIDs []string `json:"email_tag_ids,omitempty"` FolderIDs []string `json:"folder_ids,omitempty"` @@ -335,8 +405,15 @@ type CreateCampaign struct { ESPMatchMode *string `json:"esp_match_mode,omitempty"` MaxNewLeadsPerDay *int `json:"max_new_leads_per_day,omitempty"` PrioritizeNewLeads *bool `json:"prioritize_new_leads,omitempty"` + Continuous *bool `json:"continuous,omitempty"` TrackingDomain *string `json:"tracking_domain,omitempty"` + // Automatic UTM tagging (off unless sent). Empty values keep the defaults. + UTMTracking *bool `json:"utm_tracking,omitempty"` + UTMSource *string `json:"utm_source,omitempty"` + UTMMedium *string `json:"utm_medium,omitempty"` + UTMCampaign *string `json:"utm_campaign,omitempty"` + // Initial sequences (in order) — caller can also create them after. Sequences []CreateSequenceInput `json:"steps,omitempty"` diff --git a/internal/models/cli_auth.go b/internal/models/cli_auth.go new file mode 100644 index 00000000..4deca4a0 --- /dev/null +++ b/internal/models/cli_auth.go @@ -0,0 +1,91 @@ +package models + +import ( + "time" + + "github.com/google/uuid" +) + +// CLI sign-in: `warmbly auth login` opens a device-code handshake, a member +// approves it in the browser, and the approval mints an ordinary API key. + +// CLIAuthCodeStatus is the lifecycle of one handshake. +type CLIAuthCodeStatus string + +const ( + CLIAuthCodePending CLIAuthCodeStatus = "pending" + CLIAuthCodeApproved CLIAuthCodeStatus = "approved" + // Claimed: the CLI has fetched its key, the code is spent. + CLIAuthCodeClaimed CLIAuthCodeStatus = "claimed" + CLIAuthCodeDenied CLIAuthCodeStatus = "denied" +) + +// CLIAuthCode is what the approving member is shown before deciding. +type CLIAuthCode struct { + ID uuid.UUID `json:"id"` + UserCode string `json:"user_code"` + ClientName string `json:"client_name"` + Hostname string `json:"hostname"` + CLIVersion string `json:"cli_version"` + Scopes uint64 `json:"scopes"` + ScopeNames []string `json:"scope_names"` + Status CLIAuthCodeStatus `json:"status"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + // APIKeyID is set only on the approval response: the key that was minted. + APIKeyID *uuid.UUID `json:"api_key_id,omitempty"` + ExpiresAt time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +// CLIAuthStartRequest is what the CLI sends to open a handshake. Every field is +// display-only except Scopes, which bounds the key the approval mints. +type CLIAuthStartRequest struct { + ClientName string `json:"client_name"` + Hostname string `json:"hostname"` + CLIVersion string `json:"cli_version"` + Scopes uint64 `json:"scopes"` +} + +// CLIAuthStartResponse is RFC 8628 shaped, so a generic device-flow client works. +type CLIAuthStartResponse struct { + DeviceCode string `json:"device_code"` + UserCode string `json:"user_code"` + VerificationURL string `json:"verification_uri"` + // VerificationURLComplete carries the code, so the browser needs no typing. + VerificationURLComplete string `json:"verification_uri_complete"` + ExpiresIn int `json:"expires_in"` + Interval int `json:"interval"` +} + +// CLIAuthPollResponse answers one poll. Status is the only field always set; +// the key fields arrive exactly once, on the poll that claims an approved code. +type CLIAuthPollResponse struct { + Status CLIAuthCodeStatus `json:"status"` + + Token string `json:"token,omitempty"` + APIKeyID *uuid.UUID `json:"api_key_id,omitempty"` + Scopes uint64 `json:"scopes,omitempty"` + ScopeNames []string `json:"scope_names,omitempty"` + UserID *uuid.UUID `json:"user_id,omitempty"` + UserEmail string `json:"user_email,omitempty"` + UserName string `json:"user_name,omitempty"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name,omitempty"` +} + +// CLIAuthApproveRequest names the workspace the key is minted in. +type CLIAuthApproveRequest struct { + OrganizationID string `json:"organization_id"` +} + +// APIScopeNames turns a permission bitmask into the scope names the CLI and the +// approval screen show, in the canonical order of AllAPIPermissions. +func APIScopeNames(mask uint64) []string { + names := make([]string, 0, len(AllAPIPermissions)) + for _, p := range AllAPIPermissions { + if mask&p.Value == p.Value { + names = append(names, p.Name) + } + } + return names +} diff --git a/internal/models/contact.go b/internal/models/contact.go index 5bc7a6c0..5e3e6ab1 100644 --- a/internal/models/contact.go +++ b/internal/models/contact.go @@ -1,6 +1,7 @@ package models import ( + "bytes" "time" "github.com/google/uuid" @@ -62,6 +63,11 @@ type Contact struct { // leads are queued, in progress, replied, bounced, or unsubscribed. CampaignLead *ContactCampaignProgress `json:"campaign_lead,omitempty"` + // IsNew is set by the upsert write when this call inserted the row rather + // than matching an existing contact. Server-side only: it decides whether + // a contact.created event fires. + IsNew bool `json:"-"` + UpdatedAt time.Time `json:"updated_at"` CreatedAt time.Time `json:"created_at"` } @@ -315,8 +321,13 @@ type ContactEngagement struct { // ContactSuppression mirrors a row from suppressed_recipients for the // contact's email. Null on the wire when the contact is not suppressed. type ContactSuppression struct { + ID uuid.UUID `json:"id"` + // Kind is "email" for the contact's own address or "domain" when the + // whole domain is suppressed; Value is the matching list entry. + Kind string `json:"kind"` + Value string `json:"value"` Reason string `json:"reason"` - Source string `json:"source"` // bounce | complaint | unsubscribe + Source string `json:"source"` // bounce | complaint | unsubscribe | manual | import ExpiresAt *time.Time `json:"expires_at,omitempty"` CreatedAt time.Time `json:"created_at"` } @@ -409,6 +420,62 @@ const ( TimelinePageHit ContactTimelineEventType = "page_hit" ) +// ContactTimelineSource ranks the tables the timeline is merged from. It is +// the middle key of the feed's order (at, source, id): two events at the same +// instant sort by source, then by that source's row id, so a page boundary +// can never split a tie. The values are part of the cursor; never renumber. +type ContactTimelineSource int + +const ( + // campaign_contact_progress stamps, keyed by the step (sequence) id. + TimelineSourceProgressSent ContactTimelineSource = 1 + TimelineSourceProgressOpened ContactTimelineSource = 2 + TimelineSourceProgressClicked ContactTimelineSource = 3 + TimelineSourceProgressReplied ContactTimelineSource = 4 + TimelineSourceProgressBounced ContactTimelineSource = 5 + + TimelineSourceLinkClick ContactTimelineSource = 6 // email_link_clicks + TimelineSourceOpen ContactTimelineSource = 7 // email_opens + TimelineSourceReplyIntent ContactTimelineSource = 8 // reply_intents + TimelineSourceDeliverability ContactTimelineSource = 9 // deliverability_events + TimelineSourceSuppression ContactTimelineSource = 10 // suppressed_recipients + TimelineSourceNote ContactTimelineSource = 11 // contact_notes + TimelineSourceMeeting ContactTimelineSource = 12 // meeting_bookings + TimelineSourceActivity ContactTimelineSource = 13 // contact_activities + TimelineSourcePageHit ContactTimelineSource = 14 // website_page_hits +) + +// Valid reports whether s names a source the timeline is merged from. A +// cursor carrying any other rank is malformed: zero is reserved for the +// legacy bare-timestamp bound and anything above the last source would +// re-admit the events at the cursor's instant. +func (s ContactTimelineSource) Valid() bool { + return s >= TimelineSourceProgressSent && s <= TimelineSourcePageHit +} + +// ContactTimelineKey is one event's position in the merged feed. A page +// resumes strictly after the key of the last event it returned, comparing +// (At, Source, ID) as a tuple, which is what the opaque cursor carries. +type ContactTimelineKey struct { + At time.Time + Source ContactTimelineSource + ID uuid.UUID +} + +// Before reports whether k sorts after o in the feed's newest-first order, +// which is to say it is the older position: a smaller time, or the same time +// and a lower source rank, or the same time and source and a lower id (uuid +// order is the byte order Postgres uses, so Go and SQL agree). +func (k ContactTimelineKey) Before(o ContactTimelineKey) bool { + if !k.At.Equal(o.At) { + return k.At.Before(o.At) + } + if k.Source != o.Source { + return k.Source < o.Source + } + return bytes.Compare(k.ID[:], o.ID[:]) < 0 +} + // ContactTimelineEvent is one entry in the merged activity feed. The // optional fields are tagged with omitempty so the JSON stays compact // for event types that don't carry that data. @@ -416,6 +483,11 @@ type ContactTimelineEvent struct { Type ContactTimelineEventType `json:"type"` At time.Time `json:"at"` + // Position in the feed, used for the merged sort and the next-page + // cursor. Not part of the wire shape: the row ids are not unique across + // sources, so a client gets an opaque cursor instead. + Key ContactTimelineKey `json:"-"` + // Mailbox sender (email_sent / opened / clicked / replied / bounced). EmailAccountID *uuid.UUID `json:"email_account_id,omitempty"` EmailAccountEmail *string `json:"email_account_email,omitempty"` @@ -458,15 +530,69 @@ type ContactTimelineEvent struct { // Website page view (page_hit): URL, referrer, device, UTM, location. PageHit *WebsitePageHit `json:"page_hit,omitempty"` + // Engagement classification (email_opened / email_clicked). Machine is + // true when the open or click came from an automated fetcher (a mail + // privacy proxy, a security gateway walking the links) rather than a + // person; MachineReason says which rule caught it. + Machine *bool `json:"machine,omitempty"` + MachineReason *string `json:"machine_reason,omitempty"` + + // The exact link behind an email_clicked event, when the click was + // logged per link (every click since link attribution shipped). + Link *ContactLinkClick `json:"link,omitempty"` + + // Where an email_opened / email_clicked event came from, when it was + // logged per event: mail client, device and rough location. + Origin *EngagementOrigin `json:"origin,omitempty"` + // Author (notes, lifecycle events). UserID *uuid.UUID `json:"user_id,omitempty"` } +// ContactLinkClick names the link a contact clicked: where it went, the +// anchor text it was minted from, and the UTM parameters the destination +// carried (automatic or hand-written). +type ContactLinkClick struct { + ID uuid.UUID `json:"id"` + URL string `json:"url"` + Label string `json:"label,omitempty"` + UTMSource string `json:"utm_source,omitempty"` + UTMMedium string `json:"utm_medium,omitempty"` + UTMCampaign string `json:"utm_campaign,omitempty"` + UTMTerm string `json:"utm_term,omitempty"` + UTMContent string `json:"utm_content,omitempty"` + UserAgent string `json:"user_agent,omitempty"` +} + +// EngagementOrigin is what an open or click said about where it came from. +// Client names the mail client or image proxy when the user agent does +// (Gmail, Apple Mail, Outlook); the browser fields describe the rest. The +// location is resolved from the source network and the address itself is +// never stored. Every field is empty when unknown. +type EngagementOrigin struct { + Client string `json:"client,omitempty"` + DeviceType string `json:"device_type,omitempty"` + OS string `json:"os,omitempty"` + Browser string `json:"browser,omitempty"` + BrowserVersion string `json:"browser_version,omitempty"` + CountryCode string `json:"country_code,omitempty"` + Region string `json:"region,omitempty"` + City string `json:"city,omitempty"` +} + +// Empty reports whether nothing about the origin is known. +func (o EngagementOrigin) Empty() bool { + return o == EngagementOrigin{} +} + type ContactTimelineResult struct { Data []ContactTimelineEvent `json:"data"` - // True if we hit the per-call cap and the caller should paginate - // via the `before` query param. + // Deprecated: read pagination.has_more. Kept for clients written against + // the bare-timestamp pagination that predated the cursor envelope. HasMore bool `json:"has_more"` + // NextCursor is an opaque (at, source, id) position; pass it back as + // `cursor` for the next page. Total is never counted across the sources. + Pagination Pagination `json:"pagination"` } type UpdateContact struct { @@ -540,13 +666,17 @@ const ( ContactSourceAPI ContactSource = "api" ContactSourceAIAssistant ContactSource = "ai_assistant" ContactSourceForm ContactSource = "form" + // ContactSourceAutomation is a contact an automation's "create or update + // contact" action wrote; the detail is the automation's name. + ContactSourceAutomation ContactSource = "automation" ) // Valid reports whether the value is one the database accepts. func (s ContactSource) Valid() bool { switch s { case ContactSourceUnknown, ContactSourceManual, ContactSourceCampaign, ContactSourceImport, - ContactSourceSheetSync, ContactSourceAPI, ContactSourceAIAssistant, ContactSourceForm: + ContactSourceSheetSync, ContactSourceAPI, ContactSourceAIAssistant, ContactSourceForm, + ContactSourceAutomation: return true } return false diff --git a/internal/models/contact_timeline_test.go b/internal/models/contact_timeline_test.go new file mode 100644 index 00000000..231d7182 --- /dev/null +++ b/internal/models/contact_timeline_test.go @@ -0,0 +1,16 @@ +package models + +import "testing" + +func TestContactTimelineSourceValid(t *testing.T) { + for s := TimelineSourceProgressSent; s <= TimelineSourcePageHit; s++ { + if !s.Valid() { + t.Fatalf("source %d is one the feed merges and must be valid", s) + } + } + for _, s := range []ContactTimelineSource{0, -1, TimelineSourcePageHit + 1, 99} { + if s.Valid() { + t.Fatalf("source %d names no table and must be rejected in a cursor", s) + } + } +} diff --git a/internal/models/email.go b/internal/models/email.go index b0761c77..43b6400b 100644 --- a/internal/models/email.go +++ b/internal/models/email.go @@ -220,20 +220,26 @@ type Oauth2SmtpImap struct { type NewOauthAccount struct { OrganizationID *uuid.UUID - Provider InboxProvider - Name string - Email string - AccessToken string - RefreshToken string - ExpiresAt time.Time + // Allowance, when set, is enforced again inside the insert transaction + // under the organization's mailbox lock, so concurrent connects cannot + // both take the last slot. + Allowance *MailboxAllowance + Provider InboxProvider + Name string + Email string + AccessToken string + RefreshToken string + ExpiresAt time.Time } type NewSMTPIMAPAccount struct { OrganizationID *uuid.UUID - Name string - Email string - SMTP *Service - IMAP *Service + // Allowance: see NewOauthAccount. + Allowance *MailboxAllowance + Name string + Email string + SMTP *Service + IMAP *Service } // EmailOnboardingState is stored in Redis for the lifetime of an OAuth round trip. @@ -339,3 +345,44 @@ type BulkEmailTags struct { AddTags []string `json:"add_tags" binding:"max=100"` RemoveTags []string `json:"remove_tags" binding:"max=100"` } + +// MailboxBulkRowStatus is the per-row outcome of a bulk SMTP/IMAP connect. +type MailboxBulkRowStatus string + +const ( + // MailboxBulkConnected: the mailbox was validated and connected. + MailboxBulkConnected MailboxBulkRowStatus = "connected" + // MailboxBulkSkipped: the mailbox was already connected, so re-uploading + // a file is safe. + MailboxBulkSkipped MailboxBulkRowStatus = "skipped" + // MailboxBulkFailed: the row was refused; Code says why. + MailboxBulkFailed MailboxBulkRowStatus = "failed" +) + +// MailboxBulkRow is one row's answer. Row echoes the caller's own row number +// so the dashboard can hand back the failed lines of the file it uploaded. +type MailboxBulkRow struct { + Row int `json:"row"` + Email string `json:"email"` + Status MailboxBulkRowStatus `json:"status"` + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` + ID *uuid.UUID `json:"id,omitempty"` +} + +// MailboxBulkSummary counts the batch. +type MailboxBulkSummary struct { + Total int `json:"total"` + Connected int `json:"connected"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` +} + +// MailboxBulkResult is the answer to POST /emails/onboarding/smtp-imap/bulk. +type MailboxBulkResult struct { + Data []MailboxBulkRow `json:"data"` + Summary MailboxBulkSummary `json:"summary"` + // Allowance is the workspace's mailbox allowance after the batch, so the + // dashboard can say how many more rows will fit without another call. + Allowance *MailboxAllowance `json:"allowance,omitempty"` +} diff --git a/internal/models/integration.go b/internal/models/integration.go index 0bd3f417..0dde072f 100644 --- a/internal/models/integration.go +++ b/internal/models/integration.go @@ -289,6 +289,13 @@ const ( // with REALTIME_SUBSCRIBE on the org websocket) receive it with no public URL, // so it replaces an outbound webhook for "tell my system this happened". IntegrationActionFireEvent IntegrationAction = "warmbly.fire_event" + // IntegrationActionUpsertContact creates a contact from templated event + // fields, or enriches the one already holding that email, then tags it and + // enrols it in a campaign. The lead-intake action: an inbound webhook or a + // form submission becomes a contact without leaving Warmbly. + IntegrationActionUpsertContact IntegrationAction = "warmbly.upsert_contact" + // IntegrationActionAddToCampaign enrols the event's contact in a campaign. + IntegrationActionAddToCampaign IntegrationAction = "warmbly.add_to_campaign" // AI nodes mirror the campaign step types: one unified AI step plus an AI // switch router. @@ -328,6 +335,7 @@ func IsNativeAction(a IntegrationAction) bool { IntegrationActionCreateDeal, IntegrationActionMoveDealStage, IntegrationActionUnsubscribe, IntegrationActionRunAutomation, IntegrationActionLabelEmail, IntegrationActionSetVariables, IntegrationActionFireEvent, + IntegrationActionUpsertContact, IntegrationActionAddToCampaign, IntegrationActionAIStep, IntegrationActionAISwitch: return true default: diff --git a/internal/models/organization.go b/internal/models/organization.go index e2948935..22da0b08 100644 --- a/internal/models/organization.go +++ b/internal/models/organization.go @@ -351,6 +351,56 @@ type UpdateOrgOverridesRequest struct { Notes *string `json:"notes,omitempty"` } +// MailboxAllowanceBasis says where a workspace's mailbox allowance comes from, +// so the dashboard can explain the number rather than only state it. +type MailboxAllowanceBasis string + +const ( + // MailboxAllowanceUnlimited: no billing provider, or a plan with no daily + // send cap. Allowance is nil. + MailboxAllowanceUnlimited MailboxAllowanceBasis = "unlimited" + // MailboxAllowanceFree: an unsubscribed workspace, FreeWorkspaceMailboxLimit. + MailboxAllowanceFree MailboxAllowanceBasis = "free" + // MailboxAllowanceOverride: an operator-approved limit-increase request. + MailboxAllowanceOverride MailboxAllowanceBasis = "override" + // MailboxAllowancePlan: the plan carries an explicit mailbox column. + MailboxAllowancePlan MailboxAllowanceBasis = "plan" + // MailboxAllowanceFairUse: the plan's daily sends divided by + // config.FairUseSendsPerMailbox. + MailboxAllowanceFairUse MailboxAllowanceBasis = "fair_use" +) + +// MailboxAllowance is how many mailboxes a workspace may hold and why. It is +// what every connect path checks and what GET /emails/allowance returns. +type MailboxAllowance struct { + // Used is the number of mailboxes connected right now. + Used int `json:"used"` + // Allowance is the cap; nil means unlimited. + Allowance *int `json:"allowance"` + // Remaining is Allowance minus Used, never negative; nil when unlimited. + Remaining *int `json:"remaining"` + Basis MailboxAllowanceBasis `json:"basis"` + // SendsPerMailbox is the fair-use divisor, so the dashboard can say + // "one mailbox per daily send" with the real number. + SendsPerMailbox int `json:"sends_per_mailbox"` + // PlanDailySends is the plan's daily send cap when it has one. + PlanDailySends *int `json:"plan_daily_sends,omitempty"` + PlanName string `json:"plan_name,omitempty"` + // Paid is false for a free workspace, whose path to more mailboxes is a + // plan rather than a request. + Paid bool `json:"paid"` + // PendingRequest is the open limit-increase request for mailboxes, if any. + PendingRequest *LimitIncreaseRequest `json:"pending_request,omitempty"` +} + +// CanAdd reports whether n more mailboxes fit. +func (a *MailboxAllowance) CanAdd(n int) bool { + if a == nil || a.Allowance == nil { + return true + } + return a.Used+n <= *a.Allowance +} + // LimitRequestStatus mirrors the postgres enum from migration 000046. type LimitRequestStatus string diff --git a/internal/models/segment.go b/internal/models/segment.go index b92dddc0..6fa61f94 100644 --- a/internal/models/segment.go +++ b/internal/models/segment.go @@ -233,6 +233,8 @@ type SegmentAddToCampaignResult struct { CampaignID uuid.UUID `json:"campaign_id"` Added int `json:"added"` Members int `json:"members"` + // Status is the campaign's status at enrol time, for the wake/restart decision. + Status string `json:"-"` } // CampaignSegmentsMax bounds how many segments one campaign can link. @@ -243,13 +245,16 @@ type CampaignSegmentsWrite struct { SegmentIDs []string `json:"segment_ids"` } -// CampaignSegmentLink is one segment linked to a campaign, for the Leads tab. +// CampaignSegmentLink is one segment linked to a campaign, for the Leads tab; +// the counts are live: members now, members that are leads, members held out. type CampaignSegmentLink struct { SegmentID uuid.UUID `json:"segment_id"` Name string `json:"name"` Color string `json:"color"` Description string `json:"description"` ContactCount int `json:"contact_count"` + LeadCount int `json:"lead_count"` + HeldOutCount int `json:"held_out_count"` LinkedAt time.Time `json:"linked_at"` } diff --git a/internal/models/webhook.go b/internal/models/webhook.go index e79b8f19..55e6b2e7 100644 --- a/internal/models/webhook.go +++ b/internal/models/webhook.go @@ -439,9 +439,9 @@ func WebhookEventForAudit(entityType AuditEntityType, action AuditAction) (Webho return WebhookEventCampaignPaused, true } case AuditEntityContact: + // contact.created has a dedicated, richer emit in the contact service + // (the contact's fields and source), so the audit row is not bridged. switch action { - case AuditActionCreate: - return WebhookEventContactCreated, true case AuditActionUpdate: return WebhookEventContactUpdated, true case AuditActionDelete: diff --git a/internal/pkg/emsg/emsg.go b/internal/pkg/emsg/emsg.go index 8f0cdb57..ba08c3d0 100644 --- a/internal/pkg/emsg/emsg.go +++ b/internal/pkg/emsg/emsg.go @@ -16,6 +16,7 @@ const ( FlagPlainText uint32 = 1 << 0 FlagHTMLBody uint32 = 1 << 1 FlagAttachments uint32 = 1 << 2 + FlagFromName uint32 = 1 << 3 ) // Attachment is a single attachment reference carried inside the S3 body blob. @@ -33,17 +34,24 @@ type EmailBlob struct { PlainText []byte HTMLBody []byte Attachments []Attachment + // FromName is the sender display name at publish time. It rides in the + // blob so a renamed mailbox sends under its new name without the worker's + // cached ADD_EMAIL identity having to be refreshed. Empty means "use the + // worker's cached name". + FromName string } // EncodeBinary serializes the blob into binary format. Layout: // -// "EMSG" | version(1) | flags(4) | [plain] | [html] | [attachments] +// "EMSG" | version(1) | flags(4) | [plain] | [html] | [attachments] | [from] // // where each body section is uint32-length-prefixed and only present when its // flag bit is set. The attachments section, when present, is a uint32 count // followed by that many (s3key, filename, mimetype) triples of length-prefixed // strings. Attachment metadata travels here (inside the S3 body blob), not in // the Avro Kafka event, so the published worker event contract is unchanged. +// The from-name section sits last so a worker that predates it decodes the +// rest of the blob untouched and simply never reads the trailing bytes. func (b *EmailBlob) EncodeBinary() ([]byte, error) { var flags uint32 parts := make([][]byte, 0, 2) @@ -59,6 +67,9 @@ func (b *EmailBlob) EncodeBinary() ([]byte, error) { if len(b.Attachments) > 0 { flags |= FlagAttachments } + if b.FromName != "" { + flags |= FlagFromName + } buf := new(bytes.Buffer) @@ -87,6 +98,11 @@ func (b *EmailBlob) EncodeBinary() ([]byte, error) { } } + if flags&FlagFromName != 0 { + binary.Write(buf, binary.BigEndian, uint32(len(b.FromName))) + buf.WriteString(b.FromName) + } + return buf.Bytes(), nil } @@ -162,5 +178,13 @@ func DecodeBinary(r io.Reader) (*EmailBlob, error) { } } + if flags&FlagFromName != 0 { + name, err := readSection() + if err != nil { + return nil, err + } + b.FromName = string(name) + } + return b, nil } diff --git a/internal/pkg/emsg/emsg_test.go b/internal/pkg/emsg/emsg_test.go index 8f27f89a..bdd79997 100644 --- a/internal/pkg/emsg/emsg_test.go +++ b/internal/pkg/emsg/emsg_test.go @@ -103,3 +103,32 @@ func TestEmailBlob_NoAttachments(t *testing.T) { t.Errorf("expected no attachments, got %d", len(got.Attachments)) } } + +// The from-name section is trailing and flagged, so a blob written without it +// decodes as before and one written with it round-trips alongside attachments. +func TestEmailBlob_FromName(t *testing.T) { + out := roundTrip(t, &EmailBlob{ + PlainText: []byte("Hi"), + Attachments: []Attachment{{S3Key: "k", Filename: "deck.pdf", MimeType: "application/pdf"}}, + FromName: "Renée Doe, Jr.", + }) + if out.FromName != "Renée Doe, Jr." { + t.Errorf("FromName = %q", out.FromName) + } + if len(out.Attachments) != 1 || out.Attachments[0].Filename != "deck.pdf" { + t.Errorf("attachments did not survive alongside the from name: %+v", out.Attachments) + } + + legacy := roundTrip(t, &EmailBlob{PlainText: []byte("Hi")}) + if legacy.FromName != "" { + t.Errorf("blob without a from name decoded one: %q", legacy.FromName) + } + data, _ := (&EmailBlob{PlainText: []byte("Hi")}).EncodeBinary() + if binaryFlags(data)&FlagFromName != 0 { + t.Error("empty from name set the flag") + } +} + +func binaryFlags(data []byte) uint32 { + return uint32(data[5])<<24 | uint32(data[6])<<16 | uint32(data[7])<<8 | uint32(data[8]) +} diff --git a/internal/repository/attachment_step_scope_live_test.go b/internal/repository/attachment_step_scope_live_test.go new file mode 100644 index 00000000..e174a26a --- /dev/null +++ b/internal/repository/attachment_step_scope_live_test.go @@ -0,0 +1,116 @@ +package repository + +import ( + "context" + "testing" + + "github.com/google/uuid" +) + +// A campaign attachment carries an optional sequence_id, and the send path now +// honours it: a step sends the campaign-wide files plus its own, and nothing +// scoped to another step. Cover for the scoping query itself, plus the guard +// that refuses a step_id belonging to another campaign (the FK only proves the +// step exists). +// +// Run against the dev stack: +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \ +// go test ./internal/repository/ -run LiveAttachmentStepScope -v +func TestLiveAttachmentStepScope(t *testing.T) { + handle, pool := liveContactDB(t) + f := newSharedOrgFixture(t, pool) + repo := NewAttachmentRepository(handle) + ctx := context.Background() + + stepOne, stepTwo, otherCampaignStep := uuid.New(), uuid.New(), uuid.New() + for _, s := range []struct { + id uuid.UUID + campaign uuid.UUID + position int + }{{stepOne, f.campaign, 0}, {stepTwo, f.campaign, 1}, {otherCampaignStep, f.other, 0}} { + if _, err := pool.Exec(ctx, + `INSERT INTO sequences (id, campaign_id, organization_id, name, subject, body_html, body_plain, position) + VALUES ($1, $2, $3, 'Step', 'Subject', '

Body

', 'Body', $4)`, + s.id, s.campaign, f.org, s.position); err != nil { + t.Fatalf("fixture sequence: %v", err) + } + } + + insert := func(name string, step *uuid.UUID) { + t.Helper() + if _, err := pool.Exec(ctx, + `INSERT INTO campaign_attachments (campaign_id, sequence_id, user_id, filename, size, mime_type, s3_key) + VALUES ($1, $2, $3, $4, 10, 'application/pdf', $5)`, + f.campaign, step, f.owner, name, "live/"+name); err != nil { + t.Fatalf("fixture attachment %s: %v", name, err) + } + } + insert("everystep.pdf", nil) + insert("stepone.pdf", &stepOne) + insert("steptwo.pdf", &stepTwo) + + names := func(step uuid.UUID) []string { + t.Helper() + atts, err := repo.ListForStep(ctx, f.campaign, step) + if err != nil { + t.Fatalf("ListForStep: %v", err) + } + out := make([]string, 0, len(atts)) + for _, a := range atts { + out = append(out, a.Filename) + } + return out + } + + same := func(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true + } + + if got := names(stepOne); !same(got, []string{"everystep.pdf", "stepone.pdf"}) { + t.Errorf("step one sends %v, want the campaign-wide file and its own", got) + } + if got := names(stepTwo); !same(got, []string{"everystep.pdf", "steptwo.pdf"}) { + t.Errorf("step two sends %v, want the campaign-wide file and its own", got) + } + // A step of the campaign that owns no files still carries the campaign-wide + // ones, and uuid.Nil (no step in context) carries only those. + if got := names(uuid.Nil); !same(got, []string{"everystep.pdf"}) { + t.Errorf("unscoped list is %v, want the campaign-wide file alone", got) + } + + // Every file of the campaign is still listed for the dashboard. + all, err := repo.ListByCampaign(ctx, f.campaign) + if err != nil { + t.Fatalf("ListByCampaign: %v", err) + } + if len(all) != 3 { + t.Errorf("ListByCampaign returned %d files, want all 3", len(all)) + } + + for _, tc := range []struct { + name string + step uuid.UUID + want bool + }{ + {"own step", stepOne, true}, + {"another campaign's step", otherCampaignStep, false}, + {"unknown step", uuid.New(), false}, + } { + got, err := repo.StepBelongsToCampaign(ctx, f.campaign, tc.step) + if err != nil { + t.Fatalf("StepBelongsToCampaign(%s): %v", tc.name, err) + } + if got != tc.want { + t.Errorf("StepBelongsToCampaign(%s) = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/internal/repository/campaign_create_schedule_windows_live_test.go b/internal/repository/campaign_create_schedule_windows_live_test.go new file mode 100644 index 00000000..811f9681 --- /dev/null +++ b/internal/repository/campaign_create_schedule_windows_live_test.go @@ -0,0 +1,108 @@ +package repository + +import ( + "context" + "reflect" + "testing" + + "github.com/warmbly/warmbly/internal/models" +) + +// Regression cover for issue #307 item 5: POST /campaigns accepted +// schedule_windows but Create did not persist the authoritative schedule. +// +// Run against the dev stack: +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \ +// go test ./internal/repository/ -run LiveCreateCampaignPersistsScheduleWindows -v +func TestLiveCreateCampaignPersistsScheduleWindows(t *testing.T) { + handle, pool := liveContactDB(t) + f := newSharedOrgFixture(t, pool) + repo := NewCampaignRepostory(handle) + ctx := context.Background() + + t.Run("persists supplied windows", func(t *testing.T) { + want := models.ScheduleWindows{ + 1: []models.TimeInterval{{Start: 540, End: 1020}}, + 5: []models.TimeInterval{{Start: 540, End: 840}}, + } + campaign, xerr := repo.Create(ctx, f.owner.String(), &f.org, &models.CreateCampaign{ + Name: "Issue 307 schedule windows", + ScheduleWindows: &want, + }) + if xerr != nil { + t.Fatalf("Create: %v", xerr) + } + if !reflect.DeepEqual(campaign.ScheduleWindows, want) { + t.Fatalf("Create schedule_windows = %#v, want %#v", campaign.ScheduleWindows, want) + } + + var notNull bool + var stored models.ScheduleWindows + if err := pool.QueryRow(ctx, + `SELECT schedule_windows IS NOT NULL, schedule_windows FROM campaigns WHERE id = $1`, + campaign.ID).Scan(¬Null, &stored); err != nil { + t.Fatalf("select schedule_windows: %v", err) + } + if !notNull { + t.Fatal("database schedule_windows is NULL, want supplied windows") + } + if !reflect.DeepEqual(stored, want) { + t.Fatalf("database schedule_windows = %#v, want %#v", stored, want) + } + + got, err := repo.Get(ctx, f.org.String(), campaign.ID.String()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !reflect.DeepEqual(got.ScheduleWindows, want) { + t.Fatalf("Get schedule_windows = %#v, want %#v", got.ScheduleWindows, want) + } + }) + + t.Run("keeps legacy schedule null when omitted", func(t *testing.T) { + campaign, xerr := repo.Create(ctx, f.owner.String(), &f.org, &models.CreateCampaign{ + Name: "Issue 307 legacy schedule", + }) + if xerr != nil { + t.Fatalf("Create: %v", xerr) + } + if !campaign.ScheduleWindows.IsEmpty() { + t.Fatalf("Create schedule_windows = %#v, want empty", campaign.ScheduleWindows) + } + + var isNull bool + if err := pool.QueryRow(ctx, + `SELECT schedule_windows IS NULL FROM campaigns WHERE id = $1`, + campaign.ID).Scan(&isNull); err != nil { + t.Fatalf("select schedule_windows null state: %v", err) + } + if !isNull { + t.Fatal("database schedule_windows is not NULL, want legacy NULL") + } + }) + + t.Run("rejects invalid windows without inserting", func(t *testing.T) { + const name = "Issue 307 invalid schedule" + bad := models.ScheduleWindows{ + 1: []models.TimeInterval{{Start: 600, End: 540}}, + } + campaign, xerr := repo.Create(ctx, f.owner.String(), &f.org, &models.CreateCampaign{ + Name: name, + ScheduleWindows: &bad, + }) + if xerr == nil { + t.Fatalf("Create = %#v, nil error; want validation error", campaign) + } + + var count int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM campaigns WHERE organization_id = $1 AND name = $2`, + f.org, name).Scan(&count); err != nil { + t.Fatalf("count invalid campaign rows: %v", err) + } + if count != 0 { + t.Fatalf("invalid campaign rows = %d, want 0", count) + } + }) +} diff --git a/internal/repository/campaign_create_schedule_windows_test.go b/internal/repository/campaign_create_schedule_windows_test.go new file mode 100644 index 00000000..6bb04d09 --- /dev/null +++ b/internal/repository/campaign_create_schedule_windows_test.go @@ -0,0 +1,57 @@ +package repository + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +func TestCreateCampaignRejectsInvalidScheduleWindows(t *testing.T) { + requireBadRequest := func(t *testing.T, got *errx.Error) { + t.Helper() + if got == nil { + t.Fatal("Create() error = nil, want *errx.Error") + } + if got.Code != errx.BadRequest { + t.Fatalf("Create() error code = %v, want %v", got.Code, errx.BadRequest) + } + } + + nineIntervals := make([]models.TimeInterval, 9) + for i := range nineIntervals { + nineIntervals[i] = models.TimeInterval{Start: i * 10, End: i*10 + 5} + } + + tests := []struct { + name string + windows models.ScheduleWindows + }{ + { + name: "end before start", + windows: models.ScheduleWindows{ + 1: []models.TimeInterval{{Start: 600, End: 540}}, + }, + }, + { + name: "more than eight intervals", + windows: models.ScheduleWindows{ + 1: nineIntervals, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + orgID := uuid.New() + _, got := (&campaignRepository{}).Create(context.Background(), "u", &orgID, &models.CreateCampaign{ + Name: "x", + ScheduleWindows: &tt.windows, + }) + requireBadRequest(t, got) + }) + } +} diff --git a/internal/repository/contact_timeline_live_test.go b/internal/repository/contact_timeline_live_test.go index f4e7e0f8..2f435c37 100644 --- a/internal/repository/contact_timeline_live_test.go +++ b/internal/repository/contact_timeline_live_test.go @@ -2,11 +2,14 @@ package repository import ( "context" + "strconv" "testing" + "time" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" ) // Live cover for the contact timeline's lifecycle events and first-touch @@ -183,3 +186,117 @@ func TestLiveContactSourceCampaignResolvesName(t *testing.T) { t.Fatal("an unknown source must be refused, not stored") } } + +// Live cover for the timeline's cursor (issue #305): events that share a +// timestamp, within one source and across sources, must land on one side of +// a page boundary or the other, never be skipped and never repeat, and a page +// filled by a single source must still report that more follow. +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \ +// go test ./internal/repository/ -run LiveContactTimelinePages -v +func TestLiveContactTimelinePagesOnTiesWithoutGapsOrRepeats(t *testing.T) { + handle, pool := liveContactDB(t) + f := newSharedOrgFixture(t, pool) + ctx := context.Background() + repo := NewContactRepostory(handle) + + exec := func(sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(ctx, sql, args...); err != nil { + t.Fatalf("seed %q: %v", sql[:min(60, len(sql))], err) + } + } + at := time.Date(2026, 6, 9, 11, 42, 0, 250000000, time.UTC) + step := uuid.New() + exec(`INSERT INTO sequences (id, campaign_id, organization_id, name, subject, body_plain, body_html) VALUES ($1, $2, $3, 'Intro', 'Quick question', '', '')`, step, f.campaign, f.org) + // Three stamps on one lead at the same instant: sent, a summary open + // (nothing in email_opens stands for it) and a reply. + exec(`INSERT INTO campaign_contact_progress (campaign_id, contact_id, sequence_id, sent_at, opened_at, replied_at) VALUES ($1, $2, $3, $4, $4, $4)`, + f.campaign, f.contact, step, at) + for i := 0; i < 3; i++ { + exec(`INSERT INTO contact_notes (contact_id, organization_id, user_id, content, created_at) VALUES ($1, $2, $3, $4, $5)`, + f.contact, f.org, f.owner, "note "+strconv.Itoa(i), at) + } + exec(`INSERT INTO contact_notes (contact_id, organization_id, user_id, content, created_at) VALUES ($1, $2, $3, 'older', $4)`, + f.contact, f.org, f.owner, at.Add(-time.Hour)) + for i := 0; i < 2; i++ { + exec(`INSERT INTO contact_activities (contact_id, organization_id, user_id, activity_type, metadata, created_at) VALUES ($1, $2, $3, 'campaign_added', '{"campaign_name":"Agency partnerships"}', $4)`, + f.contact, f.org, f.owner, at) + } + exec(`INSERT INTO contact_activities (contact_id, organization_id, user_id, activity_type, metadata, created_at) VALUES ($1, $2, $3, 'contact_created', '{"source":"manual"}', $4)`, + f.contact, f.org, f.owner, at.Add(time.Hour)) + t.Cleanup(func() { + c := context.Background() + _, _ = pool.Exec(c, `DELETE FROM contact_activities WHERE contact_id = $1`, f.contact) + _, _ = pool.Exec(c, `DELETE FROM contact_notes WHERE contact_id = $1`, f.contact) + _, _ = pool.Exec(c, `DELETE FROM campaign_contact_progress WHERE contact_id = $1`, f.contact) + _, _ = pool.Exec(c, `DELETE FROM sequences WHERE id = $1`, step) + }) + const total = 10 // 3 stamps + 4 notes + 3 activities + + // Walk the feed three at a time: a page of three can be filled by the + // notes at `at` alone, and eight events share that instant. + var all []models.ContactTimelineEvent + var cursor *models.ContactTimelineKey + for page := 0; ; page++ { + res, xerr := repo.ListTimeline(ctx, f.owner, &f.org, f.contact, 3, cursor) + if xerr != nil { + t.Fatalf("page %d: %v", page, xerr) + } + if res.HasMore != res.Pagination.HasMore { + t.Fatalf("page %d: has_more %v disagrees with pagination.has_more %v", page, res.HasMore, res.Pagination.HasMore) + } + all = append(all, res.Data...) + if !res.HasMore { + if res.Pagination.NextCursor != nil { + t.Fatalf("last page must carry no cursor, got %q", *res.Pagination.NextCursor) + } + break + } + if res.Pagination.NextCursor == nil || len(res.Data) != 3 { + t.Fatalf("page %d: has_more with %d events and cursor %v", page, len(res.Data), res.Pagination.NextCursor) + } + cAt, cSource, cID, xerr := paging.DecodeMergedCursor(*res.Pagination.NextCursor) + if xerr != nil { + t.Fatalf("page %d: cursor does not decode: %v", page, xerr) + } + if last := res.Data[2].Key; !cAt.Equal(last.At) || models.ContactTimelineSource(cSource) != last.Source || cID != last.ID { + t.Fatalf("page %d: cursor %v/%d/%s is not the last event's key %+v", page, cAt, cSource, cID, last) + } + cursor = &models.ContactTimelineKey{At: cAt, Source: models.ContactTimelineSource(cSource), ID: cID} + if page > total { + t.Fatal("the walk never ends") + } + } + if len(all) != total { + t.Fatalf("want %d events across the pages, got %d: %+v", total, len(all), all) + } + seen := map[models.ContactTimelineKey]bool{} + for i, e := range all { + if seen[e.Key] { + t.Fatalf("event %d repeated across pages: %+v", i, e.Key) + } + seen[e.Key] = true + if i > 0 && !e.Key.Before(all[i-1].Key) { + t.Fatalf("feed out of order at %d: %+v then %+v", i, all[i-1].Key, e.Key) + } + } + for typ, want := range map[models.ContactTimelineEventType]int{ + models.TimelineEmailSent: 1, models.TimelineEmailOpened: 1, models.TimelineEmailReplied: 1, + models.TimelineNote: 4, models.TimelineCampaignAdded: 2, models.TimelineContactCreated: 1, + } { + if n := countTimeline(all, typ, nil); n != want { + t.Fatalf("want %d %s events, got %d", want, typ, n) + } + } + + // The legacy bare timestamp still means "strictly older than": rank zero + // sits below every source, so nothing at that instant qualifies. + res, xerr := repo.ListTimeline(ctx, f.owner, &f.org, f.contact, 50, &models.ContactTimelineKey{At: at}) + if xerr != nil { + t.Fatalf("before: %v", xerr) + } + if len(res.Data) != 1 || res.Data[0].Type != models.TimelineNote || res.HasMore { + t.Fatalf("want only the older note before %s, got %+v (has_more %v)", at, res.Data, res.HasMore) + } +} diff --git a/internal/repository/pg_advanced_outreach.go b/internal/repository/pg_advanced_outreach.go index d2f06941..f73c2414 100644 --- a/internal/repository/pg_advanced_outreach.go +++ b/internal/repository/pg_advanced_outreach.go @@ -31,6 +31,17 @@ type AdvancedOutreachRepository interface { IsRecipientSuppressed(ctx context.Context, organizationID uuid.UUID, email string) (*models.SuppressedRecipient, error) UpsertSuppressedRecipient(ctx context.Context, entry *models.SuppressedRecipient) error + // UpsertSuppressedRecipients writes a batch in one transaction, so a + // pasted list lands whole or not at all. + UpsertSuppressedRecipients(ctx context.Context, entries []models.SuppressedRecipient) error + // ListSuppressedRecipients pages the active list newest first. q filters by + // address or domain substring; before is the keyset (created_at, id). + ListSuppressedRecipients(ctx context.Context, organizationID uuid.UUID, q string, beforeAt *time.Time, beforeID *uuid.UUID, limit int) ([]models.SuppressedRecipient, error) + GetSuppressedRecipient(ctx context.Context, organizationID, id uuid.UUID) (*models.SuppressedRecipient, error) + DeleteSuppressedRecipient(ctx context.Context, organizationID, id uuid.UUID) (bool, error) + // DeleteSuppressionByEmail removes the address entry with the given source + // (the recipient's own resubscribe only undoes a recipient-made entry). + DeleteSuppressionByEmail(ctx context.Context, organizationID uuid.UUID, email string, source models.DeliverabilityEventType) (bool, error) CreateDeliverabilityEvent(ctx context.Context, event *models.DeliverabilityEvent) error GetDeliverabilityDashboard(ctx context.Context, organizationID uuid.UUID, from, to time.Time) (*models.DeliverabilityDashboard, error) @@ -364,20 +375,16 @@ func (r *advancedOutreachRepository) MarkVariantEvent(ctx context.Context, campa return err } -func (r *advancedOutreachRepository) IsRecipientSuppressed(ctx context.Context, organizationID uuid.UUID, email string) (*models.SuppressedRecipient, error) { - query := ` - SELECT id, organization_id, email, reason, source, campaign_id, expires_at, metadata, created_at, updated_at - FROM suppressed_recipients - WHERE organization_id = $1 - AND LOWER(email) = LOWER($2) - AND (expires_at IS NULL OR expires_at > NOW()) - ` +const suppressedRecipientColumns = `id, organization_id, email, kind, reason, source, campaign_id, expires_at, metadata, created_at, updated_at` + +func scanSuppressedRecipient(row pgx.Row) (*models.SuppressedRecipient, error) { var out models.SuppressedRecipient var metadata []byte - if err := r.db.QueryRow(ctx, query, organizationID, email).Scan( + if err := row.Scan( &out.ID, &out.OrganizationID, &out.Email, + &out.Kind, &out.Reason, &out.Source, &out.CampaignID, @@ -386,9 +393,6 @@ func (r *advancedOutreachRepository) IsRecipientSuppressed(ctx context.Context, &out.CreatedAt, &out.UpdatedAt, ); err != nil { - if err == pgx.ErrNoRows { - return nil, nil - } return nil, err } if len(metadata) > 0 { @@ -397,16 +401,50 @@ func (r *advancedOutreachRepository) IsRecipientSuppressed(ctx context.Context, return &out, nil } +// IsRecipientSuppressed returns the entry that blocks email: its own address +// row first, else a row for its domain. Same predicate as the SQL function +// recipient_suppressed() the send gates use, spelled out here because the +// caller wants the row, not a boolean. Stored values are lowercase (every +// write folds them; migration 000124 folded the rest), so the comparison is +// an equality the unique index serves. +func (r *advancedOutreachRepository) IsRecipientSuppressed(ctx context.Context, organizationID uuid.UUID, email string) (*models.SuppressedRecipient, error) { + query := ` + SELECT ` + suppressedRecipientColumns + ` + FROM suppressed_recipients + WHERE organization_id = $1 + AND (expires_at IS NULL OR expires_at > NOW()) + AND ( + (kind = 'email' AND email = LOWER($2)) + OR (kind = 'domain' AND email = split_part(LOWER($2), '@', 2)) + ) + ORDER BY (kind = 'email') DESC + LIMIT 1 + ` + out, err := scanSuppressedRecipient(r.db.QueryRow(ctx, query, organizationID, email)) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil + } + return nil, err + } + return out, nil +} + func (r *advancedOutreachRepository) UpsertSuppressedRecipient(ctx context.Context, entry *models.SuppressedRecipient) error { metadata, err := marshalJSON(entry.Metadata) if err != nil { return err } + kind := entry.Kind + if kind == "" { + kind = models.SuppressionKindEmail + } query := ` - INSERT INTO suppressed_recipients (organization_id, email, reason, source, campaign_id, expires_at, metadata, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + INSERT INTO suppressed_recipients (organization_id, email, kind, reason, source, campaign_id, expires_at, metadata, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW(), NOW()) ON CONFLICT (organization_id, email) DO UPDATE SET + kind = EXCLUDED.kind, reason = EXCLUDED.reason, source = EXCLUDED.source, campaign_id = EXCLUDED.campaign_id, @@ -414,10 +452,113 @@ func (r *advancedOutreachRepository) UpsertSuppressedRecipient(ctx context.Conte metadata = EXCLUDED.metadata, updated_at = NOW() ` - _, err = r.db.Exec(ctx, query, entry.OrganizationID, strings.ToLower(strings.TrimSpace(entry.Email)), entry.Reason, entry.Source, entry.CampaignID, entry.ExpiresAt, metadata) + _, err = r.db.Exec(ctx, query, entry.OrganizationID, strings.ToLower(strings.TrimSpace(entry.Email)), kind, entry.Reason, entry.Source, entry.CampaignID, entry.ExpiresAt, metadata) return err } +const upsertSuppressedRecipientSQL = ` + INSERT INTO suppressed_recipients (organization_id, email, kind, reason, source, campaign_id, expires_at, metadata, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW(), NOW()) + ON CONFLICT (organization_id, email) + DO UPDATE SET + kind = EXCLUDED.kind, + reason = EXCLUDED.reason, + source = EXCLUDED.source, + campaign_id = EXCLUDED.campaign_id, + expires_at = EXCLUDED.expires_at, + metadata = EXCLUDED.metadata, + updated_at = NOW() + ` + +func (r *advancedOutreachRepository) UpsertSuppressedRecipients(ctx context.Context, entries []models.SuppressedRecipient) error { + if len(entries) == 0 { + return nil + } + tx, err := r.db.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + batch := &pgx.Batch{} + for i := range entries { + e := &entries[i] + metadata, err := marshalJSON(e.Metadata) + if err != nil { + return err + } + kind := e.Kind + if kind == "" { + kind = models.SuppressionKindEmail + } + batch.Queue(upsertSuppressedRecipientSQL, e.OrganizationID, strings.ToLower(strings.TrimSpace(e.Email)), kind, e.Reason, e.Source, e.CampaignID, e.ExpiresAt, metadata) + } + res := tx.SendBatch(ctx, batch) + for range entries { + if _, err := res.Exec(); err != nil { + _ = res.Close() + return err + } + } + if err := res.Close(); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (r *advancedOutreachRepository) ListSuppressedRecipients(ctx context.Context, organizationID uuid.UUID, q string, beforeAt *time.Time, beforeID *uuid.UUID, limit int) ([]models.SuppressedRecipient, error) { + args := []any{organizationID, limit} + where := `organization_id = $1 AND (expires_at IS NULL OR expires_at > NOW())` + if q = strings.ToLower(strings.TrimSpace(q)); q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND email ILIKE $%d`, len(args)) + } + if beforeAt != nil && beforeID != nil { + args = append(args, *beforeAt, *beforeID) + where += fmt.Sprintf(` AND (created_at, id) < ($%d, $%d)`, len(args)-1, len(args)) + } + rows, err := r.db.Query(ctx, `SELECT `+suppressedRecipientColumns+` FROM suppressed_recipients WHERE `+where+` ORDER BY created_at DESC, id DESC LIMIT $2`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.SuppressedRecipient, 0, limit) + for rows.Next() { + entry, err := scanSuppressedRecipient(rows) + if err != nil { + return nil, err + } + out = append(out, *entry) + } + return out, rows.Err() +} + +func (r *advancedOutreachRepository) GetSuppressedRecipient(ctx context.Context, organizationID, id uuid.UUID) (*models.SuppressedRecipient, error) { + out, err := scanSuppressedRecipient(r.db.QueryRow(ctx, `SELECT `+suppressedRecipientColumns+` FROM suppressed_recipients WHERE organization_id = $1 AND id = $2`, organizationID, id)) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil + } + return nil, err + } + return out, nil +} + +func (r *advancedOutreachRepository) DeleteSuppressedRecipient(ctx context.Context, organizationID, id uuid.UUID) (bool, error) { + tag, err := r.db.Exec(ctx, `DELETE FROM suppressed_recipients WHERE organization_id = $1 AND id = $2`, organizationID, id) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *advancedOutreachRepository) DeleteSuppressionByEmail(ctx context.Context, organizationID uuid.UUID, email string, source models.DeliverabilityEventType) (bool, error) { + tag, err := r.db.Exec(ctx, `DELETE FROM suppressed_recipients WHERE organization_id = $1 AND kind = 'email' AND email = LOWER($2) AND source = $3`, organizationID, strings.TrimSpace(email), source) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + func (r *advancedOutreachRepository) CreateDeliverabilityEvent(ctx context.Context, event *models.DeliverabilityEvent) error { metadata, err := marshalJSON(event.Metadata) if err != nil { @@ -523,7 +664,8 @@ func (r *advancedOutreachRepository) GetDeliverabilityDashboard(ctx context.Cont SELECT COUNT(*) FROM tasks t JOIN email_accounts ea ON ea.id = t.email_account_id WHERE ea.organization_id = $1 AND t.task_type = 'campaign' AND t.status = 'completed' - AND t.completed_at >= $2 AND t.completed_at <= $3` + AND t.completed_at >= $2 AND t.completed_at <= $3 + AND ` + taskDispatchedEmail _ = r.db.QueryRow(ctx, sentQuery, organizationID, from, to).Scan(&out.EmailsSent) out.BounceRate = models.Rate(out.BounceCount, out.EmailsSent) out.ComplaintRate = models.Rate(out.ComplaintCount, out.EmailsSent) diff --git a/internal/repository/pg_advisor_snapshot.go b/internal/repository/pg_advisor_snapshot.go index 06ef2d82..8f540fd6 100644 --- a/internal/repository/pg_advisor_snapshot.go +++ b/internal/repository/pg_advisor_snapshot.go @@ -70,6 +70,7 @@ func (r *advisorRepository) loadMailboxes(ctx context.Context, orgID uuid.UUID) WHERE t.email_account_id = ea.id AND t.task_type = 'campaign' AND t.status = 'completed' AND t.completed_at > NOW() - INTERVAL '30 days' + AND ` + taskDispatchedEmail + ` ) sent ON true LEFT JOIN LATERAL ( SELECT @@ -320,11 +321,7 @@ func (r *advisorRepository) loadListStats(ctx context.Context, orgID uuid.UUID) COUNT(*) AS total, COUNT(*) FILTER (WHERE split_part(lower(ct.email), '@', 1) IN (` + rolePrefixesSQL + `)) AS role_addresses, COUNT(*) FILTER (WHERE split_part(lower(ct.email), '@', 2) IN (` + freeMailDomainsSQL + `)) AS free_mail, - COUNT(*) FILTER (WHERE EXISTS ( - SELECT 1 FROM suppressed_recipients sr - WHERE sr.organization_id = $1 AND lower(sr.email) = lower(ct.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - )) AS suppressed, + COUNT(*) FILTER (WHERE recipient_suppressed($1, ct.email)) AS suppressed, COUNT(*) FILTER (WHERE ct.subscribed IS FALSE) AS unsubscribed, COUNT(*) FILTER (WHERE btrim(ct.first_name) = '') AS missing_first_name FROM campaign_leads cl diff --git a/internal/repository/pg_analytics.go b/internal/repository/pg_analytics.go index 940f46ba..2e6541ad 100644 --- a/internal/repository/pg_analytics.go +++ b/internal/repository/pg_analytics.go @@ -18,6 +18,11 @@ type AnalyticsRepository interface { GetCampaignSummary(ctx context.Context, userID, campaignID uuid.UUID) (*models.CampaignSummary, *errx.Error) GetCampaignDailyStats(ctx context.Context, campaignID uuid.UUID, from, to time.Time) ([]models.CampaignDailyStats, *errx.Error) GetSequenceStats(ctx context.Context, campaignID uuid.UUID) ([]models.SequenceStats, *errx.Error) + // GetCampaignEngagementBreakdown groups the campaign's human opens and + // clicks by country, client and device: distinct contacts per bucket, the + // busiest `limit` buckets of each. A click counts as an open, as it does + // on the progress row. + GetCampaignEngagementBreakdown(ctx context.Context, campaignID uuid.UUID, limit int) (*models.CampaignEngagementBreakdown, *errx.Error) // Email account status GetAccountsWithErrors(ctx context.Context, userID uuid.UUID) ([]uuid.UUID, *errx.Error) @@ -97,6 +102,13 @@ func (r *analyticsRepository) GetCampaignSummary(ctx context.Context, userID, ca COUNT(CASE WHEN ccp.opened_at IS NOT NULL THEN 1 END) as unique_opens, COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.opened_machine THEN 1 END) as machine_opens, COUNT(CASE WHEN ccp.clicked_at IS NOT NULL THEN 1 END) as unique_clicks, + COUNT(CASE WHEN ccp.clicked_at IS NULL AND EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND lc.machine + ) AND NOT EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND NOT lc.machine + ) THEN 1 END) as machine_clicks, COUNT(CASE WHEN ccp.replied_at IS NOT NULL THEN 1 END) as replies, COUNT(CASE WHEN ccp.bounced_at IS NOT NULL THEN 1 END) as bounces FROM campaign_contact_progress ccp @@ -114,6 +126,7 @@ func (r *analyticsRepository) GetCampaignSummary(ctx context.Context, userID, ca &summary.UniqueOpens, &summary.MachineOpens, &summary.UniqueClicks, + &summary.MachineClicks, &summary.Replies, &summary.Bounces, ) @@ -172,6 +185,70 @@ func (r *analyticsRepository) GetCampaignDailyStats(ctx context.Context, campaig return stats, nil } +func (r *analyticsRepository) GetCampaignEngagementBreakdown(ctx context.Context, campaignID uuid.UUID, limit int) (*models.CampaignEngagementBreakdown, *errx.Error) { + if limit <= 0 { + limit = 8 + } + // One query per dimension over the union of both logs; the key + // expression is the only difference. The client falls back to the + // browser so a plain webmail open still lands in a named bucket, and + // unknown stays the empty key. + bucket := func(keyExpr string) ([]models.EngagementBucket, *errx.Error) { + query := ` + WITH ev AS ( + SELECT contact_id, 'open' AS kind, client, browser, device_type, country_code + FROM email_opens + WHERE campaign_id = $1 AND NOT machine + UNION ALL + SELECT contact_id, 'click' AS kind, client, browser, device_type, country_code + FROM email_link_clicks + WHERE campaign_id = $1 AND NOT machine + ) + SELECT ` + keyExpr + ` AS key, + COUNT(DISTINCT contact_id) AS opens, + COUNT(DISTINCT contact_id) FILTER (WHERE kind = 'click') AS clicks + FROM ev + GROUP BY 1 + ORDER BY opens + clicks DESC, key ASC + LIMIT $2 + ` + rows, err := r.DB.Query(ctx, query, campaignID, limit) + if err != nil { + db.CaptureError(err, query, []any{campaignID, limit}, "GetCampaignEngagementBreakdown") + return nil, errx.InternalError() + } + defer rows.Close() + out := []models.EngagementBucket{} + for rows.Next() { + var b models.EngagementBucket + if err := rows.Scan(&b.Key, &b.Opens, &b.Clicks); err != nil { + db.CaptureError(err, "", nil, "GetCampaignEngagementBreakdown scan") + return nil, errx.InternalError() + } + out = append(out, b) + } + if err := rows.Err(); err != nil { + db.CaptureError(err, query, []any{campaignID, limit}, "GetCampaignEngagementBreakdown rows") + return nil, errx.InternalError() + } + return out, nil + } + + countries, xerr := bucket(`country_code`) + if xerr != nil { + return nil, xerr + } + clients, xerr := bucket(`COALESCE(NULLIF(client, ''), browser)`) + if xerr != nil { + return nil, xerr + } + devices, xerr := bucket(`CASE WHEN device_type = 'unknown' THEN '' ELSE device_type END`) + if xerr != nil { + return nil, xerr + } + return &models.CampaignEngagementBreakdown{Countries: countries, Clients: clients, Devices: devices}, nil +} + func (r *analyticsRepository) GetSequenceStats(ctx context.Context, campaignID uuid.UUID) ([]models.SequenceStats, *errx.Error) { query := ` SELECT @@ -345,6 +422,13 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_opens, COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.opened_machine AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as machine_opens, COUNT(CASE WHEN ccp.clicked_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_clicks, + COUNT(CASE WHEN ccp.clicked_at IS NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 AND EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND lc.machine + ) AND NOT EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND NOT lc.machine + ) THEN 1 END) as machine_clicks, COUNT(CASE WHEN ccp.replied_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_replies, COUNT(CASE WHEN ccp.bounced_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_bounces, (SELECT COUNT(*) FROM campaigns WHERE organization_id = $1 AND status = 'active') as active_campaigns, @@ -362,6 +446,7 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI &stats.TotalOpens, &stats.MachineOpens, &stats.TotalClicks, + &stats.MachineClicks, &stats.TotalReplies, &stats.TotalBounces, &stats.ActiveCampaigns, @@ -397,9 +482,13 @@ func (r *analyticsRepository) GetRecentActivity(ctx context.Context, orgID uuid. UNION ALL - -- Clicks + -- Clicks (the first link a person clicked on the step, when logged per link) SELECT 'clicked' as type, ccp.campaign_id, c.name as campaign_name, - co.email as contact_email, ccp.contact_id, ccp.clicked_at as timestamp, NULL as link + co.email as contact_email, ccp.contact_id, ccp.clicked_at as timestamp, + (SELECT lc.destination FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id + AND lc.sequence_id = ccp.sequence_id AND lc.machine = false + ORDER BY lc.clicked_at LIMIT 1) as link FROM campaign_contact_progress ccp JOIN campaigns c ON c.id = ccp.campaign_id JOIN contacts co ON co.id = ccp.contact_id diff --git a/internal/repository/pg_attachment.go b/internal/repository/pg_attachment.go index 764de7e7..750d9b3f 100644 --- a/internal/repository/pg_attachment.go +++ b/internal/repository/pg_attachment.go @@ -17,10 +17,50 @@ type AttachmentRepository interface { Create(ctx context.Context, att *models.CampaignAttachment) error GetByID(ctx context.Context, id uuid.UUID) (*models.CampaignAttachment, error) ListByCampaign(ctx context.Context, campaignID uuid.UUID) ([]models.CampaignAttachment, error) + // ListForStep returns what one step's send carries: the campaign-wide files + // (no sequence_id) plus the ones scoped to that step. uuid.Nil asks for the + // campaign-wide set alone. + ListForStep(ctx context.Context, campaignID, sequenceID uuid.UUID) ([]models.CampaignAttachment, error) + // StepBelongsToCampaign guards the sequence_id an upload names: the FK only + // proves the step exists, not that it is a step of this campaign, and one + // pointed at another campaign's step would never be sent by either. + StepBelongsToCampaign(ctx context.Context, campaignID, sequenceID uuid.UUID) (bool, error) Delete(ctx context.Context, id uuid.UUID) error // SumStorageUsedByOrg totals the bytes of every attachment owned by the org // (joined through campaigns) — the basis for the per-plan storage quota. SumStorageUsedByOrg(ctx context.Context, orgID uuid.UUID) (int64, error) + // CreateWithinQuota inserts the row only if the organization's total stays + // within the limit. The limit is resolved by limitFn AFTER the org's + // attachment lock (LockStorageQuota) is held, and the check and the insert + // share that transaction, so two uploads in flight cannot both read the + // same total and both pass, and a plan change cannot be raced past + // (issue #326). Returns created=false, the total it saw and the limit it + // applied when the file does not fit. + CreateWithinQuota(ctx context.Context, att *models.CampaignAttachment, orgID uuid.UUID, limitFn StorageLimitFunc) (created bool, used, limit int64, err error) +} + +// StorageLimitFunc resolves an organization's storage quota in bytes. It is +// called under the quota lock so the value cannot go stale before the insert. +type StorageLimitFunc func(ctx context.Context) (int64, error) + +// LockStorageQuota serialises quota checks for one organization inside the +// calling transaction. Every writer of campaign_attachments that checks the +// quota takes it first, so the sum it reads cannot go stale before its insert. +func LockStorageQuota(ctx context.Context, tx pgx.Tx, orgID uuid.UUID) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext('campaign_attachments'), hashtext($1::text))`, orgID.String()) + return err +} + +// storageUsedTx is SumStorageUsedByOrg inside a transaction. +func storageUsedTx(ctx context.Context, tx pgx.Tx, orgID uuid.UUID) (int64, error) { + var total int64 + err := tx.QueryRow(ctx, ` + SELECT COALESCE(SUM(ca.size), 0) + FROM campaign_attachments ca + JOIN campaigns c ON c.id = ca.campaign_id + WHERE c.organization_id = $1 + `, orgID).Scan(&total) + return total, err } type attachmentRepository struct { @@ -49,6 +89,38 @@ func (r *attachmentRepository) Create(ctx context.Context, att *models.CampaignA ), att) } +func (r *attachmentRepository) CreateWithinQuota(ctx context.Context, att *models.CampaignAttachment, orgID uuid.UUID, limitFn StorageLimitFunc) (bool, int64, int64, error) { + tx, err := r.DB.Begin(ctx) + if err != nil { + return false, 0, 0, err + } + defer func() { _ = tx.Rollback(ctx) }() + + if err := LockStorageQuota(ctx, tx, orgID); err != nil { + return false, 0, 0, err + } + limit, err := limitFn(ctx) + if err != nil { + return false, 0, 0, err + } + used, err := storageUsedTx(ctx, tx, orgID) + if err != nil { + return false, 0, limit, err + } + if used+att.Size > limit { + return false, used, limit, nil + } + if err := scanAttachment(tx.QueryRow(ctx, ` + INSERT INTO campaign_attachments (campaign_id, sequence_id, user_id, filename, size, mime_type, s3_key) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING `+attachmentCols, + att.CampaignID, att.SequenceID, att.UserID, att.Filename, att.Size, att.MimeType, att.S3Key, + ), att); err != nil { + return false, used, limit, err + } + return true, used + att.Size, limit, tx.Commit(ctx) +} + func (r *attachmentRepository) GetByID(ctx context.Context, id uuid.UUID) (*models.CampaignAttachment, error) { a := &models.CampaignAttachment{} err := scanAttachment(r.DB.QueryRow(ctx, `SELECT `+attachmentCols+` FROM campaign_attachments WHERE id = $1`, id), a) @@ -78,6 +150,34 @@ func (r *attachmentRepository) ListByCampaign(ctx context.Context, campaignID uu return out, rows.Err() } +func (r *attachmentRepository) ListForStep(ctx context.Context, campaignID, sequenceID uuid.UUID) ([]models.CampaignAttachment, error) { + rows, err := r.DB.Query(ctx, `SELECT `+attachmentCols+` + FROM campaign_attachments + WHERE campaign_id = $1 AND (sequence_id IS NULL OR sequence_id = $2) + ORDER BY created_at ASC`, campaignID, sequenceID) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.CampaignAttachment, 0) + for rows.Next() { + var a models.CampaignAttachment + if err := scanAttachment(rows, &a); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +func (r *attachmentRepository) StepBelongsToCampaign(ctx context.Context, campaignID, sequenceID uuid.UUID) (bool, error) { + var ok bool + err := r.DB.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM sequences WHERE id = $1 AND campaign_id = $2)`, + sequenceID, campaignID).Scan(&ok) + return ok, err +} + func (r *attachmentRepository) Delete(ctx context.Context, id uuid.UUID) error { _, err := r.DB.Exec(ctx, `DELETE FROM campaign_attachments WHERE id = $1`, id) return err diff --git a/internal/repository/pg_campaign.go b/internal/repository/pg_campaign.go index ec8f72e7..0c725573 100644 --- a/internal/repository/pg_campaign.go +++ b/internal/repository/pg_campaign.go @@ -36,7 +36,7 @@ type CampaignRepository interface { // Search filters by name substring, folder id, and status bucket // ("draft" | "active" | "paused" | "completed"; paused matches every // paused_* variant; empty means all). - Search(ctx context.Context, userID, query string, cursor, folder *string, status string, limit int32) (*models.CampaignsResult, error) + Search(ctx context.Context, userID, query string, cursor, folder *string, status, kind string, limit int32) (*models.CampaignsResult, error) // Overview returns status-bucket counts plus per-folder totals for the // campaigns browser sidebar. Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, error) @@ -107,6 +107,11 @@ type CampaignRepository interface { // still in flight back to active, so the failed step is retried instead of // being finalised as done. Returns true when the status changed. ReopenAfterSendFailure(ctx context.Context, campaignID uuid.UUID) (bool, error) + // MarkIdle stamps idle_since on an active continuous campaign that has no + // lead left to send to; true only when it was not already idle. + MarkIdle(ctx context.Context, campaignID uuid.UUID) (bool, error) + // ClearIdle removes the idle mark once there is something to send. + ClearIdle(ctx context.Context, campaignID uuid.UUID) error // ── Campaign-scoped tracking domain (feature 5) ───────────────────── // SetCampaignTrackingDomainVerified flips the verified flag / timestamp on @@ -144,7 +149,11 @@ const CAMPAIGN_SELECT = `id, name, description, status, schedule_windows, guardrail_enabled, guardrail_bounce_rate_max, guardrail_complaint_rate_max, guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days, - guardrail_tripped_at, guardrail_reason` + guardrail_tripped_at, guardrail_reason, + kind, + utm_tracking, utm_source, utm_medium, utm_campaign, + unsubscribe_mode, + continuous, idle_since` func getCampaign(rows db.Scannable, campaign *models.Campaign, extra ...any) error { var dest []any = []any{ @@ -163,6 +172,10 @@ func getCampaign(rows db.Scannable, campaign *models.Campaign, extra ...any) err &campaign.GuardrailEnabled, &campaign.GuardrailBounceRateMax, &campaign.GuardrailComplaintRateMax, &campaign.GuardrailReplyRateMin, &campaign.GuardrailMinSample, &campaign.GuardrailWindowDays, &campaign.GuardrailTrippedAt, &campaign.GuardrailReason, + &campaign.Kind, + &campaign.UTMTracking, &campaign.UTMSource, &campaign.UTMMedium, &campaign.UTMCampaign, + &campaign.UnsubscribeMode, + &campaign.Continuous, &campaign.IdleSince, } dest = append(dest, extra...) return rows.Scan( @@ -186,6 +199,10 @@ const CAMPAIGN_SELECT_FULL = ` c.guardrail_enabled, c.guardrail_bounce_rate_max, c.guardrail_complaint_rate_max, c.guardrail_reply_rate_min, c.guardrail_min_sample, c.guardrail_window_days, c.guardrail_tripped_at, c.guardrail_reason, + c.kind, + c.utm_tracking, c.utm_source, c.utm_medium, c.utm_campaign, + c.unsubscribe_mode, + c.continuous, c.idle_since, COALESCE(array_agg(cet.tag_id) FILTER (WHERE cet.tag_id IS NOT NULL), '{}') AS email_tag_ids, COALESCE(array_agg(cec.folder_id) FILTER (WHERE cec.folder_id IS NOT NULL), '{}') AS email_folder_ids ` @@ -230,6 +247,13 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u } endTime = *data.EndTime } + var scheduleWindows models.ScheduleWindows + if data.ScheduleWindows != nil { + if err := validate.CampaignScheduleWindows(data.ScheduleWindows); err != nil { + return nil, err + } + scheduleWindows = *data.ScheduleWindows + } if data.StartDate != nil { if err := validate.CampaignStartDate(*data.StartDate); err != nil { return nil, err @@ -279,6 +303,18 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u } } + kind := models.CampaignKindSequence + if data.Kind != nil && *data.Kind != "" { + if !models.ValidCampaignKind(*data.Kind) { + return nil, errx.New(errx.BadRequest, "kind must be sequence or one_time") + } + kind = *data.Kind + } + // A one-time email is one message; follow-ups belong in a sequence. + if kind == models.CampaignKindOneTime && len(data.Sequences) > 1 { + return nil, errx.New(errx.BadRequest, "a one-time email has a single step; create a sequence campaign for follow-ups") + } + cc := data.CC if cc == nil { cc = []string{} @@ -315,6 +351,33 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u if data.RiskyEmails != nil { riskyEmails = *data.RiskyEmails } + unsubMode := string(models.UnsubscribeModeInherit) + if data.UnsubscribeMode != nil { + if !models.ValidUnsubscribeMode(*data.UnsubscribeMode) { + return nil, errx.New(errx.BadRequest, "unsubscribe_mode must be inherit, text, link or off") + } + unsubMode = *data.UnsubscribeMode + } + + utmTracking := false + if data.UTMTracking != nil { + utmTracking = *data.UTMTracking + } + utmSource, utmMedium, utmCampaign := "", "", "" + if data.UTMSource != nil { + utmSource = strings.TrimSpace(*data.UTMSource) + } + if data.UTMMedium != nil { + utmMedium = strings.TrimSpace(*data.UTMMedium) + } + if data.UTMCampaign != nil { + utmCampaign = strings.TrimSpace(*data.UTMCampaign) + } + for _, v := range []string{utmSource, utmMedium, utmCampaign} { + if err := validate.CampaignUTMValue(v); err != nil { + return nil, err + } + } // ── Net-new send controls. Defaults reproduce today's behavior exactly. ── senderStrategy := "tags" @@ -370,6 +433,10 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u if data.PrioritizeNewLeads != nil { prioritizeNewLeads = *data.PrioritizeNewLeads } + continuous := false + if data.Continuous != nil { + continuous = *data.Continuous + } trackingDomain := "" if data.TrackingDomain != nil { // Normalize first so a pasted URL or a trailing dot is reduced to the @@ -406,22 +473,26 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u stop_on_reply, open_tracking, link_tracking, text_only, daily_limit, unsubscribe_header, risky_emails, cc_addr, bcc_addr, - start_date, end_date, timezone, days, start_time, end_time, + start_date, end_date, timezone, days, start_time, end_time, schedule_windows, sender_strategy, rotation_mode, ramp_enabled, ramp_start, ramp_increment, ramp_ceiling, esp_match_mode, max_new_leads_per_day, prioritize_new_leads, - tracking_domain, + tracking_domain, kind, + utm_tracking, utm_source, utm_medium, utm_campaign, + unsubscribe_mode, continuous, created_at, updated_at ) VALUES ( gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, - $14, $15, $16, $17, $18, $19, - $20, $21, - $22, $23, $24, $25, - $26, $27, $28, - $29, + $14, $15, $16, $17, $18, $19, $20, + $21, $22, + $23, $24, $25, $26, + $27, $28, $29, + $30, $31, + $32, $33, $34, $35, + $36, $37, NOW(), NOW() ) RETURNING %s @@ -447,16 +518,24 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u days, // $17 startTime, // $18 endTime, // $19 - senderStrategy, // $20 - rotationMode, // $21 - rampEnabled, // $22 - rampStart, // $23 - rampIncrement, // $24 - rampCeiling, // $25 - espMatchMode, // $26 - maxNewLeads, // $27 - prioritizeNewLeads, // $28 - trackingDomain, // $29 + scheduleWindows, // $20 + senderStrategy, // $21 + rotationMode, // $22 + rampEnabled, // $23 + rampStart, // $24 + rampIncrement, // $25 + rampCeiling, // $26 + espMatchMode, // $27 + maxNewLeads, // $28 + prioritizeNewLeads, // $29 + trackingDomain, // $30 + kind, // $31 + utmTracking, // $32 + utmSource, // $33 + utmMedium, // $34 + utmCampaign, // $35 + unsubMode, // $36 + continuous, // $37 } row := tx.QueryRow(ctx, insertSQL, params...) @@ -650,7 +729,7 @@ func (r *campaignRepository) Get(ctx context.Context, orgID, id string) (*models return &campaign, nil } -func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cursor, folder *string, status string, limit int32) (*models.CampaignsResult, error) { +func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cursor, folder *string, status, kind string, limit int32) (*models.CampaignsResult, error) { tx, err := r.DB.Begin(ctx) if err != nil { db.CaptureError(err, "", nil, "begin") @@ -675,6 +754,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu SELECT 1 FROM campaign_folders cf WHERE cf.campaign_id = c.id AND cf.folder_id = $4 )) AND ($5 = '' OR CASE WHEN $5 = 'paused' THEN c.status::text LIKE 'paused%%' ELSE c.status::text = $5 END) + AND ($6 = '' OR c.kind = $6) GROUP BY c.id ORDER BY created_at DESC LIMIT %d`, @@ -693,6 +773,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu SELECT 1 FROM campaign_folders cf WHERE cf.campaign_id = c.id AND cf.folder_id = $3 )) AND ($4 = '' OR CASE WHEN $4 = 'paused' THEN c.status::text LIKE 'paused%' ELSE c.status::text = $4 END) + AND ($5 = '' OR c.kind = $5) ` } @@ -702,6 +783,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu query, folder, status, + kind, } rows, err := tx.Query( @@ -744,9 +826,10 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu query, folder, status, + kind, } var tmp int64 - err = tx.QueryRow(ctx, countSQL, orgID, query, folder, status).Scan(&tmp) + err = tx.QueryRow(ctx, countSQL, params...).Scan(&tmp) if err != nil { db.CaptureError(err, countSQL, params, "queryrow") return nil, err @@ -773,7 +856,8 @@ func (r *campaignRepository) Overview(ctx context.Context, orgID string) (*model COUNT(*) FILTER (WHERE status = 'active'), COUNT(*) FILTER (WHERE status::text LIKE 'paused%'), COUNT(*) FILTER (WHERE status = 'draft'), - COUNT(*) FILTER (WHERE status = 'completed') + COUNT(*) FILTER (WHERE status = 'completed'), + COUNT(*) FILTER (WHERE kind = 'one_time') FROM campaigns WHERE organization_id = $1` err := r.DB.QueryRow(ctx, countsSQL, orgID).Scan( @@ -782,6 +866,7 @@ func (r *campaignRepository) Overview(ctx context.Context, orgID string) (*model &overview.Paused, &overview.Draft, &overview.Completed, + &overview.OneTime, ) if err != nil { db.CaptureError(err, countsSQL, []any{orgID}, "queryrow") @@ -888,6 +973,14 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri args = append(args, *data.RiskyEmails) argPos++ } + if data.UnsubscribeMode != nil { + if !models.ValidUnsubscribeMode(*data.UnsubscribeMode) { + return nil, errx.New(errx.BadRequest, "unsubscribe_mode must be inherit, text, link or off") + } + setClauses = append(setClauses, fmt.Sprintf("%s = $%d", "unsubscribe_mode", argPos)) + args = append(args, *data.UnsubscribeMode) + argPos++ + } if data.CC != nil { if !validate.EmailBulk(data.CC) { return nil, errx.ErrEmail @@ -1083,6 +1176,16 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri args = append(args, *data.PrioritizeNewLeads) argPos++ } + if data.Continuous != nil { + setClauses = append(setClauses, fmt.Sprintf("%s = $%d", "continuous", argPos)) + args = append(args, *data.Continuous) + argPos++ + // Turning it off ends the wait; the next reconcile pass finishes the + // campaign if there is still nothing to send. + if !*data.Continuous { + setClauses = append(setClauses, "idle_since = NULL") + } + } if data.TrackingDomain != nil { domain := config.NormalizeTrackingHost(*data.TrackingDomain) if err := validate.CampaignTrackingDomain(domain); err != nil { @@ -1095,6 +1198,27 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri // the CNAME is re-resolved (only a verified override is honored). setClauses = append(setClauses, "tracking_domain_verified = false", "tracking_domain_verified_at = NULL") } + if data.UTMTracking != nil { + setClauses = append(setClauses, fmt.Sprintf("%s = $%d", "utm_tracking", argPos)) + args = append(args, *data.UTMTracking) + argPos++ + } + for col, val := range map[string]*string{ + "utm_source": data.UTMSource, + "utm_medium": data.UTMMedium, + "utm_campaign": data.UTMCampaign, + } { + if val == nil { + continue + } + v := strings.TrimSpace(*val) + if err := validate.CampaignUTMValue(v); err != nil { + return nil, err + } + setClauses = append(setClauses, fmt.Sprintf("%s = $%d", col, argPos)) + args = append(args, v) + argPos++ + } // Auto-pause guardrails. Each rate is a percentage in [0,100] where 0 means // "this rule is off"; the DB CHECK mirrors these bounds, but rejecting here @@ -1251,6 +1375,10 @@ func (r *campaignRepository) GetByID(ctx context.Context, campaignID uuid.UUID) &campaign.GuardrailEnabled, &campaign.GuardrailBounceRateMax, &campaign.GuardrailComplaintRateMax, &campaign.GuardrailReplyRateMin, &campaign.GuardrailMinSample, &campaign.GuardrailWindowDays, &campaign.GuardrailTrippedAt, &campaign.GuardrailReason, + &campaign.Kind, + &campaign.UTMTracking, &campaign.UTMSource, &campaign.UTMMedium, &campaign.UTMCampaign, + &campaign.UnsubscribeMode, + &campaign.Continuous, &campaign.IdleSince, &campaign.EmailTags, &campaign.Folders, ) if err != nil { @@ -1373,7 +1501,7 @@ var validCampaignTransitions = map[string]map[string]bool{ // UpdateStatus updates only the status of a campaign with state machine validation func (r *campaignRepository) UpdateStatus(ctx context.Context, campaignID uuid.UUID, status string) error { - query := `UPDATE campaigns SET status = $1, updated_at = NOW() WHERE id = $2 AND status != $1` + query := `UPDATE campaigns SET status = $1, idle_since = NULL, updated_at = NOW() WHERE id = $2 AND status != $1` // Validate that the transition is allowed var currentStatus string @@ -1400,6 +1528,7 @@ func (r *campaignRepository) StartCampaign(ctx context.Context, campaignID uuid. SET status = 'active', last_status_change_at = NOW(), updated_at = NOW(), + idle_since = NULL, ramp_level = CASE WHEN ramp_enabled AND ramp_level = 0 THEN ramp_start ELSE ramp_level END, ramp_level_date = CASE WHEN ramp_enabled AND ramp_level = 0 THEN CURRENT_DATE ELSE ramp_level_date END, -- Starting again clears any auto-pause marker: the badge describes @@ -1414,7 +1543,7 @@ func (r *campaignRepository) StartCampaign(ctx context.Context, campaignID uuid. // StopCampaign sets campaign status to paused and updates last_status_change_at func (r *campaignRepository) StopCampaign(ctx context.Context, campaignID uuid.UUID) error { - query := `UPDATE campaigns SET status = 'paused', last_status_change_at = NOW(), updated_at = NOW() WHERE id = $1` + query := `UPDATE campaigns SET status = 'paused', idle_since = NULL, last_status_change_at = NOW(), updated_at = NOW() WHERE id = $1` _, err := r.DB.Exec(ctx, query, campaignID) return err } @@ -1431,13 +1560,17 @@ func (r *campaignRepository) ValidateCampaignReady(ctx context.Context, campaign return errx.New(errx.BadRequest, "campaign must have at least one sequence") } - // Check contacts + // Check contacts. A continuous campaign may start empty: it waits for + // leads instead of needing them up front. var contactCount int - err = r.DB.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_leads WHERE campaign_id = $1`, campaignID).Scan(&contactCount) + var continuous bool + err = r.DB.QueryRow(ctx, ` + SELECT (SELECT COUNT(*) FROM campaign_leads WHERE campaign_id = $1), + (SELECT continuous FROM campaigns WHERE id = $1)`, campaignID).Scan(&contactCount, &continuous) if err != nil { return err } - if contactCount == 0 { + if contactCount == 0 && !continuous { return errx.New(errx.BadRequest, "campaign must have at least one contact") } @@ -1515,6 +1648,7 @@ func (r *campaignRepository) ListCampaignScheduleCandidates(ctx context.Context, JOIN tasks t ON t.id = ct.task_id WHERE ct.campaign_id = c.id AND t.status = 'pending' ) + ORDER BY c.idle_since ASC NULLS FIRST LIMIT $1` rows, err := r.DB.Query(ctx, query, limit) @@ -1932,6 +2066,30 @@ func (r *campaignRepository) ReopenAfterSendFailure(ctx context.Context, campaig return tag.RowsAffected() > 0, nil } +// MarkIdle records that an active continuous campaign ran out of leads and is +// waiting for more. Returns true only on the transition, so the caller logs +// and broadcasts it once rather than on every pass that finds nothing. +func (r *campaignRepository) MarkIdle(ctx context.Context, campaignID uuid.UUID) (bool, error) { + tag, err := r.DB.Exec(ctx, ` + UPDATE campaigns + SET idle_since = NOW(), updated_at = NOW() + WHERE id = $1 AND status = 'active' AND continuous AND idle_since IS NULL + `, campaignID) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +// ClearIdle ends the wait once the campaign has something to send again. +func (r *campaignRepository) ClearIdle(ctx context.Context, campaignID uuid.UUID) error { + _, err := r.DB.Exec(ctx, ` + UPDATE campaigns SET idle_since = NULL, updated_at = NOW() + WHERE id = $1 AND idle_since IS NOT NULL + `, campaignID) + return err +} + // CountNewLeadsStartedToday returns new_leads_started for the current UTC day. func (r *campaignRepository) CountNewLeadsStartedToday(ctx context.Context, campaignID uuid.UUID) (int, error) { var n int @@ -1980,7 +2138,7 @@ func (r *campaignRepository) UpdateStatusWithLock(ctx context.Context, campaignI return err } - query := `UPDATE campaigns SET status = $1, last_status_change_at = NOW(), updated_at = NOW() WHERE id = $2 AND status = 'active'` + query := `UPDATE campaigns SET status = $1, idle_since = NULL, last_status_change_at = NOW(), updated_at = NOW() WHERE id = $2 AND status = 'active'` _, err = tx.Exec(ctx, query, status, campaignID) if err != nil { db.CaptureError(err, query, []any{status, campaignID}, "exec") diff --git a/internal/repository/pg_campaign_audience.go b/internal/repository/pg_campaign_audience.go index 9ba3b1af..153654eb 100644 --- a/internal/repository/pg_campaign_audience.go +++ b/internal/repository/pg_campaign_audience.go @@ -52,11 +52,7 @@ func (r *campaignAudienceRepository) GetCampaignAudience(ctx context.Context, or // not told two different numbers for the same list. // sendable is the same predicate Deliverable counts, reused so every // verification count shares one denominator. - const sendable = `ct.subscribed IS NOT FALSE AND NOT EXISTS ( - SELECT 1 FROM suppressed_recipients sr - WHERE sr.organization_id = $1 AND lower(sr.email) = lower(ct.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - )` + const sendable = `ct.subscribed IS NOT FALSE AND NOT recipient_suppressed($1, ct.email)` err := r.DB.Pool.QueryRow(ctx, ` SELECT COUNT(*), @@ -66,11 +62,7 @@ func (r *campaignAudienceRepository) GetCampaignAudience(ctx context.Context, or -- has checked. NOT NULL, so no null branch is needed. COUNT(*) FILTER (WHERE `+sendable+` AND ct.verification_status NOT IN ('valid','invalid','risky')), COUNT(*) FILTER (WHERE `+sendable+` AND ct.is_catch_all), - COUNT(*) FILTER (WHERE EXISTS ( - SELECT 1 FROM suppressed_recipients sr - WHERE sr.organization_id = $1 AND lower(sr.email) = lower(ct.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - )), + COUNT(*) FILTER (WHERE recipient_suppressed($1, ct.email)), COUNT(*) FILTER (WHERE ct.subscribed IS FALSE), COUNT(*) FILTER (WHERE `+sendable+`), COUNT(*) FILTER (WHERE split_part(lower(ct.email), '@', 1) IN (`+rolePrefixesSQL+`)), diff --git a/internal/repository/pg_campaign_lifecycle.go b/internal/repository/pg_campaign_lifecycle.go index 37738ed9..5a87ac33 100644 --- a/internal/repository/pg_campaign_lifecycle.go +++ b/internal/repository/pg_campaign_lifecycle.go @@ -3,6 +3,7 @@ package repository import ( "context" "encoding/json" + "errors" "fmt" "github.com/google/uuid" @@ -21,8 +22,18 @@ type DuplicateCampaignInput struct { UserID uuid.UUID Name string Attachments []models.CampaignAttachment + // OrganizationID and StorageLimit make the copied attachments count + // against the quota inside the same transaction that inserts them; the + // limit is resolved under the quota lock. A nil StorageLimit skips the + // check. + OrganizationID uuid.UUID + StorageLimit StorageLimitFunc } +// ErrStorageQuotaExceeded is returned by Duplicate when the copied attachments +// would take the organization past StorageLimitBytes. Nothing is written. +var ErrStorageQuotaExceeded = errors.New("storage quota exceeded") + // Delete removes a campaign and everything that only means something inside // it. The pending tasks parked for the campaign (its wakeup chain and any // not-yet-dispatched sends) go in the same transaction: campaign_tasks only @@ -99,12 +110,13 @@ func (r *campaignRepository) Duplicate(ctx context.Context, in DuplicateCampaign contact_order_by, contact_order_dir, contact_order_field, sender_strategy, rotation_mode, ramp_enabled, ramp_start, ramp_increment, ramp_ceiling, ramp_level, ramp_level_date, - esp_match_mode, max_new_leads_per_day, prioritize_new_leads, + esp_match_mode, max_new_leads_per_day, prioritize_new_leads, continuous, tracking_domain, tracking_domain_verified, tracking_domain_verified_at, guardrail_enabled, guardrail_bounce_rate_max, guardrail_complaint_rate_max, guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days, guardrail_tripped_at, guardrail_reason, - last_status_change_at, updated_at, created_at + utm_tracking, utm_source, utm_medium, utm_campaign, + last_status_change_at, updated_at, created_at, kind ) SELECT $2, $3, organization_id, $4, description, 'draft', @@ -117,12 +129,13 @@ func (r *campaignRepository) Duplicate(ctx context.Context, in DuplicateCampaign contact_order_by, contact_order_dir, contact_order_field, sender_strategy, rotation_mode, ramp_enabled, ramp_start, ramp_increment, ramp_ceiling, 0, NULL, - esp_match_mode, max_new_leads_per_day, prioritize_new_leads, + esp_match_mode, max_new_leads_per_day, prioritize_new_leads, continuous, tracking_domain, tracking_domain_verified, tracking_domain_verified_at, guardrail_enabled, guardrail_bounce_rate_max, guardrail_complaint_rate_max, guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days, NULL, '', - NULL, NOW(), NOW() + utm_tracking, utm_source, utm_medium, utm_campaign, + NULL, NOW(), NOW(), kind FROM campaigns WHERE id = $1 ` @@ -160,6 +173,29 @@ func (r *campaignRepository) Duplicate(ctx context.Context, in DuplicateCampaign return nil, err } + if len(in.Attachments) > 0 && in.StorageLimit != nil { + if err := LockStorageQuota(ctx, tx, in.OrganizationID); err != nil { + db.CaptureError(err, "", nil, "exec") + return nil, err + } + limit, err := in.StorageLimit(ctx) + if err != nil { + return nil, err + } + used, err := storageUsedTx(ctx, tx, in.OrganizationID) + if err != nil { + db.CaptureError(err, "", nil, "queryrow") + return nil, err + } + var adding int64 + for _, att := range in.Attachments { + adding += att.Size + } + if used+adding > limit { + return nil, fmt.Errorf("%w: %d of %d bytes used, %d to add", ErrStorageQuotaExceeded, used, limit, adding) + } + } + const insertAttachment = ` INSERT INTO campaign_attachments (campaign_id, sequence_id, user_id, filename, size, mime_type, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7) diff --git a/internal/repository/pg_campaign_progress.go b/internal/repository/pg_campaign_progress.go index 977f24f2..697d5812 100644 --- a/internal/repository/pg_campaign_progress.go +++ b/internal/repository/pg_campaign_progress.go @@ -125,6 +125,14 @@ type CampaignProgressRepository interface { HasSentSteps(ctx context.Context, campaignID, contactID uuid.UUID) (bool, error) RecordEmailOpened(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, machine bool) error RecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error + // UnrecordEmailClicked clears clicked_at when every logged click on the + // step turned out to be automated (a burst recognised after the first + // click already stamped it). clicked_at keeps meaning "a person clicked". + UnrecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error + // GetStepSentAt returns when the step was dispatched (nil when it was not), + // the reference point for telling an instant machine open or click from a + // person's. + GetStepSentAt(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (*time.Time, error) RecordEmailReplied(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error RecordEmailBounced(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error RecordEmailComplained(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error @@ -427,10 +435,17 @@ func (r *campaignProgressRepository) RecordEmailOpened(ctx context.Context, camp } // RecordEmailClicked records that an email link was clicked +// RecordEmailClicked stamps a person's click. It also counts as an open: +// the person had the email in front of them whatever the pixel saw, so a +// client that blocks images no longer reads "clicked, not opened". The +// implied open shares the click's timestamp, which is how UnrecordEmailClicked +// tells it from a pixel open. func (r *campaignProgressRepository) RecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error { query := ` UPDATE campaign_contact_progress - SET clicked_at = NOW() + SET clicked_at = NOW(), + opened_at = COALESCE(opened_at, NOW()), + opened_machine = false WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 @@ -441,6 +456,59 @@ func (r *campaignProgressRepository) RecordEmailClicked(ctx context.Context, cam return err } +// UnrecordEmailClicked walks a click stamp back once no human click remains +// on the step. Guarded by the click log so a concurrent human click is never +// erased, and only a stamp written alongside a logged click is touched: a +// stamp older than the step's earliest logged click predates per-link +// logging, so it came from a person the log never saw. +func (r *campaignProgressRepository) UnrecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error { + query := ` + UPDATE campaign_contact_progress ccp + SET clicked_at = NULL, + opened_at = CASE + WHEN ccp.opened_at = ccp.clicked_at AND NOT EXISTS ( + SELECT 1 FROM email_opens o + WHERE o.campaign_id = $1 AND o.contact_id = $2 AND o.sequence_id = $3 AND o.machine = false + ) THEN NULL + ELSE ccp.opened_at + END + WHERE ccp.campaign_id = $1 + AND ccp.contact_id = $2 + AND ccp.sequence_id = $3 + AND ccp.clicked_at IS NOT NULL + AND ccp.clicked_at >= ( + SELECT MIN(lc.clicked_at) - INTERVAL '1 minute' FROM email_link_clicks lc + WHERE lc.campaign_id = $1 AND lc.contact_id = $2 AND lc.sequence_id = $3 + ) + AND NOT EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = $1 AND lc.contact_id = $2 AND lc.sequence_id = $3 AND lc.machine = false + ) + ` + + _, err := r.db.Exec(ctx, query, campaignID, contactID, sequenceID) + return err +} + +// GetStepSentAt returns the step's dispatch time, or nil when unsent/unknown. +func (r *campaignProgressRepository) GetStepSentAt(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (*time.Time, error) { + query := ` + SELECT LEAST(dispatched_at, sent_at) + FROM campaign_contact_progress + WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 + ` + + var sentAt *time.Time + err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID).Scan(&sentAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return sentAt, nil +} + // RecordEmailReplied records that a contact replied func (r *campaignProgressRepository) RecordEmailReplied(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error { query := ` @@ -672,10 +740,14 @@ func (r *campaignProgressRepository) GetCampaignRollingRates(ctx context.Context return out, err } -// GetContactProgress retrieves progress for a specific contact in a campaign +// GetContactProgress retrieves progress for a specific contact in a campaign. +// A machine open comes back as NULL: this feeds routing and instant actions, +// and an automated fetch is not intent (machine clicks never stamp at all). func (r *campaignProgressRepository) GetContactProgress(ctx context.Context, campaignID, contactID uuid.UUID) ([]CampaignContactProgress, error) { query := ` - SELECT campaign_id, contact_id, sequence_id, sent_at, opened_at, clicked_at, replied_at, bounced_at, complained_at, COALESCE(reply_class, ''), COALESCE(ai_label, '') + SELECT campaign_id, contact_id, sequence_id, sent_at, + CASE WHEN opened_machine THEN NULL ELSE opened_at END, + clicked_at, replied_at, bounced_at, complained_at, COALESCE(reply_class, ''), COALESCE(ai_label, '') FROM campaign_contact_progress WHERE campaign_id = $1 AND contact_id = $2 ORDER BY sent_at ASC @@ -747,12 +819,15 @@ func (r *campaignProgressRepository) CheckContactHasReplied(ctx context.Context, return hasReplied, err } -// CountEmailsSentTodayByOrganization returns how many campaign emails were sent today by an organization. +// CountEmailsSentTodayByOrganization returns how many campaign emails were sent +// today by an organization. Action and wait steps stamp sent_at too, for +// routing, but send nothing, so only email steps count. func (r *campaignProgressRepository) CountEmailsSentTodayByOrganization(ctx context.Context, organizationID uuid.UUID) (int, error) { query := ` SELECT COUNT(*) FROM campaign_contact_progress ccp JOIN campaigns c ON c.id = ccp.campaign_id + JOIN sequences s ON s.id = ccp.sequence_id AND s.kind = 'email' WHERE c.organization_id = $1 AND ccp.sent_at IS NOT NULL AND DATE(ccp.sent_at) = CURRENT_DATE @@ -884,7 +959,9 @@ func (r *campaignProgressRepository) FindNextRoutedPair(ctx context.Context, cam FROM campaign_leads cl JOIN contacts c ON c.id = cl.contact_id LEFT JOIN LATERAL ( - SELECT sequence_id, sent_at, opened_at, clicked_at, replied_at, reply_class, ai_label + SELECT sequence_id, sent_at, + CASE WHEN p.opened_machine THEN NULL ELSE p.opened_at END AS opened_at, + clicked_at, replied_at, reply_class, ai_label FROM campaign_contact_progress p WHERE p.campaign_id = $1 AND p.contact_id = cl.contact_id AND p.sent_at IS NOT NULL ORDER BY p.sent_at DESC LIMIT 1 @@ -906,13 +983,11 @@ func (r *campaignProgressRepository) FindNextRoutedPair(ctx context.Context, cam AND f.sent_at IS NULL AND f.failed_at IS NOT NULL AND f.send_attempts >= $2 ) - AND NOT EXISTS ( - SELECT 1 FROM suppressed_recipients sr - JOIN campaigns camp ON camp.organization_id = sr.organization_id - WHERE camp.id = $1 - AND LOWER(sr.email) = LOWER(c.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - ) + -- The workspace suppression list (addresses and domains) and the + -- contact's own subscription flag are both send gates; the audience + -- count applies the same two, so the number shown is the number sent. + AND NOT recipient_suppressed((SELECT organization_id FROM campaigns WHERE id = $1), c.email) + AND c.subscribed IS NOT FALSE -- Addresses the pre-send gates in the campaign task would refuse. -- Without this the finder keeps handing back the same undeliverable -- contact, the task skips it, and the campaign never reaches the @@ -1019,18 +1094,15 @@ func (r *campaignProgressRepository) RouteContact(ctx context.Context, campaignI AND f.sent_at IS NULL AND f.failed_at IS NOT NULL AND f.send_attempts >= $2 ) AS failed, - EXISTS ( - SELECT 1 FROM suppressed_recipients sr - JOIN campaigns camp ON camp.organization_id = sr.organization_id - WHERE camp.id = $1 - AND LOWER(sr.email) = LOWER(c.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - ) AS suppressed, + (recipient_suppressed((SELECT organization_id FROM campaigns WHERE id = $1), c.email) + OR c.subscribed IS FALSE) AS suppressed, ` + undeliverableClause("$1") + ` AS undeliverable FROM campaign_leads cl JOIN contacts c ON c.id = cl.contact_id LEFT JOIN LATERAL ( - SELECT sequence_id, sent_at, opened_at, clicked_at, replied_at, reply_class, ai_label + SELECT sequence_id, sent_at, + CASE WHEN p.opened_machine THEN NULL ELSE p.opened_at END AS opened_at, + clicked_at, replied_at, reply_class, ai_label FROM campaign_contact_progress p WHERE p.campaign_id = $1 AND p.contact_id = cl.contact_id AND p.sent_at IS NOT NULL ORDER BY p.sent_at DESC LIMIT 1 @@ -1393,7 +1465,9 @@ func (r *campaignProgressRepository) CountUndeliverableLeads(ctx context.Context FROM campaign_leads cl JOIN contacts c ON c.id = cl.contact_id LEFT JOIN LATERAL ( - SELECT sequence_id, sent_at, opened_at, clicked_at, replied_at, reply_class, ai_label + SELECT sequence_id, sent_at, + CASE WHEN p.opened_machine THEN NULL ELSE p.opened_at END AS opened_at, + clicked_at, replied_at, reply_class, ai_label FROM campaign_contact_progress p WHERE p.campaign_id = $1 AND p.contact_id = cl.contact_id AND p.sent_at IS NOT NULL ORDER BY p.sent_at DESC LIMIT 1 @@ -1415,13 +1489,11 @@ func (r *campaignProgressRepository) CountUndeliverableLeads(ctx context.Context AND f.sent_at IS NULL AND f.failed_at IS NOT NULL AND f.send_attempts >= $2 ) - AND NOT EXISTS ( - SELECT 1 FROM suppressed_recipients sr - JOIN campaigns camp ON camp.organization_id = sr.organization_id - WHERE camp.id = $1 - AND LOWER(sr.email) = LOWER(c.email) - AND (sr.expires_at IS NULL OR sr.expires_at > NOW()) - ) + -- The workspace suppression list (addresses and domains) and the + -- contact's own subscription flag are both send gates; the audience + -- count applies the same two, so the number shown is the number sent. + AND NOT recipient_suppressed((SELECT organization_id FROM campaigns WHERE id = $1), c.email) + AND c.subscribed IS NOT FALSE AND ` + undeliverableClause("$1") + ` ` rows, err := r.db.Query(ctx, query, campaignID, config.CampaignSendMaxAttempts) diff --git a/internal/repository/pg_cliauth.go b/internal/repository/pg_cliauth.go new file mode 100644 index 00000000..531dfaea --- /dev/null +++ b/internal/repository/pg_cliauth.go @@ -0,0 +1,159 @@ +package repository + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// CLIAuthRepository stores the `warmbly auth login` handshake. Rows live for +// minutes: the credential the flow produces is an ordinary API key. +type CLIAuthRepository interface { + CreateCode(ctx context.Context, deviceCodeHash, userCode string, req models.CLIAuthStartRequest, expiresAt time.Time) (*models.CLIAuthCode, error) + GetCodeByUserCode(ctx context.Context, userCode string) (*models.CLIAuthCode, error) + // ApproveCode stores the minted secret for the next poll; false when the + // code is no longer pending, which is what makes approval single-use. + ApproveCode(ctx context.Context, userCode string, orgID, approvedBy, apiKeyID uuid.UUID, secret string) (bool, error) + DenyCode(ctx context.Context, userCode string) (bool, error) + // ClaimCode hands the secret out exactly once, clearing it in the same statement. + ClaimCode(ctx context.Context, deviceCodeHash string) (*models.CLIAuthCode, string, error) + DeleteExpiredCodes(ctx context.Context) error +} + +type cliAuthRepository struct { + db *pgxpool.Pool +} + +func NewCLIAuthRepository(db *pgxpool.Pool) CLIAuthRepository { + return &cliAuthRepository{db: db} +} + +const cliAuthCodeColumns = `id, user_code, client_name, hostname, cli_version, scopes, status, organization_id, expires_at, created_at` + +func scanCLIAuthCode(row pgx.Row) (*models.CLIAuthCode, error) { + var c models.CLIAuthCode + var scopes int64 + if err := row.Scan(&c.ID, &c.UserCode, &c.ClientName, &c.Hostname, &c.CLIVersion, &scopes, &c.Status, &c.OrganizationID, &c.ExpiresAt, &c.CreatedAt); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, err + } + c.Scopes = uint64(scopes) + c.ScopeNames = models.APIScopeNames(c.Scopes) + return &c, nil +} + +func (r *cliAuthRepository) CreateCode(ctx context.Context, deviceCodeHash, userCode string, req models.CLIAuthStartRequest, expiresAt time.Time) (*models.CLIAuthCode, error) { + query := ` + INSERT INTO cli_auth_codes (device_code_hash, user_code, client_name, hostname, cli_version, scopes, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING ` + cliAuthCodeColumns + c, err := scanCLIAuthCode(r.db.QueryRow(ctx, query, deviceCodeHash, userCode, req.ClientName, req.Hostname, req.CLIVersion, int64(req.Scopes), expiresAt)) + if err != nil { + db.CaptureError(err, query, nil, "queryrow") + return nil, err + } + return c, nil +} + +func (r *cliAuthRepository) GetCodeByUserCode(ctx context.Context, userCode string) (*models.CLIAuthCode, error) { + query := `SELECT ` + cliAuthCodeColumns + ` FROM cli_auth_codes WHERE user_code = $1 AND expires_at > NOW()` + c, err := scanCLIAuthCode(r.db.QueryRow(ctx, query, userCode)) + if err != nil { + db.CaptureError(err, query, []any{userCode}, "queryrow") + return nil, err + } + return c, nil +} + +func (r *cliAuthRepository) ApproveCode(ctx context.Context, userCode string, orgID, approvedBy, apiKeyID uuid.UUID, secret string) (bool, error) { + query := ` + UPDATE cli_auth_codes + SET status = 'approved', organization_id = $2, approved_by = $3, api_key_id = $4, api_key_secret = $5 + WHERE user_code = $1 AND status = 'pending' AND expires_at > NOW() + ` + tag, err := r.db.Exec(ctx, query, userCode, orgID, approvedBy, apiKeyID, secret) + if err != nil { + db.CaptureError(err, query, nil, "exec") + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (r *cliAuthRepository) DenyCode(ctx context.Context, userCode string) (bool, error) { + query := `UPDATE cli_auth_codes SET status = 'denied' WHERE user_code = $1 AND status = 'pending'` + tag, err := r.db.Exec(ctx, query, userCode) + if err != nil { + db.CaptureError(err, query, []any{userCode}, "exec") + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (r *cliAuthRepository) ClaimCode(ctx context.Context, deviceCodeHash string) (*models.CLIAuthCode, string, error) { + // The secret comes from the locked pre-update row; RETURNING would only + // see the cleared value. + query := ` + WITH picked AS ( + SELECT id, api_key_secret + FROM cli_auth_codes + WHERE device_code_hash = $1 AND status = 'approved' AND expires_at > NOW() + FOR UPDATE + ), claimed AS ( + UPDATE cli_auth_codes p + SET status = 'claimed', api_key_secret = NULL + FROM picked + WHERE p.id = picked.id + RETURNING p.id, p.user_code, p.client_name, p.hostname, p.cli_version, p.scopes, p.status, p.organization_id, p.expires_at, p.created_at, picked.api_key_secret AS secret + ) + SELECT id, user_code, client_name, hostname, cli_version, scopes, status, organization_id, expires_at, created_at, COALESCE(secret, '') FROM claimed + UNION ALL + SELECT ` + cliAuthCodeColumns + `, '' FROM cli_auth_codes + WHERE device_code_hash = $1 AND expires_at > NOW() AND NOT EXISTS (SELECT 1 FROM claimed) + LIMIT 1 + ` + var c models.CLIAuthCode + var scopes int64 + var secret string + err := r.db.QueryRow(ctx, query, deviceCodeHash).Scan(&c.ID, &c.UserCode, &c.ClientName, &c.Hostname, &c.CLIVersion, &scopes, &c.Status, &c.OrganizationID, &c.ExpiresAt, &c.CreatedAt, &secret) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", nil + } + db.CaptureError(err, query, nil, "queryrow") + return nil, "", err + } + c.Scopes = uint64(scopes) + c.ScopeNames = models.APIScopeNames(c.Scopes) + // The claimed row reports its new status; the caller wants "approved". + if secret != "" { + c.Status = models.CLIAuthCodeApproved + } + return &c, secret, nil +} + +// DeleteExpiredCodes both destroys the plaintext secret on any expired row and +// removes rows old enough to be of no interest. +// +// The two are separate on purpose. An approved code the CLI never came back +// for would otherwise keep a usable key in plaintext for as long as the row +// survived, which is exactly what the "held only between approval and the next +// poll" intent rules out. Blanking it the moment the code expires bounds that +// to the code's own ten minutes. The key itself stays: it was legitimately +// created and is listed under Settings > API keys, but nobody holds its secret. +func (r *cliAuthRepository) DeleteExpiredCodes(ctx context.Context) error { + query := `UPDATE cli_auth_codes SET api_key_secret = NULL WHERE api_key_secret IS NOT NULL AND expires_at < NOW()` + if _, err := r.db.Exec(ctx, query); err != nil { + db.CaptureError(err, query, nil, "exec") + return err + } + _, err := r.db.Exec(ctx, `DELETE FROM cli_auth_codes WHERE expires_at < NOW() - INTERVAL '1 day'`) + return err +} diff --git a/internal/repository/pg_contact.go b/internal/repository/pg_contact.go index d7105d76..c3d44b15 100644 --- a/internal/repository/pg_contact.go +++ b/internal/repository/pg_contact.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/url" "sort" "strconv" "strings" @@ -74,6 +75,10 @@ type ContactRepository interface { ExportAll(ctx context.Context, orgID string, filters *models.SearchContacts, contactIDs []string, max int) ([]models.Contact, *errx.Error) BulkUpdate(ctx context.Context, userID string, orgID uuid.UUID, data *models.BulkEditContactsData) ([]models.Contact, *errx.Error) Update(ctx context.Context, userID, contactID string, orgID uuid.UUID, data *models.UpdateContact) (*models.Contact, *errx.Error) + // SetSubscribedByEmail flips the subscription flag on every contact in the + // organization with that address; a recipient's own opt-out or + // resubscribe is recorded on the contact as well as the suppression list. + SetSubscribedByEmail(ctx context.Context, orgID uuid.UUID, email string, subscribed bool) error BulkDelete(ctx context.Context, userID string, orgID uuid.UUID, contactIDs []string) *errx.Error Delete(ctx context.Context, userID string, orgID uuid.UUID, contactID string) *errx.Error GetContactCount(ctx context.Context, userID string) (int, *errx.Error) @@ -87,7 +92,7 @@ type ContactRepository interface { // + deliverability + reply joins are skipped (they're org-scoped). GetDetail(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID) (*models.ContactDetail, *errx.Error) ListSentEmails(ctx context.Context, userID, contactID uuid.UUID, limit int, beforeSentAt *time.Time, beforeTaskID *uuid.UUID) (*models.ContactSentEmailsResult, *errx.Error) - ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, before *time.Time) (*models.ContactTimelineResult, *errx.Error) + ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, cursor *models.ContactTimelineKey) (*models.ContactTimelineResult, *errx.Error) // ListCampaignStates returns the contact's campaigns with their flow, // this contact's progress on every step, and the derived lead status. ListCampaignStates(ctx context.Context, orgID, contactID uuid.UUID) ([]models.ContactCampaignState, *errx.Error) @@ -367,6 +372,7 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U if ncon.CustomFields == nil { ncon.CustomFields = map[string]string{} } + ncon.IsNew = inserted ncontacts = append(ncontacts, ncon) created = append(created, inserted) if inserted { @@ -409,6 +415,11 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U return nil, errx.InternalError() } campaignLinks = append(campaignLinks, added...) + // A hand-picked add ends a manual removal, as the bulk add does. + if _, err := tx.Exec(ctx, `DELETE FROM campaign_lead_removals WHERE contact_id = $1 AND campaign_id = ANY($2)`, ncontacts[i].ID, cids); err != nil { + db.CaptureError(err, "", nil, "campaign_lead_removals clear") + return nil, errx.InternalError() + } // A contact created from a campaign's Leads tab is attributed to that // campaign by name, resolved here rather than trusted from the client. if created[i] && normalized[i].Source == models.ContactSourceCampaign && normalized[i].SourceDetail == "" && len(added) > 0 { @@ -604,6 +615,14 @@ func (r *contactRepository) SetContactESP(ctx context.Context, contactID uuid.UU // UpdateContactVerification stores the outcome of a verification pass on the // contact. It is keyed only by contact id (the verifier runs in the control // plane, not in a user request) and is a no-op-safe single UPDATE. +func (r *contactRepository) SetSubscribedByEmail(ctx context.Context, orgID uuid.UUID, email string, subscribed bool) error { + _, err := r.DB.Exec(ctx, + `UPDATE contacts SET subscribed = $3, updated_at = NOW() + WHERE organization_id = $1 AND LOWER(email) = LOWER($2) AND subscribed IS DISTINCT FROM $3`, + orgID, email, subscribed) + return err +} + func (r *contactRepository) UpdateContactVerification(ctx context.Context, contactID uuid.UUID, res emailverify.Result) *errx.Error { status := string(res.Status) if status == "" { @@ -2785,15 +2804,16 @@ func (r *contactRepository) GetDetail(ctx context.Context, userID uuid.UUID, org // 2. Engagement aggregates. campaign_contact_progress is the canonical // sent/opened/clicked/replied/bounced ledger keyed by (campaign, // contact, sequence). Counts come from non-null timestamp columns, - // "last X" comes from MAX() of each. + // "last X" comes from MAX() of each. Opens count people only: a machine + // open is a delivery signal, not engagement, here as in analytics. engQuery := ` SELECT COUNT(*) FILTER (WHERE sent_at IS NOT NULL) AS sent, - COUNT(*) FILTER (WHERE opened_at IS NOT NULL) AS opened, + COUNT(*) FILTER (WHERE opened_at IS NOT NULL AND NOT opened_machine) AS opened, COUNT(*) FILTER (WHERE clicked_at IS NOT NULL) AS clicked, COUNT(*) FILTER (WHERE replied_at IS NOT NULL) AS replied, COUNT(*) FILTER (WHERE bounced_at IS NOT NULL) AS bounced, - MAX(sent_at), MAX(opened_at), MAX(clicked_at), MAX(replied_at), MAX(bounced_at) + MAX(sent_at), MAX(opened_at) FILTER (WHERE NOT opened_machine), MAX(clicked_at), MAX(replied_at), MAX(bounced_at) FROM campaign_contact_progress WHERE contact_id = $1 ` @@ -2829,16 +2849,21 @@ func (r *contactRepository) GetDetail(ctx context.Context, userID uuid.UUID, org return nil, errx.InternalError() } - // Suppression — there's at most one row per (org, email) - // thanks to the unique constraint. + // Suppression: the contact's own address row wins over a row for its + // whole domain, so the card names the entry that actually blocks mail. suppQuery := ` - SELECT reason, source, expires_at, created_at + SELECT id, kind, email, reason, source, expires_at, created_at FROM suppressed_recipients - WHERE organization_id = $1 AND LOWER(email) = LOWER($2) + WHERE organization_id = $1 + AND (expires_at IS NULL OR expires_at > NOW()) + AND ((kind = 'email' AND email = LOWER($2)) + OR (kind = 'domain' AND email = split_part(LOWER($2), '@', 2))) + ORDER BY (kind = 'email') DESC + LIMIT 1 ` var s models.ContactSuppression err := r.DB.QueryRow(ctx, suppQuery, *orgID, detail.Email).Scan( - &s.Reason, &s.Source, &s.ExpiresAt, &s.CreatedAt, + &s.ID, &s.Kind, &s.Value, &s.Reason, &s.Source, &s.ExpiresAt, &s.CreatedAt, ) switch { case err == nil: @@ -2941,11 +2966,21 @@ func (r *contactRepository) ListSentEmails(ctx context.Context, userID, contactI }, nil } +// timelineKeyset is the predicate that pages one source of the contact +// timeline: rows whose (time, source rank, id) tuple sorts strictly before +// the cursor, which the query receives as three consecutive parameters +// starting at $first. Every source uses it so SQL and the merged sort agree. +func timelineKeyset(atCol string, source models.ContactTimelineSource, idCol string, first int) string { + return fmt.Sprintf("(%s, %d, %s) < ($%d::timestamptz, $%d::int, $%d::uuid)", + atCol, source, idCol, first, first+1, first+2) +} + // ListTimeline merges per-contact events from several source tables // into a single, reverse-chronological feed. // // Sources: // - campaign_contact_progress → sent / opened / clicked / replied / bounced +// - email_link_clicks, email_opens → per-event clicks and opens // - reply_intents → received replies (with intent classification) // - deliverability_events → bounce / complaint // - suppressed_recipients → suppression added @@ -2954,18 +2989,21 @@ func (r *contactRepository) ListSentEmails(ctx context.Context, userID, contactI // - contact_activities → creation and campaign / category membership // - website_page_hits → page views from the tracking snippet // -// We pull up to (limit) candidates from each source ordered by time -// DESC, then merge-sort in Go. This avoids a 5-way UNION with -// matching column lists (each source has a different shape), and the -// per-source limit caps the read at roughly 5*limit rows. +// We pull one row past the page from each source, newest first, then +// merge-sort in Go. This avoids a 10-way UNION with matching column lists +// (each source has a different shape), and the per-source limit caps the +// read at roughly 10*limit rows. The lookahead row is what makes has_more +// right when a single source fills the page on its own. // -// The `before` cursor is a wall-clock time; everything strictly older -// than it is eligible. The caller paginates by setting `before` to -// the oldest returned event's `At` on the next call. -func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, before *time.Time) (*models.ContactTimelineResult, *errx.Error) { +// The feed is ordered by (at, source, id) and a page resumes strictly after +// the cursor on that tuple, so two events at the same instant, from the +// same table or different ones, land on one side of a page boundary or the +// other and are never skipped or repeated. A nil cursor is the first page. +func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID, limit int, cursor *models.ContactTimelineKey) (*models.ContactTimelineResult, *errx.Error) { if limit <= 0 || limit > 200 { limit = 50 } + fetch := limit + 1 // We resolve the contact's email up front because some org-scoped // joins (suppression, deliverability fallback, reply_intents) key @@ -2982,27 +3020,42 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, return nil, errx.InternalError() } - // "before" defaults to "now + 1 minute" so the first page picks - // up everything. Using a future bound keeps the SQL uniform — every - // query passes the same predicate. - bound := time.Now().Add(time.Minute) - if before != nil { - bound = *before + // The position the page resumes after. The first page starts a minute + // in the future at the lowest rank, which admits every event the same + // way a real cursor would, so every query passes the same predicate. + after := models.ContactTimelineKey{At: time.Now().Add(time.Minute)} + if cursor != nil { + after = *cursor } + afterSource := int(after.Source) events := make([]models.ContactTimelineEvent, 0, limit*2) - // 1. Engagement events from campaign_contact_progress. One progress - // row can emit up to 5 events (sent/opened/clicked/replied/bounced). - progressQuery := ` + // 1. Engagement stamps from campaign_contact_progress, unnested to one + // row per stamp so the limit and the cursor apply to events, not to + // leads: a lead whose newest stamp is past the cursor must not push + // an older lead's eligible stamp off the page. The coarse opened and + // clicked stamps are emitted only when no logged open or click stands + // for them (a stamp written before per-event logging); otherwise + // sources 9 and 10 carry the event with its origin. A logged event + // represents the stamp when it landed within a minute of it, the + // stamp being written as the event is logged. + progressQuery := fmt.Sprintf(` SELECT - ccp.sent_at, ccp.opened_at, ccp.clicked_at, ccp.replied_at, ccp.bounced_at, + ev.source, ev.at, ccp.sequence_id, ccp.opened_machine, cam.id, cam.name, seq.id, seq.name, seq.subject, ea.id, ea.email, ea.name FROM campaign_contact_progress ccp JOIN campaigns cam ON cam.id = ccp.campaign_id JOIN sequences seq ON seq.id = ccp.sequence_id + CROSS JOIN LATERAL (VALUES + (%[1]d, ccp.sent_at), + (%[2]d, ccp.opened_at), + (%[3]d, ccp.clicked_at), + (%[4]d, ccp.replied_at), + (%[5]d, ccp.bounced_at) + ) AS ev(source, at) LEFT JOIN LATERAL ( SELECT ea.id, ea.email, ea.name FROM tasks t @@ -3016,27 +3069,41 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, ) ea ON TRUE WHERE ccp.contact_id = $1 AND cam.user_id = $2 - AND COALESCE(ccp.sent_at, ccp.opened_at, ccp.clicked_at, ccp.replied_at, ccp.bounced_at) < $3 - ORDER BY GREATEST( - COALESCE(ccp.sent_at, 'epoch'), - COALESCE(ccp.opened_at, 'epoch'), - COALESCE(ccp.clicked_at, 'epoch'), - COALESCE(ccp.replied_at, 'epoch'), - COALESCE(ccp.bounced_at, 'epoch') - ) DESC - LIMIT $4 - ` - prows, err := r.DB.Query(ctx, progressQuery, contactID, userID, bound, limit) + AND ev.at IS NOT NULL + AND (ev.at, ev.source, ccp.sequence_id) < ($3::timestamptz, $4::int, $5::uuid) + AND NOT (ev.source = %[2]d AND EXISTS ( + SELECT 1 FROM email_opens o + WHERE o.campaign_id = ccp.campaign_id AND o.contact_id = ccp.contact_id AND o.sequence_id = ccp.sequence_id + AND o.opened_at BETWEEN ccp.opened_at - INTERVAL '1 minute' AND ccp.opened_at + INTERVAL '1 minute' + )) + AND NOT (ev.source = %[3]d AND EXISTS ( + SELECT 1 FROM email_link_clicks lc + WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id + AND lc.clicked_at BETWEEN ccp.clicked_at - INTERVAL '1 minute' AND ccp.clicked_at + INTERVAL '1 minute' + )) + ORDER BY ev.at DESC, ev.source DESC, ccp.sequence_id DESC + LIMIT $6 + `, + models.TimelineSourceProgressSent, + models.TimelineSourceProgressOpened, + models.TimelineSourceProgressClicked, + models.TimelineSourceProgressReplied, + models.TimelineSourceProgressBounced, + ) + prows, err := r.DB.Query(ctx, progressQuery, contactID, userID, after.At, afterSource, after.ID, fetch) if err != nil { - db.CaptureError(err, progressQuery, []any{contactID, userID, bound, limit}, "ListTimeline progress") + db.CaptureError(err, progressQuery, []any{contactID, userID, after.At, afterSource, after.ID, fetch}, "ListTimeline progress") return nil, errx.InternalError() } for prows.Next() { - var sentAt, openedAt, clickedAt, repliedAt, bouncedAt *time.Time + var source int + var at time.Time + var seqKey uuid.UUID + var openedMachine bool var campID, seqID, eaID *uuid.UUID var campName, seqName, seqSubject, eaEmail, eaName *string if err := prows.Scan( - &sentAt, &openedAt, &clickedAt, &repliedAt, &bouncedAt, + &source, &at, &seqKey, &openedMachine, &campID, &campName, &seqID, &seqName, &seqSubject, &eaID, &eaEmail, &eaName, @@ -3045,92 +3112,274 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, db.CaptureError(err, "", nil, "ListTimeline progress scan") return nil, errx.InternalError() } - baseSubject := seqSubject - makeEvent := func(t *time.Time, ty models.ContactTimelineEventType) { - if t == nil || !t.Before(bound) { - return - } - ev := models.ContactTimelineEvent{ - Type: ty, - At: *t, - EmailAccountID: eaID, - EmailAccountEmail: eaEmail, - EmailAccountName: eaName, - CampaignID: campID, - CampaignName: campName, - SequenceID: seqID, - SequenceName: seqName, - } - if baseSubject != nil && *baseSubject != "" { - ev.Subject = baseSubject - } - events = append(events, ev) + ev := models.ContactTimelineEvent{ + At: at, + Key: models.ContactTimelineKey{At: at, Source: models.ContactTimelineSource(source), ID: seqKey}, + EmailAccountID: eaID, + EmailAccountEmail: eaEmail, + EmailAccountName: eaName, + CampaignID: campID, + CampaignName: campName, + SequenceID: seqID, + SequenceName: seqName, } - makeEvent(sentAt, models.TimelineEmailSent) - makeEvent(openedAt, models.TimelineEmailOpened) - makeEvent(clickedAt, models.TimelineEmailClicked) - makeEvent(repliedAt, models.TimelineEmailReplied) - makeEvent(bouncedAt, models.TimelineEmailBounced) + if seqSubject != nil && *seqSubject != "" { + ev.Subject = seqSubject + } + switch ev.Key.Source { + case models.TimelineSourceProgressSent: + ev.Type = models.TimelineEmailSent + case models.TimelineSourceProgressOpened: + ev.Type = models.TimelineEmailOpened + machine := openedMachine + ev.Machine = &machine + case models.TimelineSourceProgressClicked: + ev.Type = models.TimelineEmailClicked + case models.TimelineSourceProgressReplied: + ev.Type = models.TimelineEmailReplied + case models.TimelineSourceProgressBounced: + ev.Type = models.TimelineEmailBounced + default: + continue + } + events = append(events, ev) } prows.Close() + if err := prows.Err(); err != nil { + db.CaptureError(err, progressQuery, nil, "ListTimeline progress rows") + return nil, errx.InternalError() + } + + // 9. Per-link clicks: which link, where it went, and whether a person or + // a scanner clicked it. Same campaign scope as the progress feed. + clickQuery := ` + SELECT lc.id, lc.task_id, lc.clicked_at, lc.destination, lc.label, lc.user_agent, lc.machine, lc.machine_reason, + lc.client, lc.device_type, lc.os, lc.browser, lc.browser_version, lc.country_code, lc.region, lc.city, + cam.id, cam.name, + seq.id, seq.name, seq.subject, + ea.id, ea.email, ea.name + FROM email_link_clicks lc + JOIN campaigns cam ON cam.id = lc.campaign_id + JOIN sequences seq ON seq.id = lc.sequence_id + LEFT JOIN LATERAL ( + SELECT ea.id, ea.email, ea.name + FROM tasks t + JOIN email_accounts ea ON ea.id = t.email_account_id + WHERE t.id = lc.task_id + ) ea ON TRUE + WHERE lc.contact_id = $1 + AND cam.user_id = $2 + AND ` + timelineKeyset("lc.clicked_at", models.TimelineSourceLinkClick, "lc.id", 3) + ` + ORDER BY lc.clicked_at DESC, lc.id DESC + LIMIT $6 + ` + crows, err := r.DB.Query(ctx, clickQuery, contactID, userID, after.At, afterSource, after.ID, fetch) + if err != nil { + db.CaptureError(err, clickQuery, []any{contactID, userID, after.At, afterSource, after.ID, fetch}, "ListTimeline link clicks") + return nil, errx.InternalError() + } + for crows.Next() { + var link models.ContactLinkClick + var origin models.EngagementOrigin + var taskID uuid.UUID + var at time.Time + var machine bool + var reason string + var campID, seqID, eaID *uuid.UUID + var campName, seqName, seqSubject, eaEmail, eaName *string + if err := crows.Scan( + &link.ID, &taskID, &at, &link.URL, &link.Label, &link.UserAgent, &machine, &reason, + &origin.Client, &origin.DeviceType, &origin.OS, &origin.Browser, &origin.BrowserVersion, + &origin.CountryCode, &origin.Region, &origin.City, + &campID, &campName, + &seqID, &seqName, &seqSubject, + &eaID, &eaEmail, &eaName, + ); err != nil { + crows.Close() + db.CaptureError(err, "", nil, "ListTimeline link clicks scan") + return nil, errx.InternalError() + } + fillUTM(&link) + ev := models.ContactTimelineEvent{ + Type: models.TimelineEmailClicked, + At: at, + Key: models.ContactTimelineKey{At: at, Source: models.TimelineSourceLinkClick, ID: link.ID}, + EmailAccountID: eaID, + EmailAccountEmail: eaEmail, + EmailAccountName: eaName, + CampaignID: campID, + CampaignName: campName, + SequenceID: seqID, + SequenceName: seqName, + Machine: &machine, + Link: &link, + TaskID: &taskID, + } + if !origin.Empty() { + o := origin + ev.Origin = &o + } + if seqSubject != nil && *seqSubject != "" { + ev.Subject = seqSubject + } + if reason != "" { + r := reason + ev.MachineReason = &r + } + events = append(events, ev) + } + crows.Close() + if err := crows.Err(); err != nil { + db.CaptureError(err, clickQuery, nil, "ListTimeline link clicks rows") + return nil, errx.InternalError() + } + + // 10. Per-event opens: each one with what it came from, machine ones + // labelled. Same campaign scope as the progress feed. + openQuery := ` + SELECT o.id, o.task_id, o.opened_at, o.user_agent, o.machine, o.machine_reason, + o.client, o.device_type, o.os, o.browser, o.browser_version, o.country_code, o.region, o.city, + cam.id, cam.name, + seq.id, seq.name, seq.subject, + ea.id, ea.email, ea.name + FROM email_opens o + JOIN campaigns cam ON cam.id = o.campaign_id + JOIN sequences seq ON seq.id = o.sequence_id + LEFT JOIN LATERAL ( + SELECT ea.id, ea.email, ea.name + FROM tasks t + JOIN email_accounts ea ON ea.id = t.email_account_id + WHERE t.id = o.task_id + ) ea ON TRUE + WHERE o.contact_id = $1 + AND cam.user_id = $2 + AND ` + timelineKeyset("o.opened_at", models.TimelineSourceOpen, "o.id", 3) + ` + ORDER BY o.opened_at DESC, o.id DESC + LIMIT $6 + ` + orows, err := r.DB.Query(ctx, openQuery, contactID, userID, after.At, afterSource, after.ID, fetch) + if err != nil { + db.CaptureError(err, openQuery, []any{contactID, userID, after.At, afterSource, after.ID, fetch}, "ListTimeline opens") + return nil, errx.InternalError() + } + for orows.Next() { + var id, taskID uuid.UUID + var origin models.EngagementOrigin + var at time.Time + var machine bool + var reason, userAgent string + var campID, seqID, eaID *uuid.UUID + var campName, seqName, seqSubject, eaEmail, eaName *string + if err := orows.Scan( + &id, &taskID, &at, &userAgent, &machine, &reason, + &origin.Client, &origin.DeviceType, &origin.OS, &origin.Browser, &origin.BrowserVersion, + &origin.CountryCode, &origin.Region, &origin.City, + &campID, &campName, + &seqID, &seqName, &seqSubject, + &eaID, &eaEmail, &eaName, + ); err != nil { + orows.Close() + db.CaptureError(err, "", nil, "ListTimeline opens scan") + return nil, errx.InternalError() + } + ev := models.ContactTimelineEvent{ + Type: models.TimelineEmailOpened, + At: at, + Key: models.ContactTimelineKey{At: at, Source: models.TimelineSourceOpen, ID: id}, + EmailAccountID: eaID, + EmailAccountEmail: eaEmail, + EmailAccountName: eaName, + CampaignID: campID, + CampaignName: campName, + SequenceID: seqID, + SequenceName: seqName, + Machine: &machine, + TaskID: &taskID, + } + if !origin.Empty() { + o := origin + ev.Origin = &o + } + if seqSubject != nil && *seqSubject != "" { + ev.Subject = seqSubject + } + if reason != "" { + r := reason + ev.MachineReason = &r + } + events = append(events, ev) + } + orows.Close() + if err := orows.Err(); err != nil { + db.CaptureError(err, openQuery, nil, "ListTimeline opens rows") + return nil, errx.InternalError() + } if orgID != nil { // 2. Reply intents (inbound replies with classification). replyQuery := ` - SELECT ri.created_at, ri.intent, ri.campaign_id, cam.name, ri.task_id + SELECT ri.id, ri.created_at, ri.intent, ri.campaign_id, cam.name, ri.task_id FROM reply_intents ri LEFT JOIN campaigns cam ON cam.id = ri.campaign_id WHERE ri.organization_id = $1 AND LOWER(ri.contact_email) = LOWER($2) - AND ri.created_at < $3 - ORDER BY ri.created_at DESC - LIMIT $4 + AND ` + timelineKeyset("ri.created_at", models.TimelineSourceReplyIntent, "ri.id", 3) + ` + ORDER BY ri.created_at DESC, ri.id DESC + LIMIT $6 ` - rrows, err := r.DB.Query(ctx, replyQuery, *orgID, contactEmail, bound, limit) + rrows, err := r.DB.Query(ctx, replyQuery, *orgID, contactEmail, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, replyQuery, nil, "ListTimeline replies") return nil, errx.InternalError() } for rrows.Next() { var ev models.ContactTimelineEvent + var id uuid.UUID var intent string - if err := rrows.Scan(&ev.At, &intent, &ev.CampaignID, &ev.CampaignName, &ev.TaskID); err != nil { + if err := rrows.Scan(&id, &ev.At, &intent, &ev.CampaignID, &ev.CampaignName, &ev.TaskID); err != nil { rrows.Close() db.CaptureError(err, "", nil, "ListTimeline replies scan") return nil, errx.InternalError() } ev.Type = models.TimelineReplyReceived + ev.Key = models.ContactTimelineKey{At: ev.At, Source: models.TimelineSourceReplyIntent, ID: id} ev.Intent = &intent events = append(events, ev) } rrows.Close() + if err := rrows.Err(); err != nil { + db.CaptureError(err, replyQuery, nil, "ListTimeline replies rows") + return nil, errx.InternalError() + } // 3. Deliverability events (bounce / complaint / unsubscribe). delivQuery := ` - SELECT de.created_at, de.event_type, de.provider, de.reason, + SELECT de.id, de.created_at, de.event_type, de.provider, de.reason, de.campaign_id, cam.name, de.task_id FROM deliverability_events de LEFT JOIN campaigns cam ON cam.id = de.campaign_id WHERE de.organization_id = $1 AND (de.contact_id = $2 OR LOWER(de.recipient_email) = LOWER($3)) - AND de.created_at < $4 - ORDER BY de.created_at DESC - LIMIT $5 + AND ` + timelineKeyset("de.created_at", models.TimelineSourceDeliverability, "de.id", 4) + ` + ORDER BY de.created_at DESC, de.id DESC + LIMIT $7 ` - drows, err := r.DB.Query(ctx, delivQuery, *orgID, contactID, contactEmail, bound, limit) + drows, err := r.DB.Query(ctx, delivQuery, *orgID, contactID, contactEmail, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, delivQuery, nil, "ListTimeline deliv") return nil, errx.InternalError() } for drows.Next() { var ev models.ContactTimelineEvent + var id uuid.UUID var eventType, provider, reason string - if err := drows.Scan(&ev.At, &eventType, &provider, &reason, &ev.CampaignID, &ev.CampaignName, &ev.TaskID); err != nil { + if err := drows.Scan(&id, &ev.At, &eventType, &provider, &reason, &ev.CampaignID, &ev.CampaignName, &ev.TaskID); err != nil { drows.Close() db.CaptureError(err, "", nil, "ListTimeline deliv scan") return nil, errx.InternalError() } ev.Type = models.TimelineDeliverability + ev.Key = models.ContactTimelineKey{At: ev.At, Source: models.TimelineSourceDeliverability, ID: id} ev.Source = &eventType ev.Provider = &provider if reason != "" { @@ -3139,92 +3388,118 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, events = append(events, ev) } drows.Close() + if err := drows.Err(); err != nil { + db.CaptureError(err, delivQuery, nil, "ListTimeline deliv rows") + return nil, errx.InternalError() + } - // 4. Suppression — emit one event at create time. We treat - // later updates as the same event for now. + // 4. Suppression: one event per matching entry (the address itself + // and its domain), at create time. Later updates are the same event. suppQuery := ` - SELECT created_at, reason, source + SELECT id, created_at, reason, source FROM suppressed_recipients WHERE organization_id = $1 - AND LOWER(email) = LOWER($2) - AND created_at < $3 - ORDER BY created_at DESC - LIMIT 1 + AND ((kind = 'email' AND email = LOWER($2)) + OR (kind = 'domain' AND email = split_part(LOWER($2), '@', 2))) + AND ` + timelineKeyset("created_at", models.TimelineSourceSuppression, "id", 3) + ` + ORDER BY created_at DESC, id DESC + LIMIT $6 ` - var sAt time.Time - var sReason, sSource string - if err := r.DB.QueryRow(ctx, suppQuery, *orgID, contactEmail, bound).Scan(&sAt, &sReason, &sSource); err == nil { + srows, err := r.DB.Query(ctx, suppQuery, *orgID, contactEmail, after.At, afterSource, after.ID, fetch) + if err != nil { + db.CaptureError(err, suppQuery, nil, "ListTimeline suppression") + return nil, errx.InternalError() + } + for srows.Next() { + var id uuid.UUID + var sAt time.Time + var sReason, sSource string + if err := srows.Scan(&id, &sAt, &sReason, &sSource); err != nil { + srows.Close() + db.CaptureError(err, "", nil, "ListTimeline suppression scan") + return nil, errx.InternalError() + } ev := models.ContactTimelineEvent{ Type: models.TimelineSuppressed, At: sAt, + Key: models.ContactTimelineKey{At: sAt, Source: models.TimelineSourceSuppression, ID: id}, Source: &sSource, } if sReason != "" { ev.Reason = &sReason } events = append(events, ev) - } else if err != pgx.ErrNoRows { - db.CaptureError(err, suppQuery, nil, "ListTimeline suppression") + } + srows.Close() + if err := srows.Err(); err != nil { + db.CaptureError(err, suppQuery, nil, "ListTimeline suppression rows") return nil, errx.InternalError() } // 5. Notes. notesQuery := ` - SELECT created_at, user_id, content + SELECT id, created_at, user_id, content FROM contact_notes WHERE contact_id = $1 AND organization_id = $2 - AND created_at < $3 - ORDER BY created_at DESC - LIMIT $4 + AND ` + timelineKeyset("created_at", models.TimelineSourceNote, "id", 3) + ` + ORDER BY created_at DESC, id DESC + LIMIT $6 ` - nrows, err := r.DB.Query(ctx, notesQuery, contactID, *orgID, bound, limit) + nrows, err := r.DB.Query(ctx, notesQuery, contactID, *orgID, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, notesQuery, nil, "ListTimeline notes") return nil, errx.InternalError() } for nrows.Next() { var ev models.ContactTimelineEvent - var uid uuid.UUID + var id, uid uuid.UUID var content string - if err := nrows.Scan(&ev.At, &uid, &content); err != nil { + if err := nrows.Scan(&id, &ev.At, &uid, &content); err != nil { nrows.Close() db.CaptureError(err, "", nil, "ListTimeline notes scan") return nil, errx.InternalError() } ev.Type = models.TimelineNote + ev.Key = models.ContactTimelineKey{At: ev.At, Source: models.TimelineSourceNote, ID: id} ev.UserID = &uid ev.Content = &content events = append(events, ev) } nrows.Close() + if err := nrows.Err(); err != nil { + db.CaptureError(err, notesQuery, nil, "ListTimeline notes rows") + return nil, errx.InternalError() + } // 6. Meetings booked through a connected scheduling provider. The event // time is when the booking arrived; scheduled_for carries the call // window so the UI can render "Meeting on ". meetingQuery := ` - SELECT created_at, status, source, event_name, scheduled_for, join_url, canceled_reason + SELECT id, created_at, status, source, event_name, scheduled_for, join_url, canceled_reason FROM meeting_bookings WHERE contact_id = $1 AND organization_id = $2 - AND created_at < $3 - ORDER BY created_at DESC - LIMIT $4 + AND ` + timelineKeyset("created_at", models.TimelineSourceMeeting, "id", 3) + ` + ORDER BY created_at DESC, id DESC + LIMIT $6 ` - mrows, err := r.DB.Query(ctx, meetingQuery, contactID, *orgID, bound, limit) + mrows, err := r.DB.Query(ctx, meetingQuery, contactID, *orgID, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, meetingQuery, nil, "ListTimeline meetings") return nil, errx.InternalError() } for mrows.Next() { var ev models.ContactTimelineEvent + var id uuid.UUID var status, source, eventName, joinURL, canceledReason string var scheduledFor *time.Time - if err := mrows.Scan(&ev.At, &status, &source, &eventName, &scheduledFor, &joinURL, &canceledReason); err != nil { + if err := mrows.Scan(&id, &ev.At, &status, &source, &eventName, &scheduledFor, &joinURL, &canceledReason); err != nil { mrows.Close() db.CaptureError(err, "", nil, "ListTimeline meetings scan") return nil, errx.InternalError() } + ev.Key = models.ContactTimelineKey{At: ev.At, Source: models.TimelineSourceMeeting, ID: id} switch status { case "rescheduled": ev.Type = models.TimelineMeetingRescheduled @@ -3251,36 +3526,42 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, events = append(events, ev) } mrows.Close() + if err := mrows.Err(); err != nil { + db.CaptureError(err, meetingQuery, nil, "ListTimeline meetings rows") + return nil, errx.InternalError() + } // 7. Lifecycle: creation (with its first-touch source) and campaign / // category membership changes, from contact_activities. Names were // resolved when the row was written, so a renamed or deleted // campaign still reads correctly. lifeQuery := ` - SELECT created_at, user_id, activity_type, metadata + SELECT id, created_at, user_id, activity_type, metadata FROM contact_activities WHERE contact_id = $1 AND organization_id = $2 AND activity_type IN ('contact_created', 'campaign_added', 'campaign_removed', 'category_added', 'category_removed', 'form_submitted') - AND created_at < $3 - ORDER BY created_at DESC - LIMIT $4 + AND ` + timelineKeyset("created_at", models.TimelineSourceActivity, "id", 3) + ` + ORDER BY created_at DESC, id DESC + LIMIT $6 ` - lrows, err := r.DB.Query(ctx, lifeQuery, contactID, *orgID, bound, limit) + lrows, err := r.DB.Query(ctx, lifeQuery, contactID, *orgID, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, lifeQuery, nil, "ListTimeline lifecycle") return nil, errx.InternalError() } for lrows.Next() { var ev models.ContactTimelineEvent + var rowID uuid.UUID var typ string var meta map[string]any - if err := lrows.Scan(&ev.At, &ev.UserID, &typ, &meta); err != nil { + if err := lrows.Scan(&rowID, &ev.At, &ev.UserID, &typ, &meta); err != nil { lrows.Close() db.CaptureError(err, "", nil, "ListTimeline lifecycle scan") return nil, errx.InternalError() } ev.Type = models.ContactTimelineEventType(typ) + ev.Key = models.ContactTimelineKey{At: ev.At, Source: models.TimelineSourceActivity, ID: rowID} str := func(k string) *string { if v, ok := meta[k].(string); ok && v != "" { return &v @@ -3312,6 +3593,10 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, events = append(events, ev) } lrows.Close() + if err := lrows.Err(); err != nil { + db.CaptureError(err, lifeQuery, nil, "ListTimeline lifecycle rows") + return nil, errx.InternalError() + } // 8. Website page views from any browser tied to the contact through // an email-link ticket. @@ -3325,11 +3610,11 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, FROM website_page_hits h WHERE h.organization_id = $1 AND h.visitor_id IN (SELECT id FROM website_visitors WHERE contact_id = $2) - AND h.occurred_at < $3 - ORDER BY h.occurred_at DESC - LIMIT $4 + AND ` + timelineKeyset("h.occurred_at", models.TimelineSourcePageHit, "h.id", 3) + ` + ORDER BY h.occurred_at DESC, h.id DESC + LIMIT $6 ` - hrows, err := r.DB.Query(ctx, hitQuery, *orgID, contactID, bound, limit) + hrows, err := r.DB.Query(ctx, hitQuery, *orgID, contactID, after.At, afterSource, after.ID, fetch) if err != nil { db.CaptureError(err, hitQuery, nil, "ListTimeline page hits") return nil, errx.InternalError() @@ -3349,7 +3634,12 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, return nil, errx.InternalError() } hit := h - ev := models.ContactTimelineEvent{Type: models.TimelinePageHit, At: h.OccurredAt, PageHit: &hit} + ev := models.ContactTimelineEvent{ + Type: models.TimelinePageHit, + At: h.OccurredAt, + Key: models.ContactTimelineKey{At: h.OccurredAt, Source: models.TimelineSourcePageHit, ID: h.ID}, + PageHit: &hit, + } subject := h.Title if subject == "" { subject = h.Path @@ -3358,21 +3648,42 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, events = append(events, ev) } hrows.Close() + if err := hrows.Err(); err != nil { + db.CaptureError(err, hitQuery, nil, "ListTimeline page hits rows") + return nil, errx.InternalError() + } } - // Merge sort: newest first. - sort.Slice(events, func(i, j int) bool { return events[i].At.After(events[j].At) }) + // Merge sort: newest first, ties broken exactly as each source query + // broke them, so the page boundary is the same position everywhere. + sort.Slice(events, func(i, j int) bool { return events[j].Key.Before(events[i].Key) }) - hasMore := false + res := &models.ContactTimelineResult{Data: events} if len(events) > limit { - hasMore = true - events = events[:limit] + res.Data = events[:limit] + last := res.Data[limit-1].Key + res.HasMore = true + res.Pagination = models.Pagination{ + NextCursor: paging.EncodeMerged(last.At, int(last.Source), last.ID), + HasMore: true, + } } + return res, nil +} - return &models.ContactTimelineResult{ - Data: events, - HasMore: hasMore, - }, nil +// fillUTM reads the UTM parameters off a clicked link's destination, whether +// the send path stamped them or the author wrote them by hand. +func fillUTM(link *models.ContactLinkClick) { + u, err := url.Parse(link.URL) + if err != nil { + return + } + q := u.Query() + link.UTMSource = q.Get("utm_source") + link.UTMMedium = q.Get("utm_medium") + link.UTMCampaign = q.Get("utm_campaign") + link.UTMTerm = q.Get("utm_term") + link.UTMContent = q.Get("utm_content") } // verificationFromRequest normalises a verdict a caller supplied with a diff --git a/internal/repository/pg_email.go b/internal/repository/pg_email.go index eb82a608..b8b6fd4f 100644 --- a/internal/repository/pg_email.go +++ b/internal/repository/pg_email.go @@ -305,6 +305,29 @@ func (r *emailRepository) CountForOrganization(ctx context.Context, orgID uuid.U return count, nil } +// reserveMailboxSlotTx is the final allowance check, run inside the insert +// transaction under a per-organization lock so two connects that both saw one +// slot free cannot both take it. The service's earlier read is for feedback; +// this is what enforces. +func reserveMailboxSlotTx(ctx context.Context, tx pgx.Tx, orgID *uuid.UUID, a *models.MailboxAllowance) *errx.Error { + if a == nil || a.Allowance == nil || orgID == nil { + return nil + } + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext('email_accounts'), hashtext($1::text))`, orgID.String()); err != nil { + db.CaptureError(err, "", nil, "exec") + return errx.InternalError() + } + var count int + if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM email_accounts WHERE organization_id = $1`, *orgID).Scan(&count); err != nil { + db.CaptureError(err, "", nil, "queryrow") + return errx.InternalError() + } + if count >= *a.Allowance { + return errx.MailboxAllowanceReached(count, *a.Allowance, a.Paid) + } + return nil +} + func (r *emailRepository) NewOauthAccount(ctx context.Context, userID string, data models.NewOauthAccount) (*models.Email, *errx.Error) { if data.Provider == models.InboxProviderSMTPIMAP { sentry.CaptureException(errors.New("invalid inbox provider")) @@ -331,6 +354,10 @@ func (r *emailRepository) NewOauthAccount(ctx context.Context, userID string, da } defer tx.Rollback(ctx) + if xerr := reserveMailboxSlotTx(ctx, tx, data.OrganizationID, data.Allowance); xerr != nil { + return nil, xerr + } + sigplain := utils.GetSignaturePlain(data.Name) sightml := utils.GetSignatureHTML(data.Name) @@ -462,6 +489,10 @@ func (r *emailRepository) NewSMTPIMAPAccount(ctx context.Context, userID string, } defer tx.Rollback(ctx) + if xerr := reserveMailboxSlotTx(ctx, tx, data.OrganizationID, data.Allowance); xerr != nil { + return nil, xerr + } + sigplain := utils.GetSignaturePlain(data.Name) sightml := utils.GetSignatureHTML(data.Name) diff --git a/internal/repository/pg_email_opens.go b/internal/repository/pg_email_opens.go new file mode 100644 index 00000000..efe98f89 --- /dev/null +++ b/internal/repository/pg_email_opens.go @@ -0,0 +1,99 @@ +package repository + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/models" +) + +// EmailOpen is one recorded open of one step: when, from what client and +// where. The progress row keeps only the first open per step; this keeps +// them all, machine ones included and labelled, for the contact's timeline +// and the campaign's audience breakdown. +type EmailOpen struct { + ID uuid.UUID + TaskID uuid.UUID + CampaignID uuid.UUID + ContactID uuid.UUID + SequenceID uuid.UUID + OpenedAt time.Time + Machine bool + MachineReason string + UserAgent string + IPHash string + Origin models.EngagementOrigin +} + +// Machine-open reasons stored in email_opens.machine_reason. +const ( + EmailOpenReasonPrefetch = "prefetch" // a mail client prefetch or a fetch with no browser + EmailOpenReasonInstant = "instant" // arrived inside the machine window after dispatch +) + +// EmailOpenRepository is the per-event open log. Only the tracking consumer +// writes it. +type EmailOpenRepository interface { + Insert(ctx context.Context, open *EmailOpen) error + // HasHumanOpen reports whether a person's own open is on record for the + // step, which decides whether an open a click implied can be withdrawn. + HasHumanOpen(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error) + // Cleanup deletes opens older than the retention window. + Cleanup(ctx context.Context, olderThanDays int) (int64, error) +} + +type emailOpenRepository struct { + db *pgxpool.Pool +} + +// NewEmailOpenRepository creates a new email open repository. +func NewEmailOpenRepository(db *pgxpool.Pool) EmailOpenRepository { + return &emailOpenRepository{db: db} +} + +func (r *emailOpenRepository) Insert(ctx context.Context, o *EmailOpen) error { + if o.ID == uuid.Nil { + o.ID = uuid.New() + } + if o.OpenedAt.IsZero() { + o.OpenedAt = time.Now() + } + query := ` + INSERT INTO email_opens + (id, task_id, campaign_id, contact_id, sequence_id, opened_at, + machine, machine_reason, user_agent, ip_hash, + client, device_type, os, browser, browser_version, country_code, region, city) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + ` + _, err := r.db.Exec(ctx, query, + o.ID, o.TaskID, o.CampaignID, o.ContactID, o.SequenceID, o.OpenedAt, + o.Machine, o.MachineReason, o.UserAgent, o.IPHash, + o.Origin.Client, o.Origin.DeviceType, o.Origin.OS, o.Origin.Browser, o.Origin.BrowserVersion, + o.Origin.CountryCode, o.Origin.Region, o.Origin.City, + ) + return err +} + +func (r *emailOpenRepository) HasHumanOpen(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error) { + query := ` + SELECT EXISTS ( + SELECT 1 FROM email_opens + WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 AND machine = false + ) + ` + var ok bool + err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID).Scan(&ok) + return ok, err +} + +func (r *emailOpenRepository) Cleanup(ctx context.Context, olderThanDays int) (int64, error) { + tag, err := r.db.Exec(ctx, + `DELETE FROM email_opens WHERE opened_at < NOW() - $1 * INTERVAL '1 day'`, + olderThanDays) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} diff --git a/internal/repository/pg_link_clicks.go b/internal/repository/pg_link_clicks.go new file mode 100644 index 00000000..30c315c0 --- /dev/null +++ b/internal/repository/pg_link_clicks.go @@ -0,0 +1,262 @@ +package repository + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/models" +) + +// LinkClick is one recorded click on one tracked link. Machine clicks (a +// security gateway walking every link at delivery time) are kept for the +// record with the reason they were flagged, but never count as engagement. +type LinkClick struct { + ID uuid.UUID + TrackedLinkID *uuid.UUID + TaskID uuid.UUID + CampaignID uuid.UUID + ContactID uuid.UUID + SequenceID uuid.UUID + Destination string + Label string + UserAgent string + IPHash string + Machine bool + MachineReason string + ClickedAt time.Time + // Origin is what the click said about where it came from. + Origin models.EngagementOrigin + // AnnouncePending marks a person's click whose effects wait for the + // burst window; the row is the durable record of that work. + AnnouncePending bool +} + +// Machine-click reasons stored in email_link_clicks.machine_reason. +const ( + LinkClickReasonPrefetch = "prefetch" // no user agent: never a person's browser + LinkClickReasonInstant = "instant" // arrived inside the machine window after dispatch + LinkClickReasonBurst = "burst" // a second link of the same email from the same source within seconds +) + +// LinkClickRepository is the per-link click log behind the contact timeline +// and the scanner heuristics. Only the tracking consumer writes it. +type LinkClickRepository interface { + Insert(ctx context.Context, click *LinkClick) error + // CountRecentOtherLinks counts clicks from the same source on OTHER links + // of the same email since the given time: the burst signal. A link is + // identified by its ticket when known, else by destination (events from + // an older tracking build). + CountRecentOtherLinks(ctx context.Context, taskID uuid.UUID, ipHash string, linkID *uuid.UUID, destination string, since time.Time) (int, error) + // MarkBurst flags the source's earlier human-labelled clicks on the email + // since the given time as machine, once a burst is recognised. + MarkBurst(ctx context.Context, taskID uuid.UUID, ipHash string, since time.Time) (int64, error) + // HasHumanClick reports whether any click on the step is still labelled + // as a person's. + HasHumanClick(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error) + // IsMachine reads a logged click's current classification, which a burst + // recognised after the fact may have changed. + IsMachine(ctx context.Context, id uuid.UUID) (bool, error) + // HasHumanClickOn reports whether a person already clicked this exact + // link of the email, so a repeat is not logged twice. The ticket is the + // identity when known (two links may share a destination); the + // destination is the fallback for events from an older tracking build. + HasHumanClickOn(ctx context.Context, taskID uuid.UUID, linkID *uuid.UUID, destination string) (bool, error) + // ClaimAnnounce leases a pending click's announcement for one attempt + // and reports the click's classification at that moment. claimed is + // false when another attempt holds a live lease or the announcement is + // done. A lease that expires without CompleteAnnounce is offered again. + ClaimAnnounce(ctx context.Context, id uuid.UUID) (claimed bool, machine bool, err error) + // CompleteAnnounce records that the click's effects ran, so neither the + // timer nor the sweep offers it again. + CompleteAnnounce(ctx context.Context, id uuid.UUID) error + // ListPendingAnnouncements returns clicks whose announcement is still + // pending, not under a live lease, and whose burst window closed before + // `before`: what a consumer restart or a failed attempt left behind. + ListPendingAnnouncements(ctx context.Context, before time.Time, limit int) ([]LinkClick, error) + // Cleanup deletes clicks older than the retention window. + Cleanup(ctx context.Context, olderThanDays int) (int64, error) +} + +type linkClickRepository struct { + db *pgxpool.Pool +} + +// NewLinkClickRepository creates a new link click repository. +func NewLinkClickRepository(db *pgxpool.Pool) LinkClickRepository { + return &linkClickRepository{db: db} +} + +func (r *linkClickRepository) Insert(ctx context.Context, c *LinkClick) error { + if c.ID == uuid.Nil { + c.ID = uuid.New() + } + if c.ClickedAt.IsZero() { + c.ClickedAt = time.Now() + } + query := ` + INSERT INTO email_link_clicks + (id, tracked_link_id, task_id, campaign_id, contact_id, sequence_id, + destination, label, user_agent, ip_hash, machine, machine_reason, clicked_at, + client, device_type, os, browser, browser_version, country_code, region, city, + announce_pending) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, + $14, $15, $16, $17, $18, $19, $20, $21, $22) + ` + _, err := r.db.Exec(ctx, query, + c.ID, c.TrackedLinkID, c.TaskID, c.CampaignID, c.ContactID, c.SequenceID, + c.Destination, c.Label, c.UserAgent, c.IPHash, c.Machine, c.MachineReason, c.ClickedAt, + c.Origin.Client, c.Origin.DeviceType, c.Origin.OS, c.Origin.Browser, c.Origin.BrowserVersion, + c.Origin.CountryCode, c.Origin.Region, c.Origin.City, + c.AnnouncePending, + ) + return err +} + +// announceLease is how long one attempt at a click's effects may take before +// the sweep is allowed to try again. +const announceLease = "2 minutes" + +func (r *linkClickRepository) ClaimAnnounce(ctx context.Context, id uuid.UUID) (bool, bool, error) { + query := ` + UPDATE email_link_clicks + SET announce_claimed_at = NOW() + WHERE id = $1 AND announce_pending + AND (announce_claimed_at IS NULL OR announce_claimed_at < NOW() - INTERVAL '` + announceLease + `') + RETURNING machine + ` + var machine bool + err := r.db.QueryRow(ctx, query, id).Scan(&machine) + if errors.Is(err, pgx.ErrNoRows) { + return false, false, nil + } + if err != nil { + return false, false, err + } + return true, machine, nil +} + +func (r *linkClickRepository) CompleteAnnounce(ctx context.Context, id uuid.UUID) error { + _, err := r.db.Exec(ctx, `UPDATE email_link_clicks SET announce_pending = false WHERE id = $1`, id) + return err +} + +func (r *linkClickRepository) ListPendingAnnouncements(ctx context.Context, before time.Time, limit int) ([]LinkClick, error) { + query := ` + SELECT id, tracked_link_id, task_id, campaign_id, contact_id, sequence_id, + destination, label, machine, machine_reason, clicked_at, + client, device_type, os, browser, browser_version, country_code, region, city + FROM email_link_clicks + WHERE announce_pending AND clicked_at < $1 + AND (announce_claimed_at IS NULL OR announce_claimed_at < NOW() - INTERVAL '` + announceLease + `') + ORDER BY clicked_at ASC + LIMIT $2 + ` + rows, err := r.db.Query(ctx, query, before, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []LinkClick + for rows.Next() { + var c LinkClick + if err := rows.Scan(&c.ID, &c.TrackedLinkID, &c.TaskID, &c.CampaignID, &c.ContactID, &c.SequenceID, + &c.Destination, &c.Label, &c.Machine, &c.MachineReason, &c.ClickedAt, + &c.Origin.Client, &c.Origin.DeviceType, &c.Origin.OS, &c.Origin.Browser, &c.Origin.BrowserVersion, + &c.Origin.CountryCode, &c.Origin.Region, &c.Origin.City); err != nil { + return nil, err + } + c.AnnouncePending = true + out = append(out, c) + } + return out, rows.Err() +} + +func (r *linkClickRepository) Cleanup(ctx context.Context, olderThanDays int) (int64, error) { + tag, err := r.db.Exec(ctx, + `DELETE FROM email_link_clicks WHERE clicked_at < NOW() - $1 * INTERVAL '1 day'`, + olderThanDays) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (r *linkClickRepository) CountRecentOtherLinks(ctx context.Context, taskID uuid.UUID, ipHash string, linkID *uuid.UUID, destination string, since time.Time) (int, error) { + // A row is "another link" when both sides have a ticket and they differ; + // a row without a ticket (older tracking build) is compared by + // destination so a repeat click on one link never reads as a burst. + query := ` + SELECT COUNT(*) + FROM email_link_clicks + WHERE task_id = $1 + AND ip_hash = $2 + AND clicked_at >= $3 + AND CASE + WHEN tracked_link_id IS NOT NULL AND $4::uuid IS NOT NULL THEN tracked_link_id <> $4::uuid + ELSE destination <> $5 + END + ` + var n int + err := r.db.QueryRow(ctx, query, taskID, ipHash, since, linkID, destination).Scan(&n) + return n, err +} + +func (r *linkClickRepository) MarkBurst(ctx context.Context, taskID uuid.UUID, ipHash string, since time.Time) (int64, error) { + query := ` + UPDATE email_link_clicks + SET machine = true, machine_reason = $4, announce_pending = false + WHERE task_id = $1 + AND ip_hash = $2 + AND clicked_at >= $3 + AND machine = false + ` + tag, err := r.db.Exec(ctx, query, taskID, ipHash, since, LinkClickReasonBurst) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (r *linkClickRepository) HasHumanClick(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error) { + query := ` + SELECT EXISTS ( + SELECT 1 FROM email_link_clicks + WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 AND machine = false + ) + ` + var ok bool + err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID).Scan(&ok) + return ok, err +} + +func (r *linkClickRepository) IsMachine(ctx context.Context, id uuid.UUID) (bool, error) { + var machine bool + err := r.db.QueryRow(ctx, `SELECT machine FROM email_link_clicks WHERE id = $1`, id).Scan(&machine) + return machine, err +} + +func (r *linkClickRepository) HasHumanClickOn(ctx context.Context, taskID uuid.UUID, linkID *uuid.UUID, destination string) (bool, error) { + query := ` + SELECT EXISTS ( + SELECT 1 FROM email_link_clicks + WHERE task_id = $1 AND destination = $2 AND machine = false + ) + ` + args := []any{taskID, destination} + if linkID != nil { + query = ` + SELECT EXISTS ( + SELECT 1 FROM email_link_clicks + WHERE task_id = $1 AND tracked_link_id = $2 AND machine = false + ) + ` + args = []any{taskID, *linkID} + } + var ok bool + err := r.db.QueryRow(ctx, query, args...).Scan(&ok) + return ok, err +} diff --git a/internal/repository/pg_organization.go b/internal/repository/pg_organization.go index 28a0c057..e355562a 100644 --- a/internal/repository/pg_organization.go +++ b/internal/repository/pg_organization.go @@ -641,6 +641,7 @@ func (r *organizationRepository) GetEmailsSentTodayCount(ctx context.Context, or AND t.task_type = 'campaign' AND t.status = 'completed' AND t.completed_at >= CURRENT_DATE + AND `+taskDispatchedEmail+` `, orgID).Scan(&count) return count, err } diff --git a/internal/repository/pg_segment.go b/internal/repository/pg_segment.go index ba3db5b3..214e819e 100644 --- a/internal/repository/pg_segment.go +++ b/internal/repository/pg_segment.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -34,6 +35,10 @@ type SegmentRepository interface { ListForCampaign(ctx context.Context, orgID, campaignID uuid.UUID) ([]models.CampaignSegmentLink, *errx.Error) // SetForCampaign replaces the campaign's linked segments. SetForCampaign(ctx context.Context, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) *errx.Error + // ReplaceForCampaign replaces the links and enrols the members in one + // transaction, so a failed enrolment leaves no half-applied link set. + // Returns how many leads were new and the campaign's status. + ReplaceForCampaign(ctx context.Context, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) (int, string, *errx.Error) // SyncCampaignSegments enrols every current member of the campaign's // linked segments that is not yet a lead; returns how many were added. SyncCampaignSegments(ctx context.Context, orgID, campaignID uuid.UUID) (int, *errx.Error) @@ -273,14 +278,14 @@ func (r *segmentRepository) AddToCampaign(ctx context.Context, orgID uuid.UUID, } defer tx.Rollback(ctx) - var exists bool - if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM campaigns WHERE id = $1 AND organization_id = $2)`, campaignID, orgID).Scan(&exists); err != nil { - db.CaptureError(err, "campaign exists", nil, "queryrow") + var status string + if err := tx.QueryRow(ctx, `SELECT status FROM campaigns WHERE id = $1 AND organization_id = $2`, campaignID, orgID).Scan(&status); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, errx.New(errx.NotFound, "campaign not found") + } + db.CaptureError(err, "campaign status", nil, "queryrow") return nil, errx.InternalError() } - if !exists { - return nil, errx.New(errx.NotFound, "campaign not found") - } args := []any{orgID} clause, args, err := compileSavedSegment(ctx, tx, orgID, segmentID, args) @@ -308,7 +313,7 @@ func (r *segmentRepository) AddToCampaign(ctx context.Context, orgID uuid.UUID, db.CaptureError(err, "", nil, "commit") return nil, errx.InternalError() } - return &models.SegmentAddToCampaignResult{CampaignID: campaignID, Added: len(links), Members: members}, nil + return &models.SegmentAddToCampaignResult{CampaignID: campaignID, Added: len(links), Members: members, Status: status}, nil } // insertSegmentLeads enrols every contact matching the precompiled segment @@ -358,7 +363,7 @@ func (r *segmentRepository) ListForCampaign(ctx context.Context, orgID, campaign return nil, errx.New(errx.NotFound, "campaign not found") } rows, err := r.DB.Query(ctx, ` - SELECT s.id, s.name, s.color, s.description, s.match, s.conditions, cs.created_at + SELECT s.id, s.name, s.color, s.description, cs.created_at FROM campaign_segments cs JOIN segments s ON s.id = cs.segment_id WHERE cs.campaign_id = $1 @@ -369,27 +374,13 @@ func (r *segmentRepository) ListForCampaign(ctx context.Context, orgID, campaign } defer rows.Close() out := []models.CampaignSegmentLink{} - // Held alongside so the live counts below evaluate the same definitions. - var matches []models.SegmentMatch - var conds [][]models.SegmentCondition for rows.Next() { var l models.CampaignSegmentLink - var match string - var raw []byte - if err := rows.Scan(&l.SegmentID, &l.Name, &l.Color, &l.Description, &match, &raw, &l.LinkedAt); err != nil { + if err := rows.Scan(&l.SegmentID, &l.Name, &l.Color, &l.Description, &l.LinkedAt); err != nil { db.CaptureError(err, "", nil, "scan") return nil, errx.InternalError() } - cs := []models.SegmentCondition{} - if len(raw) > 0 { - if err := json.Unmarshal(raw, &cs); err != nil { - db.CaptureError(err, "", nil, "scan") - return nil, errx.InternalError() - } - } out = append(out, l) - matches = append(matches, models.SegmentMatch(match)) - conds = append(conds, cs) } // A mid-stream read failure ends Next() early with no scan error; without // this the Leads tab would render a truncated link list as the truth. @@ -397,55 +388,72 @@ func (r *segmentRepository) ListForCampaign(ctx context.Context, orgID, campaign db.CaptureError(err, "campaign segments list", nil, "rows") return nil, errx.InternalError() } - for i := range out { - n, xerr := r.Count(ctx, orgID, &out[i].SegmentID, matches[i], conds[i]) - if xerr != nil { - return nil, xerr - } - out[i].ContactCount = n + if len(out) == 0 { + return out, nil + } + if xerr := r.campaignLinkCounts(ctx, orgID, campaignID, out); xerr != nil { + return nil, xerr } return out, nil } -func (r *segmentRepository) SetForCampaign(ctx context.Context, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) *errx.Error { - // A nil slice would reach Postgres as ANY(NULL) and skip the delete. - if segmentIDs == nil { - segmentIDs = []uuid.UUID{} +// campaignLinkCounts fills the live counts of every link in one contacts scan: +// members, members that are leads, and members held out (a manual removal and +// not a lead, exactly the pairs the sync skips). +func (r *segmentRepository) campaignLinkCounts(ctx context.Context, orgID, campaignID uuid.UUID, links []models.CampaignSegmentLink) *errx.Error { + roots := make([]uuid.UUID, len(links)) + for i := range links { + roots[i] = links[i].SegmentID } + graph, err := loadSegmentGraph(ctx, r.DB, orgID, roots) + if err != nil { + db.CaptureError(err, "segment compile", nil, "query") + return errx.InternalError() + } + b := &segmentBuilder{orgID: orgID, args: []any{orgID}, graph: graph} + clauses := make([]string, len(links)) + for i := range links { + // A segment deleted between the two reads compiles to FALSE; its + // link row cascades away with it. + clauses[i] = "FALSE" + if def, ok := graph[links[i].SegmentID]; ok { + clauses[i] = b.segmentClause(def, true, map[uuid.UUID]bool{}) + } + } + args := append(b.args, campaignID) + cp := fmt.Sprintf("$%d", len(args)) + cols := make([]string, 0, len(links)*3) + for _, cl := range clauses { + cols = append(cols, + `COUNT(*) FILTER (WHERE (`+cl+`))`, + `COUNT(cl.contact_id) FILTER (WHERE (`+cl+`))`, + `COUNT(lr.contact_id) FILTER (WHERE cl.contact_id IS NULL AND (`+cl+`))`) + } + query := `SELECT ` + strings.Join(cols, ", ") + ` + FROM contacts c + LEFT JOIN campaign_leads cl ON cl.campaign_id = ` + cp + ` AND cl.contact_id = c.id + LEFT JOIN campaign_lead_removals lr ON lr.campaign_id = ` + cp + ` AND lr.contact_id = c.id + WHERE c.organization_id = $1 AND ((` + strings.Join(clauses, ") OR (") + `))` + dest := make([]any, 0, len(links)*3) + for i := range links { + dest = append(dest, &links[i].ContactCount, &links[i].LeadCount, &links[i].HeldOutCount) + } + if err := r.DB.QueryRow(ctx, query, args...).Scan(dest...); err != nil { + db.CaptureError(err, query, args, "queryrow") + return errx.InternalError() + } + return nil +} + +func (r *segmentRepository) SetForCampaign(ctx context.Context, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) *errx.Error { tx, err := r.DB.Begin(ctx) if err != nil { db.CaptureError(err, "", nil, "begin") return errx.InternalError() } defer tx.Rollback(ctx) - - var exists bool - if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM campaigns WHERE id = $1 AND organization_id = $2)`, campaignID, orgID).Scan(&exists); err != nil { - db.CaptureError(err, "campaign exists", nil, "queryrow") - return errx.InternalError() - } - if !exists { - return errx.New(errx.NotFound, "campaign not found") - } - if len(segmentIDs) > 0 { - var n int - if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM segments WHERE organization_id = $1 AND id = ANY($2::uuid[])`, orgID, segmentIDs).Scan(&n); err != nil { - db.CaptureError(err, "segments verify", nil, "queryrow") - return errx.InternalError() - } - if n != len(segmentIDs) { - return errx.New(errx.BadRequest, "a linked segment does not exist") - } - } - if _, err := tx.Exec(ctx, `DELETE FROM campaign_segments WHERE campaign_id = $1 AND NOT (segment_id = ANY($2::uuid[]))`, campaignID, segmentIDs); err != nil { - db.CaptureError(err, "campaign segments delete", nil, "exec") - return errx.InternalError() - } - if len(segmentIDs) > 0 { - if _, err := tx.Exec(ctx, `INSERT INTO campaign_segments (campaign_id, segment_id) SELECT $1, unnest($2::uuid[]) ON CONFLICT DO NOTHING`, campaignID, segmentIDs); err != nil { - db.CaptureError(err, "campaign segments insert", nil, "exec") - return errx.InternalError() - } + if _, xerr := setForCampaignTx(ctx, tx, orgID, campaignID, segmentIDs); xerr != nil { + return xerr } if err := tx.Commit(ctx); err != nil { db.CaptureError(err, "", nil, "commit") @@ -454,6 +462,73 @@ func (r *segmentRepository) SetForCampaign(ctx context.Context, orgID, campaignI return nil } +func (r *segmentRepository) ReplaceForCampaign(ctx context.Context, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) (int, string, *errx.Error) { + tx, err := r.DB.Begin(ctx) + if err != nil { + db.CaptureError(err, "", nil, "begin") + return 0, "", errx.InternalError() + } + defer tx.Rollback(ctx) + status, xerr := setForCampaignTx(ctx, tx, orgID, campaignID, segmentIDs) + if xerr != nil { + return 0, "", xerr + } + added, xerr := syncCampaignSegmentsTx(ctx, tx, orgID, campaignID) + if xerr != nil { + return 0, "", xerr + } + if err := tx.Commit(ctx); err != nil { + db.CaptureError(err, "", nil, "commit") + return 0, "", errx.InternalError() + } + return added, status, nil +} + +// setForCampaignTx replaces the links under a lock on the campaign row, so two +// concurrent replacements cannot commit the union of their sets; the status +// read under that lock is what the caller reacts to. +func setForCampaignTx(ctx context.Context, tx pgx.Tx, orgID, campaignID uuid.UUID, segmentIDs []uuid.UUID) (string, *errx.Error) { + // A nil slice would reach Postgres as ANY(NULL) and skip the delete. + if segmentIDs == nil { + segmentIDs = []uuid.UUID{} + } + var status string + if err := tx.QueryRow(ctx, `SELECT status FROM campaigns WHERE id = $1 AND organization_id = $2 FOR UPDATE`, campaignID, orgID).Scan(&status); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", errx.New(errx.NotFound, "campaign not found") + } + db.CaptureError(err, "campaign lock", nil, "queryrow") + return "", errx.InternalError() + } + if len(segmentIDs) > 0 { + var n int + if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM segments WHERE organization_id = $1 AND id = ANY($2::uuid[])`, orgID, segmentIDs).Scan(&n); err != nil { + db.CaptureError(err, "segments verify", nil, "queryrow") + return "", errx.InternalError() + } + if n != len(segmentIDs) { + return "", errx.New(errx.BadRequest, "a linked segment does not exist") + } + } + if _, err := tx.Exec(ctx, `DELETE FROM campaign_segments WHERE campaign_id = $1 AND NOT (segment_id = ANY($2::uuid[]))`, campaignID, segmentIDs); err != nil { + db.CaptureError(err, "campaign segments delete", nil, "exec") + return "", errx.InternalError() + } + if len(segmentIDs) > 0 { + if _, err := tx.Exec(ctx, `INSERT INTO campaign_segments (campaign_id, segment_id) SELECT $1, unnest($2::uuid[]) ON CONFLICT DO NOTHING`, campaignID, segmentIDs); err != nil { + db.CaptureError(err, "campaign segments insert", nil, "exec") + return "", errx.InternalError() + } + // A live audience is the reason to keep running: linking turns the + // setting on, and the owner can turn it off again in preferences. + if _, err := tx.Exec(ctx, `UPDATE campaigns SET continuous = true, updated_at = NOW() WHERE id = $1 AND NOT continuous`, campaignID); err != nil { + db.CaptureError(err, "campaign continuous", nil, "exec") + return "", errx.InternalError() + } + } + return status, nil +} + func (r *segmentRepository) SyncCampaignSegments(ctx context.Context, orgID, campaignID uuid.UUID) (int, *errx.Error) { tx, err := r.DB.Begin(ctx) if err != nil { @@ -461,7 +536,18 @@ func (r *segmentRepository) SyncCampaignSegments(ctx context.Context, orgID, cam return 0, errx.InternalError() } defer tx.Rollback(ctx) + added, xerr := syncCampaignSegmentsTx(ctx, tx, orgID, campaignID) + if xerr != nil { + return 0, xerr + } + if err := tx.Commit(ctx); err != nil { + db.CaptureError(err, "", nil, "commit") + return 0, errx.InternalError() + } + return added, nil +} +func syncCampaignSegmentsTx(ctx context.Context, tx pgx.Tx, orgID, campaignID uuid.UUID) (int, *errx.Error) { rows, err := tx.Query(ctx, ` SELECT cs.segment_id FROM campaign_segments cs JOIN campaigns cp ON cp.id = cs.campaign_id @@ -507,10 +593,6 @@ func (r *segmentRepository) SyncCampaignSegments(ctx context.Context, orgID, cam } total += len(links) } - if err := tx.Commit(ctx); err != nil { - db.CaptureError(err, "", nil, "commit") - return 0, errx.InternalError() - } return total, nil } diff --git a/internal/repository/pg_segment_sql.go b/internal/repository/pg_segment_sql.go index 6a10ad47..3df65a06 100644 --- a/internal/repository/pg_segment_sql.go +++ b/internal/repository/pg_segment_sql.go @@ -197,7 +197,7 @@ func (b *segmentBuilder) condition(c models.SegmentCondition, visited map[uuid.U case "is_catch_all": expr = "c.is_catch_all" case "suppressed": - expr = fmt.Sprintf("EXISTS (SELECT 1 FROM suppressed_recipients sr WHERE sr.organization_id = %s AND lower(sr.email) = lower(c.email) AND (sr.expires_at IS NULL OR sr.expires_at > now()))", b.bind(b.orgID)) + expr = fmt.Sprintf("recipient_suppressed(%s, c.email)", b.bind(b.orgID)) default: return "FALSE" } diff --git a/internal/repository/pg_sequence.go b/internal/repository/pg_sequence.go index 54ca5668..01b2de71 100644 --- a/internal/repository/pg_sequence.go +++ b/internal/repository/pg_sequence.go @@ -129,9 +129,12 @@ func (r *sequenceRepository) Create(ctx context.Context, userID string, campaign } defer tx.Rollback(ctx) + // FOR UPDATE serialises step creation per campaign, so two concurrent + // inserts on a one-time campaign cannot both see zero email steps. query := ` - SELECT user_id, organization_id + SELECT user_id, organization_id, kind FROM campaigns WHERE id = $1 + FOR UPDATE ` params := []any{ @@ -140,11 +143,12 @@ func (r *sequenceRepository) Create(ctx context.Context, userID string, campaign var ownerID string var orgID uuid.UUID + var kind string err = tx.QueryRow( ctx, query, params..., - ).Scan(&ownerID, &orgID) + ).Scan(&ownerID, &orgID, &kind) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, errx.ErrNotFound @@ -157,6 +161,19 @@ func (r *sequenceRepository) Create(ctx context.Context, userID string, campaign return nil, errx.ErrForbidden } + // A one-time email is a single message: a second email step would turn + // it into a sequence without the list, status wording or docs saying so. + if kind == models.CampaignKindOneTime { + var emailSteps int + if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM sequences WHERE campaign_id = $1 AND kind = 'email'`, campaignID).Scan(&emailSteps); err != nil { + db.CaptureError(err, "", params, "queryrow") + return nil, errx.InternalError() + } + if emailSteps > 0 { + return nil, errx.New(errx.BadRequest, "a one-time email has a single message; create a sequence campaign for follow-ups") + } + } + // Get the next position for this campaign's sequences var nextPos int _ = tx.QueryRow(ctx, `SELECT COALESCE(MAX(position), 0) + 1 FROM sequences WHERE campaign_id = $1`, campaignID).Scan(&nextPos) diff --git a/internal/repository/pg_task.go b/internal/repository/pg_task.go index e172dde6..2db2816b 100644 --- a/internal/repository/pg_task.go +++ b/internal/repository/pg_task.go @@ -398,15 +398,27 @@ func (r *taskRepository) GetEmailTask(ctx context.Context, taskID uuid.UUID) (*E return emailTask, err } -// CountCampaignEmailsSentToday counts only campaign tasks completed today (excludes warmup) +// taskDispatchedEmail is the WHERE fragment for "this completed task put an +// email on the wire", for every query that counts or times a mailbox's sends +// (alias t). Only campaign tasks need it: a campaign is one self-perpetuating +// task, and its wake-ups complete without sending (a deferral, an auto-pause, +// an action step), so status alone charged each of them to the mailbox's daily +// budget and reset its min-gap clock (issue #306). A campaign send is the task +// holding a step's reservation, or one the worker answered with a Message-ID. +const taskDispatchedEmail = `(t.task_type <> 'campaign' OR t.message_id <> '' OR EXISTS ( + SELECT 1 FROM campaign_contact_progress ccp WHERE ccp.dispatch_task_id = t.id))` + +// CountCampaignEmailsSentToday counts the campaign emails a mailbox dispatched +// today (excludes warmup, and the campaign chain's own wake-ups). func (r *taskRepository) CountCampaignEmailsSentToday(ctx context.Context, accountID uuid.UUID) (int, error) { query := ` SELECT COUNT(*) - FROM tasks - WHERE email_account_id = $1 - AND status = 'completed' - AND task_type = 'campaign' - AND DATE(completed_at) = CURRENT_DATE + FROM tasks t + WHERE t.email_account_id = $1 + AND t.status = 'completed' + AND t.task_type = 'campaign' + AND DATE(t.completed_at) = CURRENT_DATE + AND ` + taskDispatchedEmail + ` ` var count int @@ -467,10 +479,11 @@ func (r *taskRepository) CreateEmailTaskFull(ctx context.Context, task *Task, em func (r *taskRepository) CountEmailsSentToday(ctx context.Context, accountID uuid.UUID) (int, error) { query := ` SELECT COUNT(*) - FROM tasks - WHERE email_account_id = $1 - AND status = 'completed' - AND DATE(completed_at) = CURRENT_DATE + FROM tasks t + WHERE t.email_account_id = $1 + AND t.status = 'completed' + AND DATE(t.completed_at) = CURRENT_DATE + AND ` + taskDispatchedEmail + ` ` var count int @@ -493,13 +506,15 @@ func (r *taskRepository) CountWarmupEmailsSentToday(ctx context.Context, account return count, err } -// GetLastEmailTime gets the last email send time for an account +// GetLastEmailTime gets the last email send time for an account. It is the +// min-gap clock, so it reads real sends only. func (r *taskRepository) GetLastEmailTime(ctx context.Context, accountID uuid.UUID) (*time.Time, error) { query := ` - SELECT MAX(completed_at) - FROM tasks - WHERE email_account_id = $1 - AND status = 'completed' + SELECT MAX(t.completed_at) + FROM tasks t + WHERE t.email_account_id = $1 + AND t.status = 'completed' + AND ` + taskDispatchedEmail + ` ` var lastTime *time.Time @@ -528,13 +543,14 @@ func (r *taskRepository) GetLastSendTimes(ctx context.Context, accountIDs []uuid } query := ` - SELECT email_account_id, MAX(completed_at) - FROM tasks - WHERE email_account_id = ANY($1) - AND status = 'completed' - AND task_type = $2::task_type - AND completed_at IS NOT NULL - GROUP BY email_account_id + SELECT t.email_account_id, MAX(t.completed_at) + FROM tasks t + WHERE t.email_account_id = ANY($1) + AND t.status = 'completed' + AND t.task_type = $2::task_type + AND t.completed_at IS NOT NULL + AND ` + taskDispatchedEmail + ` + GROUP BY t.email_account_id ` rows, err := r.db.Query(ctx, query, accountIDs, taskType) diff --git a/internal/repository/pg_tracked_links.go b/internal/repository/pg_tracked_links.go index 8ca4979c..3d011a99 100644 --- a/internal/repository/pg_tracked_links.go +++ b/internal/repository/pg_tracked_links.go @@ -16,7 +16,10 @@ type TrackedLink struct { TaskID uuid.UUID CampaignID uuid.UUID Destination string - CreatedAt time.Time + // Label is the anchor text the link was minted from ("Pricing"), so a + // click can be named without re-parsing the email. Empty for image links. + Label string + CreatedAt time.Time } // TrackedLinkRepository is the server-side click-link store. Only the send @@ -47,12 +50,12 @@ func (r *trackedLinkRepository) CreateBatch(ctx context.Context, links []Tracked rows := make([][]any, 0, len(links)) for _, l := range links { - rows = append(rows, []any{l.ID, l.TaskID, l.CampaignID, l.Destination}) + rows = append(rows, []any{l.ID, l.TaskID, l.CampaignID, l.Destination, l.Label}) } _, err := r.db.CopyFrom(ctx, pgx.Identifier{"tracked_links"}, - []string{"id", "task_id", "campaign_id", "destination"}, + []string{"id", "task_id", "campaign_id", "destination", "label"}, pgx.CopyFromRows(rows), ) return err @@ -61,13 +64,13 @@ func (r *trackedLinkRepository) CreateBatch(ctx context.Context, links []Tracked // GetByID resolves a ticket to its destination. nil, nil when unknown. func (r *trackedLinkRepository) GetByID(ctx context.Context, id uuid.UUID) (*TrackedLink, error) { query := ` - SELECT id, task_id, campaign_id, destination, created_at + SELECT id, task_id, campaign_id, destination, label, created_at FROM tracked_links WHERE id = $1 ` var l TrackedLink - err := r.db.QueryRow(ctx, query, id).Scan(&l.ID, &l.TaskID, &l.CampaignID, &l.Destination, &l.CreatedAt) + err := r.db.QueryRow(ctx, query, id).Scan(&l.ID, &l.TaskID, &l.CampaignID, &l.Destination, &l.Label, &l.CreatedAt) if err == pgx.ErrNoRows { return nil, nil } diff --git a/internal/repository/pg_warmup_content.go b/internal/repository/pg_warmup_content.go index 89b493f6..08910df5 100644 --- a/internal/repository/pg_warmup_content.go +++ b/internal/repository/pg_warmup_content.go @@ -69,6 +69,10 @@ type WarmupContentRepository interface { // ListActiveBatchJobs returns batch-mode jobs still in flight (running with a // non-terminal OpenAI batch status), for the poller to reconcile. ListActiveBatchJobs(ctx context.Context) ([]models.WarmupGenerationJob, error) + // MarkBatchCancelling records an admin cancel on a job only while it is still + // in flight, so it never overwrites a job the poller finished meanwhile. + // It reports whether the job was claimed. + MarkBatchCancelling(ctx context.Context, id uuid.UUID, reason string) (bool, error) GeneratedCountSince(ctx context.Context, since time.Time) (int, error) ExpireStaleScheduledJobs(ctx context.Context, before time.Time) (int64, error) WarmupSendsSince(ctx context.Context, since time.Time) (int, error) @@ -378,6 +382,22 @@ func (r *warmupContentRepository) UpdateGenerationJob(ctx context.Context, j *mo return err } +func (r *warmupContentRepository) MarkBatchCancelling(ctx context.Context, id uuid.UUID, reason string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE warmup_generation_jobs + SET batch_status = 'cancelling', + error = CASE WHEN error = '' THEN $2 ELSE error END, + updated_at = NOW() + WHERE id = $1 + AND status = 'running' + AND batch_status NOT IN ('completed', 'failed', 'expired', 'cancelled', 'cancelling') + `, id, reason) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + // ExpireStaleScheduledJobs releases reservations left behind if a backend dies // after reserving a batch but before receiving the provider batch ID. func (r *warmupContentRepository) ExpireStaleScheduledJobs(ctx context.Context, before time.Time) (int64, error) { diff --git a/internal/repository/segment_live_test.go b/internal/repository/segment_live_test.go index a68a815f..f1b0b5f7 100644 --- a/internal/repository/segment_live_test.go +++ b/internal/repository/segment_live_test.go @@ -265,14 +265,18 @@ func TestLiveSegmentCampaignLinks(t *testing.T) { t.Fatalf("unknown segment accepted") } links, xerr := repo.ListForCampaign(ctx, f.org, f.other) - if xerr != nil || len(links) != 1 || links[0].SegmentID != acme.ID || links[0].ContactCount != 2 { + if xerr != nil || len(links) != 1 || links[0].SegmentID != acme.ID || links[0].ContactCount != 2 || links[0].LeadCount != 0 { t.Fatalf("links = %+v, %v", links, xerr) } - // Sync enrols current members once; a second pass adds nothing. - added, xerr := repo.SyncCampaignSegments(ctx, f.org, f.other) - if xerr != nil || added != 2 { - t.Fatalf("sync = %d, %v", added, xerr) + // Replacing the set enrols current members in the same transaction; a + // second pass adds nothing. + added, status, xerr := repo.ReplaceForCampaign(ctx, f.org, f.other, []uuid.UUID{acme.ID}) + if xerr != nil || added != 2 || status == "" { + t.Fatalf("replace = %d, %q, %v", added, status, xerr) + } + if links, _ = repo.ListForCampaign(ctx, f.org, f.other); len(links) != 1 || links[0].LeadCount != 2 || links[0].HeldOutCount != 0 { + t.Fatalf("links after replace = %+v", links) } if added, _ = repo.SyncCampaignSegments(ctx, f.org, f.other); added != 0 { t.Fatalf("second sync = %d, want 0", added) @@ -298,6 +302,12 @@ func TestLiveSegmentCampaignLinks(t *testing.T) { if added, _ = repo.SyncCampaignSegments(ctx, f.org, f.other); added != 0 { t.Fatalf("sync after manual removal = %d, want 0", added) } + // The link reports the split the Leads tab explains: three members, two + // of them leads, one held out by the removal. + links, xerr = repo.ListForCampaign(ctx, f.org, f.other) + if xerr != nil || len(links) != 1 || links[0].ContactCount != 3 || links[0].LeadCount != 2 || links[0].HeldOutCount != 1 { + t.Fatalf("links after removal = %+v, %v", links, xerr) + } if _, xerr := contacts.BulkUpdate(ctx, f.owner.String(), f.org, &models.BulkEditContactsData{ Contacts: []string{f.bob.String()}, AddCampaigns: []string{f.other.String()}, }); xerr != nil { @@ -307,6 +317,9 @@ func TestLiveSegmentCampaignLinks(t *testing.T) { if err := handle.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_lead_removals WHERE campaign_id = $1`, f.other).Scan(&removals); err != nil || removals != 0 { t.Fatalf("removals after manual re-add = %d, %v", removals, err) } + if links, _ = repo.ListForCampaign(ctx, f.org, f.other); len(links) != 1 || links[0].LeadCount != 3 || links[0].HeldOutCount != 0 { + t.Fatalf("links after manual re-add = %+v", links) + } if _, xerr := contacts.BulkUpdate(ctx, f.owner.String(), f.org, removeBob); xerr != nil { t.Fatalf("remove again: %v", xerr) } diff --git a/internal/sandbox/personas.go b/internal/sandbox/personas.go index e61c387c..01480805 100644 --- a/internal/sandbox/personas.go +++ b/internal/sandbox/personas.go @@ -11,6 +11,10 @@ type persona struct { Opens bool Clicks bool Replies bool + // Scanned marks a contact behind a corporate security gateway that + // opens the pixel and follows every link seconds after delivery with an + // ordinary browser user agent, so the sandbox exercises machine detection. + Scanned bool Flavor replyFlavor } @@ -34,6 +38,7 @@ func personaFor(email string) persona { Opens: n%100 < 92, Clicks: n%7 < 4, // ~57% of openers Replies: n%11 < 5, // ~45% of openers + Scanned: n%5 == 0, // ~20% sit behind a link-scanning gateway } switch n % 10 { case 0, 1, 2, 3: diff --git a/internal/sandbox/simulate.go b/internal/sandbox/simulate.go index 1bd3c3c9..70e318cb 100644 --- a/internal/sandbox/simulate.go +++ b/internal/sandbox/simulate.go @@ -212,9 +212,25 @@ func (s *simulator) actAsContact(ctx context.Context, c contactInfo, msg *mailpi body = msg.Text } + // A security gateway scans the message at delivery: pixel plus every + // link, one after another, before anyone could have read it. The + // consumer must label these as machine and never count them as clicks. + if p.Scanned { + s.sleep(ctx, 500*time.Millisecond, 2*time.Second) + if task := firstMatch(pixelRe, body); task != "" { + s.hitTracking(ctx, "/t/o/"+task+".png", c.Email) + } + for _, ticket := range allMatches(clickRe, body) { + s.hitTracking(ctx, "/c/"+ticket, c.Email) + } + fmt.Printf("scanned %-34s %q\n", c.Email, msg.Subject) + } + if p.Opens { if task := firstMatch(pixelRe, body); task != "" { - s.sleep(ctx, 5*time.Second, 40*time.Second) + // Never inside the machine window: a person needs the message + // delivered, noticed and opened first. + s.sleep(ctx, 15*time.Second, 40*time.Second) s.hitTracking(ctx, "/t/o/"+task+".png", c.Email) fmt.Printf("opened %-34s %q\n", c.Email, msg.Subject) } @@ -274,3 +290,17 @@ func firstMatch(re *regexp.Regexp, body string) string { } return m[1] } + +// allMatches returns every distinct first capture group, in document order. +func allMatches(re *regexp.Regexp, body string) []string { + var out []string + seen := map[string]bool{} + for _, m := range re.FindAllStringSubmatch(body, -1) { + if len(m) < 2 || seen[m[1]] { + continue + } + seen[m[1]] = true + out = append(out, m[1]) + } + return out +} diff --git a/internal/scheduler/campaign_scheduler.go b/internal/scheduler/campaign_scheduler.go index 38d4144e..ed3c44c9 100644 --- a/internal/scheduler/campaign_scheduler.go +++ b/internal/scheduler/campaign_scheduler.go @@ -2,7 +2,9 @@ package scheduler import ( "context" + "fmt" "math/rand" + "strings" "time" "github.com/google/uuid" @@ -199,6 +201,11 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode s.logCampaignDecision(ctx, campaignID, eventType, message, metadata) } } + logDecisionOnce := func(eventType, message string, metadata map[string]interface{}) { + if !preview { + s.logCampaignDecisionOnce(ctx, campaignID, eventType, message, metadata) + } + } // STEP 3.5: Resolve the recipient ESP/provider for ESP matching. Cheap: // prefer the cached contact.esp_provider, else derive from the domain @@ -368,6 +375,16 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode authGated := 0 lifecycleGated := 0 + // Why the other mailboxes were left out. A reason that clears on its own + // (the budget resets at midnight, the mailbox's hours reopen, a health hold + // expires) makes an empty pool a deferral; only one that never clears makes + // it a pause. reopensAt is the earliest reopening among hours-closed + // mailboxes. + budgetSpent := 0 + hoursClosed := 0 + healthHeld := 0 + var reopensAt time.Time + var candidates []AccountCandidate for _, acct := range accounts { // Sending-domain authentication. This runs before the daily-count query @@ -401,6 +418,7 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode // Skip accounts that have reached their daily limit if remaining <= 0 { + budgetSpent++ continue } @@ -419,11 +437,13 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode switch state { case models.WarmupHealthQuarantined, models.WarmupHealthBlocked: if blockedUntil == nil || blockedUntil.After(time.Now()) { + healthHeld++ continue } case models.WarmupHealthWatch, models.WarmupHealthThrottled: remaining = int(float64(remaining) * adjustmentFor(state).volumeMultiplier) if remaining <= 0 { + budgetSpent++ continue } } @@ -452,6 +472,7 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode // cold cap. remaining = s.behaviorDailyCap(ctx, bhv, remaining, openAt) if remaining <= 0 { + budgetSpent++ continue } } else if acct.Timezone != "" && acct.Timezone != campaign.Timezone { @@ -459,7 +480,12 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode acctLocal := candidateTime.In(acctTZ) acctHour := acctLocal.Hour() if acctHour < 8 || acctHour >= 20 { - continue // outside account's business hours + // Outside the account's business hours. + hoursClosed++ + if open := businessHoursReopen(candidateTime, acctTZ); reopensAt.IsZero() || open.Before(reopensAt) { + reopensAt = open + } + continue } } @@ -512,13 +538,63 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode return time.Time{}, nil, uuid.Nil, ErrDomainAuthFailing } - // Every mailbox is out of cold rotation. Say so rather than letting the - // campaign look stalled for no visible reason; they return on their own. - if len(candidates) == 0 && lifecycleGated == len(accounts) { - logDecision("mailboxes_resting", - "No mailbox is in cold rotation: all are resting or held in reserve", - map[string]interface{}{"resting_mailboxes": lifecycleGated, "pool_size": len(accounts)}) - return s.deferToNextDay(campaign), nil, accounts[0].ID, ErrCampaignDeferred + // An empty pool is a pause only when nothing in it will change on its own. + // A pool with every mailbox at its daily cap used to fall through to the + // pause below, and the campaign had to be restarted by hand the next + // morning (issue #306); it is a deferral, like every other gate that lifts + // by itself. The conditions a deferred chain re-finds every few minutes + // until midnight are logged once a day, or the feed drowns in them. + if len(candidates) == 0 { + switch { + case budgetSpent > 0 || hoursClosed > 0: + // Resume when the first of them can send again: tomorrow for a + // spent budget, the reopening of the mailbox's own 8am-8pm band + // otherwise. A closed band is routine and short, so it earns no + // line in the activity log; a pool whose every usable mailbox is + // capped does. + var resume time.Time + if budgetSpent > 0 { + resume = s.deferToNextDay(campaign) + } + if hoursClosed > 0 { + if open := nextScheduleSlot(reopensAt, windows, campaignTZ); resume.IsZero() || open.Before(resume) { + resume = open + } + } + if hoursClosed == 0 { + logDecisionOnce("daily_cap_reached", + "Every available mailbox has used its daily budget; sending resumes tomorrow", + map[string]interface{}{"capped_mailboxes": budgetSpent, "pool_size": len(accounts)}) + } + return resume, nil, accounts[0].ID, ErrCampaignDeferred + case lifecycleGated == len(accounts): + // Every mailbox is out of cold rotation. Say so rather than letting + // the campaign look stalled for no visible reason; they return on + // their own. + logDecisionOnce("mailboxes_resting", + "No mailbox is in cold rotation: all are resting or held in reserve", + map[string]interface{}{"resting_mailboxes": lifecycleGated, "pool_size": len(accounts)}) + return s.deferToNextDay(campaign), nil, accounts[0].ID, ErrCampaignDeferred + case healthHeld > 0 || lifecycleGated > 0: + var why []string + if healthHeld > 0 { + why = append(why, fmt.Sprintf("%d held by warmup health", healthHeld)) + } + if lifecycleGated > 0 { + why = append(why, fmt.Sprintf("%d resting or in reserve", lifecycleGated)) + } + if authGated > 0 { + why = append(why, fmt.Sprintf("%d failing domain authentication", authGated)) + } + logDecisionOnce("mailboxes_unavailable", + "No mailbox can send right now: "+strings.Join(why, ", "), + map[string]interface{}{"health_held": healthHeld, "resting_mailboxes": lifecycleGated, + "auth_gated": authGated, "pool_size": len(accounts)}) + return s.deferToNextDay(campaign), nil, accounts[0].ID, ErrCampaignDeferred + } + // What is left was gated by a sending-behaviour profile with no working + // days, which no amount of waiting fixes. + return time.Time{}, nil, uuid.Nil, ErrNoEligibleMailbox } // STEP 8.25: Apply ESP matching to the under-budget candidate set. @@ -555,57 +631,12 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode // STEP 8.5: Select best account per the campaign's rotation mode. pool is // the set selection actually ran over, kept for the pacing maths below. + // Every candidate has budget left today, so it has weight and the selector + // always picks one; the guard only keeps a nil from being dereferenced. pool := candidates selected := selectAccountByRotationMode(campaign.RotationMode, candidates) if selected == nil { - // ALL accounts at capacity today — push to next day and recompute with - // tomorrow's full (ramp-clamped) capacity. The ramp clamp AND the ESP - // filter MUST be re-applied here, or tomorrow's recompute over-budgets a - // mailbox past its ramp ceiling / picks a cross-provider sender. - candidateTime = candidateTime.Add(24 * time.Hour) - candidateTime = nextScheduleSlot(candidateTime, windows, campaignTZ) - - var tomorrow []AccountCandidate - for i := range candidates { - acct := candidates[i].Account - // ESP-strict: keep only matching mailboxes for tomorrow too. - if campaign.ESPMatchMode == "strict" && recipientProvider != "" && !candidates[i].ProviderMatch { - continue - } - acctLimit := effectiveCap(acct) // same ramp clamp as STEP 8 - c := candidates[i] - c.RemainingToday = acctLimit - c.Weight = computeWeight(acctLimit, candidates[i].WarmupAgeDays) - tomorrow = append(tomorrow, c) - } - // ESP-prefer: restrict tomorrow to matching mailboxes when any exist. - if campaign.ESPMatchMode == "prefer" && recipientProvider != "" { - var matchingTomorrow []AccountCandidate - for _, c := range tomorrow { - if c.ProviderMatch { - matchingTomorrow = append(matchingTomorrow, c) - } - } - if len(matchingTomorrow) > 0 { - tomorrow = matchingTomorrow - } - } - - pool = tomorrow - selected = selectAccountByRotationMode(campaign.RotationMode, tomorrow) - if selected == nil { - // ESP-strict with no matching mailbox at all: defer rather than - // complete or send cross-provider. - if campaign.ESPMatchMode == "strict" && recipientProvider != "" { - logDecision("provider_match_deferred", - "No same-provider mailbox available tomorrow; deferring", - map[string]interface{}{"recipient_provider": recipientProvider}) - // Deferral, not a send (see above). - return s.deferToNextDay(campaign), nil, accounts[0].ID, ErrCampaignDeferred - } - // The pool was not empty, every mailbox in it was gated out. - return time.Time{}, nil, uuid.Nil, ErrNoEligibleMailbox - } + return time.Time{}, nil, uuid.Nil, ErrNoEligibleMailbox } account := &selected.Account @@ -740,8 +771,9 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode } // deferToNextDay pushes a candidate time to the next valid campaign day within -// the campaign's send window. Used by the ESP-strict and new-lead-cap deferral -// paths so a campaign reschedules instead of completing or busy-looping. +// the campaign's send window. Used by the ESP-strict, new-lead-cap and +// daily-cap deferral paths so a campaign reschedules instead of completing, +// pausing or busy-looping. func (s *schedulerService) deferToNextDay(campaign *models.Campaign) time.Time { tz := loadLocation(campaign.Timezone) t := nextScheduleSlot(time.Now().Add(24*time.Hour), effectiveWindows(campaign), tz) @@ -763,3 +795,24 @@ func (s *schedulerService) logCampaignDecision(ctx context.Context, campaignID u Metadata: metadata, }) } + +// logCampaignDecisionOnce is logCampaignDecision for a condition the deferred +// chain re-finds on every wake-up until the day rolls over: one line per UTC +// day, the day the budgets reset on. Best-effort and nil-safe. +func (s *schedulerService) logCampaignDecisionOnce(ctx context.Context, campaignID uuid.UUID, eventType, message string, metadata map[string]interface{}) { + if s.campaignLogRepo == nil { + return + } + if metadata == nil { + metadata = map[string]interface{}{} + } + dayStart := time.Now().UTC().Truncate(24 * time.Hour) + day := dayStart.Format("2006-01-02") + metadata["day"] = day + _, _ = s.campaignLogRepo.CreateLogOnce(ctx, &repository.CampaignLogEntry{ + CampaignID: campaignID, + EventType: eventType, + Message: message, + Metadata: metadata, + }, "day", day, dayStart) +} diff --git a/internal/scheduler/daily_budget_live_test.go b/internal/scheduler/daily_budget_live_test.go new file mode 100644 index 00000000..2974a055 --- /dev/null +++ b/internal/scheduler/daily_budget_live_test.go @@ -0,0 +1,284 @@ +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/pkg/encrypt" + "github.com/warmbly/warmbly/internal/repository" +) + +// Live checks for issue #306: the per-mailbox daily budget counted every +// completed campaign task as a sent email, so the chain's own wake-ups (a +// deferral, a pause) spent the budget and reset the min-gap clock, and a pool +// with every mailbox at its cap paused the campaign instead of waiting for +// tomorrow. Same harness and env var as live_integration_test.go. + +// completeCampaignTask writes one campaign task the mailbox completed today. +// With a step it is a send: the step's reservation points at the task, the way +// ReserveSend leaves it. Without one it is a wake-up that sent nothing. +func (f *liveFixture) completeCampaignTask(t *testing.T, contact, step *uuid.UUID, completedAt time.Time) uuid.UUID { + t.Helper() + ctx := context.Background() + taskID := uuid.New() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO tasks (id, task_type, email_account_id, status, message_id, scheduled_at, completed_at, created_at, updated_at) + VALUES ($1, 'campaign', $2, 'completed', '', $3, $3, $3, $3)`, taskID, f.mailbox, completedAt); err != nil { + t.Fatalf("complete task: %v", err) + } + if _, err := f.pool.Exec(ctx, `INSERT INTO campaign_tasks (task_id, campaign_id, contact_id, sequence_id) + VALUES ($1, $2, $3, $4)`, taskID, f.campaign, contact, step); err != nil { + t.Fatalf("link task: %v", err) + } + if contact != nil && step != nil { + if _, err := f.pool.Exec(ctx, `INSERT INTO campaign_contact_progress + (campaign_id, contact_id, sequence_id, sent_at, dispatched_at, dispatch_task_id) + VALUES ($1, $2, $3, $4, $4, $5)`, f.campaign, *contact, *step, completedAt, taskID); err != nil { + t.Fatalf("reserve step: %v", err) + } + } + return taskID +} + +// addSentLead attaches a second lead whose first step this mailbox already +// sent today, so the pool has one real send on the books while the fixture's +// own lead is still waiting to go. +func (f *liveFixture) addSentLead(t *testing.T) { + t.Helper() + ctx := context.Background() + var step uuid.UUID + if err := f.pool.QueryRow(ctx, `SELECT id FROM sequences WHERE campaign_id = $1 ORDER BY position LIMIT 1`, f.campaign).Scan(&step); err != nil { + t.Fatalf("find step: %v", err) + } + contact := uuid.New() + if _, err := f.pool.Exec(ctx, `INSERT INTO contacts (id, user_id, organization_id, email, first_name, last_name, company, phone, custom_fields) + VALUES ($1, $2, $3, $4, 'Sent', 'Lead', '', '', '{}')`, + contact, f.user, f.org, "sent-"+contact.String()[:8]+"@test.local"); err != nil { + t.Fatalf("add contact: %v", err) + } + if _, err := f.pool.Exec(ctx, `INSERT INTO campaign_leads (campaign_id, contact_id, position) VALUES ($1, $2, 1)`, + f.campaign, contact); err != nil { + t.Fatalf("link lead: %v", err) + } + f.completeCampaignTask(t, &contact, &step, time.Now().Add(-2*time.Hour)) +} + +// loggedScheduler is liveScheduler with the activity log wired, for the +// checks on what a deferral records. +func loggedScheduler(t *testing.T, f *liveFixture) SchedulerService { + t.Helper() + handle, pool := liveDB(t) + enc, err := encrypt.NewEncrypter([]byte("0123456789abcdef0123456789abcdef")) + if err != nil { + t.Fatalf("encrypter: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM campaign_logs WHERE campaign_id = $1`, f.campaign) + }) + return NewSchedulerService( + repository.NewTaskRepository(pool), + repository.NewWarmupRepository(pool), + repository.NewCampaignProgressRepository(pool), + repository.NewEmailRepostory(handle, enc), + repository.NewCampaignRepostory(handle), + repository.NewContactRepostory(handle), + repository.NewCampaignLogRepository(handle), + ) +} + +// TestLiveWakeupsDoNotSpendTheDailyBudget: a mailbox whose chain woke up fifty +// times today without sending has its whole budget left, and no min-gap to +// wait out. +func TestLiveWakeupsDoNotSpendTheDailyBudget(t *testing.T) { + handle, pool := liveDB(t) + f := newLiveFixture(t, pool, "UTC") + ctx := context.Background() + + // Fifty completed wake-ups, the last one seconds ago: exactly the cap, and + // inside the 600s min-gap. + for i := 0; i < 50; i++ { + f.completeCampaignTask(t, nil, nil, time.Now().Add(-time.Duration(i)*time.Second)) + } + + taskRepo := repository.NewTaskRepository(pool) + sent, err := taskRepo.CountCampaignEmailsSentToday(ctx, f.mailbox) + if err != nil { + t.Fatalf("count: %v", err) + } + if sent != 0 { + t.Fatalf("CountCampaignEmailsSentToday = %d for a mailbox that sent nothing (issue #306)", sent) + } + last, err := taskRepo.GetLastEmailTime(ctx, f.mailbox) + if err != nil { + t.Fatalf("last email time: %v", err) + } + if last != nil { + t.Fatalf("GetLastEmailTime = %s for a mailbox that sent nothing; a wake-up reset the min-gap clock", last) + } + + at, pair, accountID, err := liveScheduler(t, handle, pool).CalculateNextCampaignTime(ctx, f.campaign) + if err != nil { + t.Fatalf("the campaign should be sendable, got %v", err) + } + if pair == nil || accountID != f.mailbox { + t.Fatalf("no sendable pair from the fixture mailbox: pair=%v account=%s", pair, accountID) + } + assertFuture(t, at) +} + +// TestLiveRealSendsStillSpendTheDailyBudget is the other half: the counter has +// to keep seeing the sends it exists for, both the reserved one and the one +// only the worker's Message-ID vouches for. +func TestLiveRealSendsStillSpendTheDailyBudget(t *testing.T) { + _, pool := liveDB(t) + f := newLiveFixture(t, pool, "UTC") + ctx := context.Background() + + f.addSentLead(t) + // A send whose reservation is gone but whose task carries the worker's + // Message-ID (a step walked back and re-sent, or one that predates the + // reservation). + confirmed := f.completeCampaignTask(t, nil, nil, time.Now().Add(-time.Hour)) + if _, err := pool.Exec(ctx, `UPDATE tasks SET message_id = '' WHERE id = $1`, confirmed); err != nil { + t.Fatal(err) + } + // And one wake-up, which must not count. + f.completeCampaignTask(t, nil, nil, time.Now().Add(-time.Minute)) + + taskRepo := repository.NewTaskRepository(pool) + sent, err := taskRepo.CountCampaignEmailsSentToday(ctx, f.mailbox) + if err != nil { + t.Fatalf("count: %v", err) + } + if sent != 2 { + t.Fatalf("CountCampaignEmailsSentToday = %d, want 2 (the reserved send and the confirmed one)", sent) + } + last, err := taskRepo.GetLastEmailTime(ctx, f.mailbox) + if err != nil { + t.Fatalf("last email time: %v", err) + } + if last == nil || time.Since(*last) < 50*time.Minute || time.Since(*last) > 70*time.Minute { + t.Fatalf("GetLastEmailTime = %v, want the confirmed send an hour ago, not the wake-up a minute ago", last) + } +} + +// addClosedHoursMailbox attaches a second active mailbox whose own 8am-8pm +// band is closed right now, and returns its timezone. Picked from a spread of +// offsets no wider than the closed band, so one always qualifies whatever the +// hour. It must still be closed when the scheduler runs a moment later, or a +// mailbox picked at 07:59 would be open by then and the pass would send. +func (f *liveFixture) addClosedHoursMailbox(t *testing.T) *time.Location { + t.Helper() + ctx := context.Background() + closed := func(at time.Time) bool { return at.Hour() < 8 || at.Hour() >= 20 } + var loc *time.Location + for _, name := range []string{"Pacific/Honolulu", "America/Los_Angeles", "America/New_York", "Europe/London", + "Europe/Berlin", "Asia/Dubai", "Asia/Tokyo", "Pacific/Auckland"} { + l, err := time.LoadLocation(name) + if err != nil { + continue + } + now := time.Now().In(l) + if closed(now) && closed(now.Add(10*time.Minute)) { + loc = l + break + } + } + if loc == nil { + t.Fatal("no timezone in the spread stays outside 8am-8pm for the next ten minutes") + } + mailbox := uuid.New() + if _, err := f.pool.Exec(ctx, `INSERT INTO email_accounts (id, user_id, organization_id, email, name, + signature_plain, signature_html, provider, status, campaign_limit, min_wait_time, timezone) + VALUES ($1, $2, $3, $4, 'Closed', '', '', 'smtp_imap', 'active', 50, 600, $5)`, + mailbox, f.user, f.org, "closed-"+mailbox.String()[:8]+"@test.local", loc.String()); err != nil { + t.Fatalf("add mailbox: %v", err) + } + t.Cleanup(func() { + c := context.Background() + _, _ = f.pool.Exec(c, `DELETE FROM campaign_tasks WHERE task_id IN (SELECT id FROM tasks WHERE email_account_id = $1)`, mailbox) + _, _ = f.pool.Exec(c, `DELETE FROM tasks WHERE email_account_id = $1`, mailbox) + _, _ = f.pool.Exec(c, `DELETE FROM email_accounts WHERE id = $1`, mailbox) + }) + return loc +} + +// TestLiveMixedPoolResumesAtTheEarlierMailbox: one mailbox at its cap and one +// merely outside its own hours resume when the second reopens, not tomorrow. +func TestLiveMixedPoolResumesAtTheEarlierMailbox(t *testing.T) { + _, pool := liveDB(t) + f := newLiveFixture(t, pool, "UTC") + ctx := context.Background() + + if _, err := pool.Exec(ctx, `UPDATE campaigns SET daily_limit = 1 WHERE id = $1`, f.campaign); err != nil { + t.Fatal(err) + } + f.addSentLead(t) + loc := f.addClosedHoursMailbox(t) + + s := loggedScheduler(t, f) + at, pair, _, err := s.CalculateNextCampaignTime(ctx, f.campaign) + if !errors.Is(err, ErrCampaignDeferred) || pair != nil { + t.Fatalf("want a deferral, got err=%v pair=%v", err, pair) + } + // Exactly the reopening, within a minute: later means the capped mailbox's + // "tomorrow" won, earlier means the pass never really deferred. + reopen := businessHoursReopen(time.Now(), loc) + if at.Before(reopen.Add(-time.Minute)) || at.After(reopen.Add(time.Minute)) { + t.Fatalf("deferred to %s, but the second mailbox reopens at %s", at, reopen) + } + assertFuture(t, at) + var logged int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id = $1 AND event_type = 'daily_cap_reached'`, + f.campaign).Scan(&logged); err != nil { + t.Fatal(err) + } + if logged != 0 { + t.Fatalf("daily_cap_reached logged for a pool that still has a mailbox coming back today") + } +} + +// TestLiveDailyCapDefersInsteadOfPausing: with every mailbox at its cap the +// campaign waits for tomorrow, says so once, and is never paused. +func TestLiveDailyCapDefersInsteadOfPausing(t *testing.T) { + _, pool := liveDB(t) + f := newLiveFixture(t, pool, "UTC") + ctx := context.Background() + + if _, err := pool.Exec(ctx, `UPDATE campaigns SET daily_limit = 1 WHERE id = $1`, f.campaign); err != nil { + t.Fatal(err) + } + f.addSentLead(t) + + s := loggedScheduler(t, f) + at, pair, _, err := s.CalculateNextCampaignTime(ctx, f.campaign) + if errors.Is(err, ErrNoEmailAccounts) { + t.Fatalf("a pool at its daily cap paused the campaign: %v", err) + } + if !errors.Is(err, ErrCampaignDeferred) { + t.Fatalf("want ErrCampaignDeferred, got err=%v pair=%v", err, pair) + } + if pair != nil { + t.Fatal("a deferral must never hand back a sendable pair") + } + if !at.After(time.Now().Add(23 * time.Hour)) { + t.Fatalf("deferred to %s, want tomorrow", at) + } + + // A second pass on the same day logs nothing new. + if _, _, _, err := s.CalculateNextCampaignTime(ctx, f.campaign); !errors.Is(err, ErrCampaignDeferred) { + t.Fatalf("second pass: %v", err) + } + var logged int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id = $1 AND event_type = 'daily_cap_reached'`, + f.campaign).Scan(&logged); err != nil { + t.Fatal(err) + } + if logged != 1 { + t.Fatalf("daily_cap_reached logged %d times over two passes, want once per day", logged) + } +} diff --git a/internal/scheduler/errors.go b/internal/scheduler/errors.go index 54a9a63e..e9d2f7f6 100644 --- a/internal/scheduler/errors.go +++ b/internal/scheduler/errors.go @@ -25,15 +25,17 @@ var ( ErrNoEmailAccounts = errors.New("no email accounts available for this campaign") // ErrNoEligibleMailbox is the narrower case: the campaign HAS mailboxes, - // but every one was gated out for both today and tomorrow (daily cap - // reached, warmup health, or outside its own sending window). Reporting - // that as ErrNoEmailAccounts sent people looking at their tag configuration - // for a problem that was never there. + // but none can send under its current settings, and waiting will not + // change that (a sending-behaviour profile with no working days). A gate + // that lifts on its own, such as a spent daily budget, a closed sending + // window or a warmup health hold, is a deferral instead, never this. + // Reporting this as ErrNoEmailAccounts sent people looking at their tag + // configuration for a problem that was never there. // // It wraps ErrNoEmailAccounts so existing callers that pause the campaign // on errors.Is(err, ErrNoEmailAccounts) keep behaving exactly as before. ErrNoEligibleMailbox = fmt.Errorf( - "%w: every mailbox is outside its sending window or over its daily budget", ErrNoEmailAccounts) + "%w: no mailbox can send under its current sending settings", ErrNoEmailAccounts) // ErrDomainAuthFailing is the narrower case again: every mailbox in the // campaign's pool was gated by the sending-domain authentication check. diff --git a/internal/scheduler/helpers.go b/internal/scheduler/helpers.go index 7b2b3403..2c2ec00e 100644 --- a/internal/scheduler/helpers.go +++ b/internal/scheduler/helpers.go @@ -132,6 +132,18 @@ func ensureBusinessHours(t time.Time, timezone string) time.Time { return ensureTimeWindow(t, "08:00", "20:00", loc) } +// businessHoursReopen is when the 8am-8pm band next opens for an instant +// that sits outside it: 8am the same local day before the band, 8am the next +// local day after it. +func businessHoursReopen(t time.Time, loc *time.Location) time.Time { + local := t.In(loc) + open := time.Date(local.Year(), local.Month(), local.Day(), 8, 0, 0, 0, loc) + if local.Hour() >= 20 { + open = open.AddDate(0, 0, 1) + } + return open +} + // calculateHoursRemainingUntil calculates hours remaining until a specific end time func calculateHoursRemainingUntil(timezone, endTime string) float64 { loc := loadLocation(timezone) diff --git a/internal/seed/admin.go b/internal/seed/admin.go index adebfac0..3bce84c7 100644 --- a/internal/seed/admin.go +++ b/internal/seed/admin.go @@ -19,7 +19,7 @@ func seedAdminAudit(ctx context.Context, pool *pgxpool.Pool, _ *Result) error { {uuid.MustParse("00000000-0000-0000-0000-0000000000f1"), "user.banned", "user", UserViewerID, `{"reason":"seed example"}`}, {uuid.MustParse("00000000-0000-0000-0000-0000000000f2"), "user.unbanned", "user", UserViewerID, `{"reason":"seed example"}`}, {uuid.MustParse("00000000-0000-0000-0000-0000000000f3"), "worker.activated", "worker", WorkerFreeID, `{}`}, - {uuid.MustParse("00000000-0000-0000-0000-0000000000f4"), "plan.created", "plan", PlanEnterpriseID, `{"name":"Enterprise"}`}, + {uuid.MustParse("00000000-0000-0000-0000-0000000000f4"), "plan.created", "plan", PlanEnterpriseID, `{"name":"Business"}`}, } for _, e := range entries { _, err := pool.Exec(ctx, ` diff --git a/internal/seed/plans.go b/internal/seed/plans.go index e29ec7d3..4d2e18f5 100644 --- a/internal/seed/plans.go +++ b/internal/seed/plans.go @@ -65,38 +65,41 @@ func seedPlans(ctx context.Context, pool *pgxpool.Pool, r *Result) error { maxTeamMembers: intPtr(1), maxEmailAccounts: intPtr(2), monthlyCredits: 50, }, + // Paid plans mirror the pricing page: mailboxes are unlimited on every + // one (max_email_accounts nil, the fair-use allowance derives from the + // daily sends), and the daily sends are the marketed numbers. { - id: PlanStarterID, name: "Starter", maxContacts: 1_000, dailyEmails: 100, - ai: false, accountLimit: 3, price: 29, discounted: 29, + id: PlanStarterID, name: "Starter", maxContacts: 1_000, dailyEmails: 150, + ai: false, accountLimit: 0, price: 29, discounted: 29, duration: DurationMonthID, savings: 0, public: true, - dedicatedWorkers: 0, dailyCampaignLimit: intPtr(100), + dedicatedWorkers: 0, dailyCampaignLimit: intPtr(150), maxCampaigns: intPtr(5), maxActiveCampaigns: intPtr(2), - maxTeamMembers: intPtr(2), maxEmailAccounts: intPtr(3), + maxTeamMembers: intPtr(2), maxEmailAccounts: nil, monthlyCredits: 250, }, { - id: PlanProMonthlyID, name: "Pro", maxContacts: 25_000, dailyEmails: 1_000, - ai: true, accountLimit: 20, price: 99, discounted: 99, + id: PlanProMonthlyID, name: "Grow", maxContacts: 25_000, dailyEmails: 3_000, + ai: true, accountLimit: 0, price: 89, discounted: 89, duration: DurationMonthID, savings: 0, public: true, - dedicatedWorkers: 1, dailyCampaignLimit: intPtr(1_000), + dedicatedWorkers: 0, dailyCampaignLimit: intPtr(3_000), maxCampaigns: intPtr(50), maxActiveCampaigns: intPtr(20), - maxTeamMembers: intPtr(10), maxEmailAccounts: intPtr(20), + maxTeamMembers: intPtr(10), maxEmailAccounts: nil, monthlyCredits: 2_000, }, { - id: PlanProYearlyID, name: "Pro (Annual)", maxContacts: 25_000, dailyEmails: 1_000, - ai: true, accountLimit: 20, price: 1188, discounted: 990, - duration: DurationYearID, savings: 17, public: true, - dedicatedWorkers: 1, dailyCampaignLimit: intPtr(1_000), + id: PlanProYearlyID, name: "Grow (Annual)", maxContacts: 25_000, dailyEmails: 3_000, + ai: true, accountLimit: 0, price: 1068, discounted: 852, + duration: DurationYearID, savings: 20, public: true, + dedicatedWorkers: 0, dailyCampaignLimit: intPtr(3_000), maxCampaigns: intPtr(50), maxActiveCampaigns: intPtr(20), - maxTeamMembers: intPtr(10), maxEmailAccounts: intPtr(20), + maxTeamMembers: intPtr(10), maxEmailAccounts: nil, monthlyCredits: 2_000, }, { - id: PlanEnterpriseID, name: "Enterprise", maxContacts: 1_000_000, dailyEmails: 10_000, - ai: true, accountLimit: 500, price: 0, discounted: 0, - duration: DurationMonthID, savings: 0, public: false, - dedicatedWorkers: 3, dailyCampaignLimit: intPtr(10_000), + id: PlanEnterpriseID, name: "Business", maxContacts: 1_000_000, dailyEmails: 15_000, + ai: true, accountLimit: 0, price: 329, discounted: 329, + duration: DurationMonthID, savings: 0, public: true, + dedicatedWorkers: 1, dailyCampaignLimit: intPtr(15_000), maxCampaigns: nil, maxActiveCampaigns: nil, maxTeamMembers: nil, maxEmailAccounts: nil, monthlyCredits: 25_000, diff --git a/internal/tasks/auto_pause_reason_test.go b/internal/tasks/auto_pause_reason_test.go index 5d78e32a..6e63a461 100644 --- a/internal/tasks/auto_pause_reason_test.go +++ b/internal/tasks/auto_pause_reason_test.go @@ -25,7 +25,7 @@ func TestAutoPauseReason(t *testing.T) { { name: "no eligible mailbox", err: scheduler.ErrNoEligibleMailbox, - want: "Campaign auto-paused: every mailbox is outside its sending window or over its daily budget", + want: "Campaign auto-paused: no mailbox can send under its current sending settings (check each mailbox's sending behaviour profile and timezone)", }, { name: "generic no accounts", diff --git a/internal/tasks/campaign_daily_budget_live_test.go b/internal/tasks/campaign_daily_budget_live_test.go new file mode 100644 index 00000000..feef821b --- /dev/null +++ b/internal/tasks/campaign_daily_budget_live_test.go @@ -0,0 +1,133 @@ +package tasks + +import ( + "context" + "testing" +) + +// End-to-end checks for issue #306 over the real campaign tick: the chain's +// own wake-ups (a deferral, a pause) must not spend the mailbox's daily +// budget, and a campaign whose mailboxes are all at their cap waits for +// tomorrow instead of pausing. Same harness and env var as +// campaign_send_live_test.go. + +// setCaps sets the per-mailbox cap on both the mailbox and the campaign, so +// the effective cap is exactly n. +func (f *sendFixture) setCaps(t *testing.T, n int) { + t.Helper() + ctx := context.Background() + if _, err := f.pool.Exec(ctx, `UPDATE email_accounts SET campaign_limit = $2 WHERE id = $1`, f.mailbox, n); err != nil { + t.Fatalf("set mailbox cap: %v", err) + } + if _, err := f.pool.Exec(ctx, `UPDATE campaigns SET daily_limit = $2 WHERE id = $1`, f.campaign, n); err != nil { + t.Fatalf("set campaign cap: %v", err) + } +} + +func (f *sendFixture) setMinGap(t *testing.T, seconds int) { + t.Helper() + if _, err := f.pool.Exec(context.Background(), `UPDATE email_accounts SET min_wait_time = $2 WHERE id = $1`, f.mailbox, seconds); err != nil { + t.Fatalf("set min gap: %v", err) + } +} + +func (f *sendFixture) campaignStatus(t *testing.T) string { + t.Helper() + var status string + if err := f.pool.QueryRow(context.Background(), `SELECT status FROM campaigns WHERE id = $1`, f.campaign).Scan(&status); err != nil { + t.Fatalf("campaign status: %v", err) + } + return status +} + +func (f *sendFixture) countLogs(t *testing.T, eventType string) int { + t.Helper() + var n int + if err := f.pool.QueryRow(context.Background(), `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id = $1 AND event_type = $2`, + f.campaign, eventType).Scan(&n); err != nil { + t.Fatalf("count logs: %v", err) + } + return n +} + +// TestLiveDeferredTicksDoNotSpendTheDailyBudget: one send, then three ticks +// that defer on the mailbox's min-gap, then the second lead must still go out. +// Before the fix the three deferrals were three sends against a cap of two, +// and the fourth tick paused the campaign with a lead never emailed. +func TestLiveDeferredTicksDoNotSpendTheDailyBudget(t *testing.T) { + f := newSendFixture(t) + f.setCaps(t, 2) + + f.tick(t) + if f.sender.count() != 1 { + t.Fatalf("first tick dispatched %d sends, want 1", f.sender.count()) + } + + // A 600s gap after that send: every tick inside it defers without sending. + f.setMinGap(t, 600) + for i := 0; i < 3; i++ { + f.tick(t) + } + if f.sender.count() != 1 { + t.Fatalf("deferred ticks dispatched sends: total %d, want 1", f.sender.count()) + } + if status := f.campaignStatus(t); status != "active" { + t.Fatalf("campaign is %q after three deferrals, want active", status) + } + sent, err := f.svc.taskRepo.CountCampaignEmailsSentToday(context.Background(), f.mailbox) + if err != nil { + t.Fatal(err) + } + if sent != 1 { + t.Fatalf("the mailbox is charged %d sends today, want 1 (the deferrals were counted, issue #306)", sent) + } + + // Gap lifted: the second lead is still within budget and goes out. + f.setMinGap(t, 0) + f.tick(t) + if f.sender.count() != 2 { + t.Fatalf("the second lead was not sent after the deferrals (total %d); campaign is %q", f.sender.count(), f.campaignStatus(t)) + } + if row := f.progressFor(t, f.leadB); row == nil || row.sentAt == nil { + t.Fatalf("lead B was not served: %+v", row) + } +} + +// TestLiveCampaignAtDailyCapWaitsForTomorrow: with the cap spent the campaign +// stays active with a parked wake-up, and says why once. +func TestLiveCampaignAtDailyCapWaitsForTomorrow(t *testing.T) { + f := newSendFixture(t) + f.setCaps(t, 1) + ctx := context.Background() + + f.tick(t) + if f.sender.count() != 1 { + t.Fatalf("first tick dispatched %d sends, want 1", f.sender.count()) + } + + for i := 0; i < 2; i++ { + f.tick(t) + if status := f.campaignStatus(t); status != "active" { + t.Fatalf("tick %d at the daily cap left the campaign %q, want active (it used to be paused_no_accounts)", i+2, status) + } + } + if f.sender.count() != 1 { + t.Fatalf("ticks at the cap dispatched sends: total %d, want 1", f.sender.count()) + } + + // The chain is parked, not dropped. + var pending int + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM tasks t JOIN campaign_tasks ct ON ct.task_id = t.id + WHERE ct.campaign_id = $1 AND t.status = 'pending'`, f.campaign).Scan(&pending); err != nil { + t.Fatal(err) + } + if pending != 1 { + t.Fatalf("%d pending wake-ups after the cap was reached, want 1", pending) + } + if n := f.countLogs(t, "daily_cap_reached"); n != 1 { + t.Fatalf("daily_cap_reached logged %d times over two ticks, want once per day", n) + } + if n := f.countLogs(t, "auto_paused"); n != 0 { + t.Fatalf("the campaign was auto-paused %d time(s) at its daily cap", n) + } +} diff --git a/internal/tasks/campaign_reconciler.go b/internal/tasks/campaign_reconciler.go index 2ea610bb..3859c238 100644 --- a/internal/tasks/campaign_reconciler.go +++ b/internal/tasks/campaign_reconciler.go @@ -76,6 +76,7 @@ func (s *tasksService) ReconcileCampaignSchedules(ctx context.Context, limit int log.Warn().Err(err).Str("campaign_id", id.String()).Msg("campaign reconcile: re-seed failed") continue } + s.clearIdle(ctx, campaign) seeded++ case errors.Is(cerr, scheduler.ErrNoEmailAccounts): // No mailbox to send from: pause rather than spin every pass, and @@ -89,6 +90,12 @@ func (s *tasksService) ReconcileCampaignSchedules(ctx context.Context, limit int s.pauseUndeliverable(ctx, id, uuid.Nil, n) continue } + // A continuous campaign sits here idle by design; every pass + // re-checks it so a lead the wake path missed is picked up. + if campaign.Continuous { + s.idleCampaign(ctx, campaign, uuid.Nil) + continue + } } s.campaignRepo.UpdateStatus(ctx, id, "completed") default: diff --git a/internal/tasks/campaign_task.go b/internal/tasks/campaign_task.go index 63530065..077684f0 100644 --- a/internal/tasks/campaign_task.go +++ b/internal/tasks/campaign_task.go @@ -204,9 +204,13 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { } if errors.Is(err, scheduler.ErrCampaignDeferred) { // A valid contact exists but no eligible mailbox right now (ESP-strict - // has no same-provider mailbox, or the daily new-lead cap is reached). + // has no same-provider mailbox, the daily new-lead cap is reached, or + // every mailbox has spent its daily budget or is outside its hours). // Reschedule at the deferred slot WITHOUT sending and WITHOUT touching // progress / daily counters / rotation — mirrors the daily-limit path. + // This task completes without a send, and completing it must not + // spend the mailbox's budget either (issue #306): the budget counts + // reserved sends, never bare wake-ups. // Capped: the next-due moment can be days out, and until this chain // wakes nothing re-reads the campaign, so leads imported meanwhile // would sit queued until then. @@ -214,6 +218,7 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { if cerr := s.createCampaignTask(ctx, campaign.ID, accountID, scheduledNext); cerr != nil { log.Warn().Err(cerr).Str("campaign_id", campaign.ID.String()).Str("task_id", taskID.String()).Msg("Failed to schedule deferred campaign task") } + s.clearIdle(ctx, campaign) s.taskRepo.UpdateTaskStatus(ctx, taskID, "completed") executionStatus = "completed" return nil @@ -238,6 +243,13 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { } reason = fmt.Sprintf("%s (%d lead(s) skipped: address verification refused them)", reason, n) } + // A continuous campaign out of leads is waiting, not finished + // (issue #336). Only its end date ends it. + if errors.Is(err, scheduler.ErrCampaignCompleted) && campaign.Continuous { + s.idleCampaign(ctx, campaign, taskID) + executionStatus = "completed" + return nil + } s.campaignRepo.UpdateStatus(ctx, campaign.ID, "completed") if s.campaignLogRepo != nil { s.campaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ @@ -295,6 +307,8 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { return errx.InternalError() } + s.clearIdle(ctx, campaign) + // STEP 7: Load contact and sequence contact, xerr := s.contactRepo.GetByID(ctx, nextPair.ContactID) if xerr != nil { @@ -413,23 +427,10 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { return nil } - // Load campaign attachments (campaign-wide; metadata only — the worker - // fetches the bytes from object storage by S3 key at send time). - var attachmentRefs []models.AttachmentRef - if s.attachmentRepo != nil { - atts, attErr := s.attachmentRepo.ListByCampaign(ctx, campaign.ID) - if attErr != nil { - log.Warn().Err(attErr).Str("campaign_id", campaign.ID.String()).Str("task_id", taskID.String()).Msg("Failed to load campaign attachments") - } else { - for _, a := range atts { - attachmentRefs = append(attachmentRefs, models.AttachmentRef{ - S3Key: a.S3Key, - Filename: a.Filename, - MimeType: a.MimeType, - }) - } - } - } + // Load this step's attachments (campaign-wide files plus the step's own; + // metadata only — the worker fetches the bytes from object storage by S3 + // key at send time). + attachmentRefs := s.campaignAttachmentRefs(ctx, campaign.ID, sequence.ID) // STEP 7.5: Update campaign task with contact_id and sequence_id for tracking // This allows the tracking consumer to find the correct contact/sequence when @@ -457,12 +458,23 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { rawSubject, rawBodyHTML, rawBodyPlain := sequence.Subject, sequence.BodyHTML, sequence.BodyPlain s.resolveFormLinks(ctx, orgID, campaign, contact, &rawSubject, &rawBodyHTML, &rawBodyPlain) + // STEP 9.75: The recipient's opt-out. The signed link (when the instance + // can mint one) backs the List-Unsubscribe header, the link-mode footer + // and any {{.UnsubscribeLink}} the step places by hand; the footer mode + // comes from Settings > Sending unless the campaign overrides it. + optOut := s.resolveOptOut(ctx, orgID, campaign) + var unsubscribeURL string + if s.unsubLinks != nil && s.unsubLinks.Enabled() { + unsubscribeURL = s.unsubLinks.URL(orgID, campaign.ID, contact.ID, time.Now()) + } + extra := map[string]string{UnsubscribeLinkVar: unsubscribeURL} + // STEP 10: Render email template with contact variables, then expand any // {a|b|c} spintax per-recipient (only real |-groups; literal braces/CSS are // left intact) so each send varies for deliverability. - subject := expandSpintax(RenderTemplate(rawSubject, *contact)) - bodyHTML := expandSpintax(RenderTemplate(rawBodyHTML, *contact)) - bodyPlain := expandSpintax(RenderTemplate(rawBodyPlain, *contact)) + subject := expandSpintax(RenderTemplateWith(rawSubject, *contact, extra)) + bodyHTML := expandSpintax(RenderTemplateWith(rawBodyHTML, *contact, extra)) + bodyPlain := expandSpintax(RenderTemplateWith(rawBodyPlain, *contact, extra)) // If no plain text provided, extract from HTML if bodyPlain == "" && bodyHTML != "" { @@ -475,9 +487,16 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { return sxerr } if selection != nil { - subject = selection.Subject - bodyHTML = selection.BodyHTML - bodyPlain = selection.BodyPlain + // A chosen variant is stored template text, so it goes through the + // same render as the step's own copy; the control arm comes back + // already rendered, for which this pass is a no-op. + subject = expandSpintax(RenderTemplateWith(selection.Subject, *contact, extra)) + bodyHTML = expandSpintax(RenderTemplateWith(selection.BodyHTML, *contact, extra)) + bodyPlain = expandSpintax(RenderTemplateWith(selection.BodyPlain, *contact, extra)) + // A variant may carry HTML only; keep the plain-text alternative. + if bodyPlain == "" && bodyHTML != "" { + bodyPlain = ExtractPlainTextFromHTML(bodyHTML) + } } } @@ -512,6 +531,16 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { } } + // STEP 10.6: A plain-text campaign ships no HTML part at all. Tracking + // below only rewrites HTML, so dropping it here is what makes the + // setting's "disables tracking" promise true. + if campaign.TextOnly { + if bodyPlain == "" && bodyHTML != "" { + bodyPlain = ExtractPlainTextFromHTML(bodyHTML) + } + bodyHTML = "" + } + // STEP 10.75: Score the copy the recipient will actually receive, after // merge fields, spintax, A/B and AI blocks have resolved. Advisory: it // warns once per step and never blocks or delays the send. @@ -537,18 +566,30 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { bodyHTML = AddOpenTrackingPixel(bodyHTML, taskID, trackingDomain) } - if campaign.LinkTracking && bodyHTML != "" { - wrapped, links := WrapLinksForTracking(bodyHTML, taskID, campaign.ID, trackingDomain) + if bodyHTML != "" && (campaign.LinkTracking || campaign.UTMTracking) { + linkOpts := LinkTracking{ + TaskID: taskID, + CampaignID: campaign.ID, + TrackingDomain: trackingDomain, + Wrap: campaign.LinkTracking, + UTM: CampaignUTM(campaign), + } + tracked, links := TrackLinks(bodyHTML, linkOpts) if len(links) == 0 { - bodyHTML = wrapped + bodyHTML = tracked } else if err := s.trackedLinkRepo.CreateBatch(ctx, links); err != nil { // Tracking is a nicety: ship the original working links rather - // than tickets that would 404 at the tracking service. + // than tickets that would 404 at the tracking service. UTM tags + // need no ticket, so they still go on. log.Warn().Err(err).Str("campaign_id", campaign.ID.String()).Str("task_id", taskID.String()).Msg("Failed to store tracked links; sending untracked") + bodyHTML, _ = TrackLinks(bodyHTML, LinkTracking{TrackingDomain: linkOpts.TrackingDomain, UTM: linkOpts.UTM}) } else { - bodyHTML = wrapped + bodyHTML = tracked } } + if campaign.UTMTracking && bodyPlain != "" { + bodyPlain = TagPlainTextLinks(bodyPlain, CampaignUTM(campaign), trackingDomain) + } // STEP 12: Add signature if account.SignatureSync { @@ -560,6 +601,10 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { } } + // STEP 12.5: Opt-out footer, after the signature and after click tracking + // so the link is never rewritten into a tracked ticket. + bodyHTML, bodyPlain = appendOptOut(bodyHTML, bodyPlain, optOut, unsubscribeURL) + // STEP 13: Warm the organization DEK so the publisher's encrypt pass (the // one whose ciphertext is actually sent) fails fast here if KMS is down. if account.OrganizationID == nil { @@ -575,8 +620,10 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { messageID := generateMessageID(account.Email) // STEP 15: Build tracking info (worker receives the already-resolved host). + // A plain-text send carries none: there is no HTML for a pixel or a + // wrapped link to live in. var tracking *models.TrackingInfo - if campaign.OpenTracking || campaign.LinkTracking { + if !campaign.TextOnly && (campaign.OpenTracking || campaign.LinkTracking) { tracking = &models.TrackingInfo{ OpenTracking: campaign.OpenTracking, LinkTracking: campaign.LinkTracking, @@ -584,11 +631,12 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { } } - // STEP 15.5: Generate List-Unsubscribe URL if enabled - var unsubscribeURL string + // STEP 15.5: The List-Unsubscribe header carries the same signed link. + // Off when the campaign disabled it, or when no link could be minted: a + // header pointing nowhere is worse than none. + headerURL := "" if campaign.UnsubscribeHeader { - unsubscribeURL = fmt.Sprintf("https://%s/unsubscribe?cid=%s&rid=%s", - config.Domain, campaign.ID.String(), contact.ID.String()) + headerURL = unsubscribeURL } // STEP 15.9: Reserve the send BEFORE it goes on the bus. Once the command is @@ -637,7 +685,7 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { MessageID: messageID, IsWarmup: false, Tracking: tracking, - UnsubscribeURL: unsubscribeURL, + UnsubscribeURL: headerURL, Attachments: attachmentRefs, } @@ -861,7 +909,7 @@ func autoPauseReason(err error) string { case errors.Is(err, scheduler.ErrDomainAuthFailing): return "Campaign auto-paused: every mailbox is sending from a domain that fails SPF or DMARC authentication" case errors.Is(err, scheduler.ErrNoEligibleMailbox): - return "Campaign auto-paused: every mailbox is outside its sending window or over its daily budget" + return "Campaign auto-paused: no mailbox can send under its current sending settings (check each mailbox's sending behaviour profile and timezone)" default: return "Campaign auto-paused: no active email accounts available" } @@ -898,6 +946,56 @@ func (s *tasksService) pauseUndeliverable(ctx context.Context, campaignID, taskI } } +// CampaignIdleEventType is the activity log entry written when a continuous +// campaign runs out of leads and waits; CampaignIdleMessage is its text. +const ( + CampaignIdleEventType = "idle" + CampaignIdleMessage = "Waiting for new leads: every lead has finished the sequence. The campaign stays active and sends to leads as they arrive." +) + +// idleCampaign parks a continuous campaign that has nothing left to send. It +// stays active with no chain: a lead add wakes it, and the reconciler re-checks +// it every pass. Logged and broadcast once per wait, not once per pass. +func (s *tasksService) idleCampaign(ctx context.Context, campaign *models.Campaign, taskID uuid.UUID) { + if taskID != uuid.Nil { + s.taskRepo.UpdateTaskStatus(ctx, taskID, "completed") + } + transitioned, err := s.campaignRepo.MarkIdle(ctx, campaign.ID) + if err != nil { + log.Warn().Err(err).Str("campaign_id", campaign.ID.String()).Msg("could not mark the campaign idle") + return + } + if !transitioned { + return + } + if s.campaignLogRepo != nil { + s.campaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ + CampaignID: campaign.ID, + EventType: CampaignIdleEventType, + Message: CampaignIdleMessage, + }) + } + if s.streamingPublisher != nil { + s.streamingPublisher.PublishCampaignEvent(ctx, &pubsub.CampaignEvent{ + BaseEvent: pubsub.BaseEvent{EventType: pubsub.EventCampaignIdle, UserID: campaign.UserID}, + OrgID: campaignOrgID(campaign), + CampaignID: campaign.ID.String(), + Name: campaign.Name, + Status: "active", + }) + } +} + +// clearIdle ends an idle wait once the campaign has something to send again. +func (s *tasksService) clearIdle(ctx context.Context, campaign *models.Campaign) { + if campaign.IdleSince == nil { + return + } + if err := s.campaignRepo.ClearIdle(ctx, campaign.ID); err != nil { + log.Warn().Err(err).Str("campaign_id", campaign.ID.String()).Msg("could not clear the campaign's idle mark") + } +} + // autoPauseCampaign parks a campaign that has nothing it can send from. The // reason is carried through to the activity log because "paused_no_accounts" // covers several very different fixes (connect a mailbox, widen a sending @@ -1283,3 +1381,16 @@ func (s *tasksService) recordSchedulerFailure(ctx context.Context, campaignID uu Metadata: meta, }) } + +// resolveOptOut is the effective in-body opt-out for a campaign: the +// workspace setting with the campaign's own mode applied. A settings read +// failure falls back to the defaults rather than sending without an opt-out. +func (s *tasksService) resolveOptOut(ctx context.Context, orgID uuid.UUID, campaign *models.Campaign) models.UnsubscribeSettings { + base := models.DefaultAdvancedOutreachSettings().Unsubscribe + if s.advanced != nil { + if settings, xerr := s.advanced.GetOrganizationSettings(ctx, orgID); xerr == nil && settings != nil { + base = settings.Unsubscribe + } + } + return base.Effective(campaign.UnsubscribeMode) +} diff --git a/internal/tasks/email_sender.go b/internal/tasks/email_sender.go index 0a250a61..1d362c01 100644 --- a/internal/tasks/email_sender.go +++ b/internal/tasks/email_sender.go @@ -132,6 +132,9 @@ func (s *emailSender) Send(ctx context.Context, taskID uuid.UUID, msg EmailMessa WarmupToken: msg.WarmupToken, UnsubscribeURL: msg.UnsubscribeURL, Attachments: msg.Attachments, + // The name as saved now, so a rename applies to the very next send + // instead of waiting for the worker's cached identity to be rebuilt. + FromName: strings.TrimSpace(account.Name), } // Publish send email event to worker diff --git a/internal/tasks/links.go b/internal/tasks/links.go new file mode 100644 index 00000000..10b5bbb7 --- /dev/null +++ b/internal/tasks/links.go @@ -0,0 +1,302 @@ +package tasks + +import ( + "fmt" + "html" + "net/url" + "regexp" + "strings" + "unicode" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// UTMParams are the campaign-level UTM values stamped on every link. +// utm_content is not here because it is per link: the link's own text. +type UTMParams struct { + Source string + Medium string + Campaign string +} + +// LinkTracking describes what the send path does to the links of one email: +// rewrite them to click tickets, tag them with UTM parameters, or both. +type LinkTracking struct { + TaskID uuid.UUID + CampaignID uuid.UUID + // TrackingDomain hosts the click tickets. Empty means no ticket can be + // resolved, so links are never wrapped even when Wrap is set. + TrackingDomain string + Wrap bool + // UTM, when set, tags every link that does not already carry the + // parameter. Hand-written UTM values always win. + UTM *UTMParams +} + +// Defaults for automatic UTM tagging when the campaign leaves a value empty. +const ( + utmDefaultSource = "warmbly" + utmDefaultMedium = "email" + utmMaxLabelRunes = 120 + utmMaxSlugRunes = 64 +) + +// CampaignUTM resolves the campaign's UTM settings to the values the send +// path stamps, filling defaults for anything left empty. nil when the +// campaign does not tag links. +func CampaignUTM(c *models.Campaign) *UTMParams { + if c == nil || !c.UTMTracking { + return nil + } + p := &UTMParams{ + Source: strings.TrimSpace(c.UTMSource), + Medium: strings.TrimSpace(c.UTMMedium), + Campaign: strings.TrimSpace(c.UTMCampaign), + } + if p.Source == "" { + p.Source = utmDefaultSource + } + if p.Medium == "" { + p.Medium = utmDefaultMedium + } + if p.Campaign == "" { + p.Campaign = utmSlug(c.Name) + } + if p.Campaign == "" { + p.Campaign = "campaign" + } + return p +} + +// anchorTag matches one ..., capturing the href value +// (double-quoted, single-quoted or bare) and the inner HTML the label is +// read from. The attribute must follow whitespace so data-href and the like +// never pass for it. Only anchors are touched: a in the head is +// a stylesheet, and redirecting it through a click ticket breaks it. +var anchorTag = regexp.MustCompile(`(?is)]*?)\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>` + "`" + `]+))([^>]*)>(.*?)`) + +var ( + htmlTag = regexp.MustCompile(`(?s)<[^>]*>`) + imgAlt = regexp.MustCompile(`(?is)]*\balt\s*=\s*(?:"([^"]*)"|'([^']*)')`) + whitespace = regexp.MustCompile(`\s+`) +) + +// TrackLinks rewrites the anchors of an HTML body per opts and returns the +// click tickets it minted (nil when nothing was wrapped). A wrapped link +// carries only the opaque ticket (https:///c/); the destination, +// UTM parameters included, lives in the returned row. The caller MUST persist +// the rows before using the rewritten body, and fall back on failure, so an +// email can never ship dead tickets. +// +// Skipped, and left exactly as written: anchors, mailto: and tel: links, +// data: and javascript: URLs, anything that is not http(s), and links that +// already point at the tracking host. +func TrackLinks(htmlBody string, opts LinkTracking) (string, []repository.TrackedLink) { + trackingDomain := config.NormalizeTrackingHost(opts.TrackingDomain) + wrap := opts.Wrap && trackingDomain != "" + if !wrap && opts.UTM == nil { + return htmlBody, nil + } + + var links []repository.TrackedLink + position := 0 + + result := anchorTag.ReplaceAllStringFunc(htmlBody, func(match string) string { + m := anchorTag.FindStringSubmatch(match) + before, quoted, single, bare, after, inner := m[1], m[2], m[3], m[4], m[5], m[6] + rawHref := quoted + if rawHref == "" { + rawHref = single + } + if rawHref == "" { + rawHref = bare + } + dest := strings.TrimSpace(html.UnescapeString(rawHref)) + if !trackableURL(dest, trackingDomain) { + return match + } + position++ + label := linkLabel(inner) + + if opts.UTM != nil { + dest = withUTM(dest, opts.UTM, utmContent(label, position)) + } + + href := html.EscapeString(dest) + if wrap { + id := uuid.New() + links = append(links, repository.TrackedLink{ + ID: id, + TaskID: opts.TaskID, + CampaignID: opts.CampaignID, + Destination: dest, + Label: label, + }) + href = config.TrackingURL(trackingDomain, "/c/"+id.String()) + } + return `` + inner + `` + }) + + return result, links +} + +// WrapLinksForTracking rewrites every external link to a click ticket and +// returns the minted rows. Kept as the wrap-only form of TrackLinks. +func WrapLinksForTracking(htmlBody string, taskID, campaignID uuid.UUID, trackingDomain string) (string, []repository.TrackedLink) { + return TrackLinks(htmlBody, LinkTracking{ + TaskID: taskID, + CampaignID: campaignID, + TrackingDomain: trackingDomain, + Wrap: true, + }) +} + +// trackableURL reports whether a destination is one a click ticket can +// redirect to and a UTM tag makes sense on. A link already pointing at the +// tracking host (its host, not merely a URL mentioning it) is left alone. +func trackableURL(dest, trackingDomain string) bool { + lower := strings.ToLower(dest) + if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") { + return false + } + // An unsubscribe link is never a click to count or a redirect to bounce + // through, and a UTM tag on it would only clutter the opt-out page URL. + if strings.Contains(lower, unsubscribePathMarker) { + return false + } + if trackingDomain == "" { + return true + } + // A destination the URL parser rejects cannot be redirected to, so it + // is left as written rather than turned into a dead ticket. + u, err := url.Parse(dest) + if err != nil || u.Hostname() == "" { + return false + } + return config.NormalizeTrackingHost(u.Host) != trackingDomain +} + +// bareURL matches an http(s) URL in plain text, stopping at whitespace and +// the characters that close it in prose. +var bareURL = regexp.MustCompile(`(?i)https?://[^\s<>"'` + "`" + `]+`) + +// TagPlainTextLinks appends UTM parameters to every bare URL of a plain-text +// body. There is no anchor text, so utm_content numbers the links in order. +// Trailing punctuation that belongs to the sentence stays outside the URL. +func TagPlainTextLinks(body string, utm *UTMParams, trackingDomain string) string { + if utm == nil || body == "" { + return body + } + trackingDomain = config.NormalizeTrackingHost(trackingDomain) + position := 0 + return bareURL.ReplaceAllStringFunc(body, func(match string) string { + dest := strings.TrimRight(match, ".,;:!?)]}") + trail := match[len(dest):] + if !trackableURL(dest, trackingDomain) { + return match + } + position++ + return withUTM(dest, utm, utmContent("", position)) + trail + }) +} + +// linkLabel is the anchor's visible text: tags stripped, entities decoded, +// whitespace collapsed. An image link falls back to the image's alt text. +func linkLabel(inner string) string { + text := html.UnescapeString(htmlTag.ReplaceAllString(inner, " ")) + text = strings.TrimSpace(whitespace.ReplaceAllString(text, " ")) + if text == "" { + if m := imgAlt.FindStringSubmatch(inner); m != nil { + alt := m[1] + if alt == "" { + alt = m[2] + } + text = strings.TrimSpace(whitespace.ReplaceAllString(html.UnescapeString(alt), " ")) + } + } + return truncateRunes(text, utmMaxLabelRunes) +} + +// utmContent names the link inside the email: its text as a slug, or its +// ordinal when it has none (a bare image or icon). +func utmContent(label string, position int) string { + if s := utmSlug(label); s != "" { + return s + } + return fmt.Sprintf("link_%d", position) +} + +// utmSlug lowercases a label and joins its words with underscores +// ("See our Pricing!" -> "see_our_pricing"), the shape analytics tools +// expect in a utm value. +func utmSlug(s string) string { + var b strings.Builder + pendingSep := false + for _, r := range strings.ToLower(strings.TrimSpace(s)) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + if pendingSep && b.Len() > 0 { + b.WriteByte('_') + } + pendingSep = false + b.WriteRune(r) + continue + } + pendingSep = true + } + return truncateRunes(b.String(), utmMaxSlugRunes) +} + +// withUTM appends the UTM parameters the destination does not already carry, +// keeping the existing query and fragment exactly as written. The URL is +// never re-encoded: a customer's signed or oddly-encoded link stays intact. +func withUTM(dest string, p *UTMParams, content string) string { + u, err := url.Parse(dest) + if err != nil { + return dest + } + existing := u.Query() + pairs := [][2]string{ + {"utm_source", p.Source}, + {"utm_medium", p.Medium}, + {"utm_campaign", p.Campaign}, + {"utm_content", content}, + } + var add []string + for _, kv := range pairs { + if kv[1] == "" || existing.Has(kv[0]) { + continue + } + add = append(add, kv[0]+"="+url.QueryEscape(kv[1])) + } + if len(add) == 0 { + return dest + } + + base, fragment, hasFragment := strings.Cut(dest, "#") + sep := "?" + if strings.Contains(base, "?") { + sep = "&" + if strings.HasSuffix(base, "?") || strings.HasSuffix(base, "&") { + sep = "" + } + } + out := base + sep + strings.Join(add, "&") + if hasFragment { + out += "#" + fragment + } + return out +} + +func truncateRunes(s string, n int) string { + if n <= 0 { + return "" + } + r := []rune(s) + if len(r) <= n { + return s + } + return strings.TrimRight(string(r[:n]), "_ ") +} diff --git a/internal/tasks/optout.go b/internal/tasks/optout.go new file mode 100644 index 00000000..a329c585 --- /dev/null +++ b/internal/tasks/optout.go @@ -0,0 +1,63 @@ +package tasks + +import ( + "html" + "strings" + + "github.com/warmbly/warmbly/internal/models" +) + +// UnsubscribeLinkVar is the template variable a step can place by hand +// ({{.UnsubscribeLink}}); it resolves to the recipient's own signed link. +const UnsubscribeLinkVar = "UnsubscribeLink" + +// unsubscribePathMarker is the path segment every minted link carries. Click +// tracking leaves such links alone so an opt-out is never counted as a click +// or bounced through a redirect. +const unsubscribePathMarker = "/unsubscribe/" + +// optOutFooter renders the in-body opt-out for one recipient, as HTML and as +// plain text, or empty strings when the effective mode is off. Link mode with +// no link available (the instance has no public API URL) falls back to the +// text line so a recipient is never left without a way out. +func optOutFooter(settings models.UnsubscribeSettings, linkURL string) (htmlPart, plainPart string) { + switch settings.Mode { + case models.UnsubscribeModeOff: + return "", "" + case models.UnsubscribeModeLink: + if linkURL == "" { + break + } + intro := strings.TrimSpace(settings.LinkIntro) + text := strings.TrimSpace(settings.LinkText) + htmlPart = `

` + + html.EscapeString(intro) + ` ` + html.EscapeString(text) + `

` + plainPart = intro + " " + text + ": " + linkURL + return htmlPart, strings.TrimSpace(plainPart) + } + line := strings.TrimSpace(settings.Text) + if line == "" { + return "", "" + } + return `

` + html.EscapeString(line) + `

`, line +} + +// appendOptOut adds the footer after everything else (signature included) so +// it sits where a reader expects an opt-out: last. +func appendOptOut(bodyHTML, bodyPlain string, settings models.UnsubscribeSettings, linkURL string) (string, string) { + htmlPart, plainPart := optOutFooter(settings, linkURL) + if htmlPart == "" { + return bodyHTML, bodyPlain + } + if bodyHTML != "" { + if strings.Contains(bodyHTML, "") { + bodyHTML = strings.Replace(bodyHTML, "", htmlPart+"", 1) + } else { + bodyHTML += htmlPart + } + } + if bodyPlain != "" { + bodyPlain += "\n\n" + plainPart + } + return bodyHTML, bodyPlain +} diff --git a/internal/tasks/optout_test.go b/internal/tasks/optout_test.go new file mode 100644 index 00000000..562e6506 --- /dev/null +++ b/internal/tasks/optout_test.go @@ -0,0 +1,43 @@ +package tasks + +import ( + "strings" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestUnsubscribeLinkIsNeverTracked(t *testing.T) { + body := `

Hi pricing and unsubscribe

` + out, links := WrapLinksForTracking(body, uuid.New(), uuid.New(), "t.example.com") + if len(links) != 1 || links[0].Destination != "https://acme.com/pricing" { + t.Fatalf("expected only the pricing link to be ticketed, got %+v", links) + } + if !strings.Contains(out, `href="https://api.example.com/unsubscribe/abc123"`) { + t.Fatalf("unsubscribe link was rewritten: %s", out) + } +} + +func TestOptOutFooter(t *testing.T) { + text := models.UnsubscribeSettings{Mode: models.UnsubscribeModeText, Text: "Reply and I'll stop."} + h, p := appendOptOut("

Hi

", "Hi", text, "") + if !strings.Contains(h, "Reply and I'll stop.") || !strings.HasSuffix(p, "Reply and I'll stop.") { + t.Fatalf("text footer missing: %q / %q", h, p) + } + + link := models.UnsubscribeSettings{Mode: models.UnsubscribeModeLink, Text: "fallback", LinkIntro: "Not interested?", LinkText: "Unsubscribe"} + h, p = appendOptOut("

Hi

", "Hi", link, "https://api.example.com/unsubscribe/tok") + if !strings.Contains(h, `href="https://api.example.com/unsubscribe/tok"`) || !strings.Contains(p, "Unsubscribe: https://api.example.com/unsubscribe/tok") { + t.Fatalf("link footer missing: %q / %q", h, p) + } + // No link to mint: link mode degrades to the text line, never to nothing. + h, _ = appendOptOut("

Hi

", "Hi", link, "") + if !strings.Contains(h, "fallback") { + t.Fatalf("link mode without a link should fall back to text: %q", h) + } + h, p = appendOptOut("

Hi

", "Hi", models.UnsubscribeSettings{Mode: models.UnsubscribeModeOff}, "x") + if h != "

Hi

" || p != "Hi" { + t.Fatalf("off mode changed the body: %q / %q", h, p) + } +} diff --git a/internal/tasks/preview.go b/internal/tasks/preview.go new file mode 100644 index 00000000..e578d26f --- /dev/null +++ b/internal/tasks/preview.go @@ -0,0 +1,130 @@ +package tasks + +import ( + "context" + "strings" + "time" + + "github.com/google/uuid" + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/models" +) + +// EmailPreviewInput is one step's templates plus the context the send path +// would have: the contact to render for, the campaign (opt-out footer, +// attachments, plain-text rule) and the sending mailbox (signature, From). +// Campaign and Account are optional; without them the preview is templates only. +type EmailPreviewInput struct { + Subject string + BodyHTML string + BodyPlain string + Contact models.Contact + Campaign *models.Campaign + Account *models.Email + // SequenceID names the step being previewed, so the attachment list is the + // one that step sends. Zero lists the campaign-wide files only. + SequenceID uuid.UUID +} + +// EmailPreviewFrom is the sender as the recipient will see it. +type EmailPreviewFrom struct { + Name string `json:"name"` + Email string `json:"email"` +} + +// EmailPreviewAttachment is an attachment the send would carry, metadata only. +type EmailPreviewAttachment struct { + ID uuid.UUID `json:"id"` + Filename string `json:"filename"` + Size int64 `json:"size"` + MimeType string `json:"mime_type"` +} + +// EmailPreview is the rendered message with everything the send path adds +// after the template: signature, opt-out footer, sender and attachments. +type EmailPreview struct { + TemplatePreview + From *EmailPreviewFrom `json:"from,omitempty"` + Attachments []EmailPreviewAttachment `json:"attachments,omitempty"` +} + +// PreviewEmail renders a step the way the send path assembles it for one +// contact: template and spintax, then the plain-text rule, the mailbox +// signature and the opt-out footer, in send order. Tracking is left out since +// it only rewrites URLs. The opt-out link names no contact, so it can never +// suppress anyone if clicked. +func (s *tasksService) PreviewEmail(ctx context.Context, orgID uuid.UUID, in EmailPreviewInput) *EmailPreview { + unsubURL := PreviewUnsubscribeLink + var optOut *models.UnsubscribeSettings + textOnly := false + if in.Campaign != nil { + if s.unsubLinks != nil && s.unsubLinks.Enabled() { + unsubURL = s.unsubLinks.URL(orgID, in.Campaign.ID, uuid.Nil, time.Now()) + } + settings := s.resolveOptOut(ctx, orgID, in.Campaign) + optOut = &settings + textOnly = in.Campaign.TextOnly + } + + out := &EmailPreview{TemplatePreview: previewTemplatesWith(in.Subject, in.BodyHTML, in.BodyPlain, in.Contact, unsubURL)} + out.BodyHTML, out.BodyPlain = finishBody(out.BodyHTML, out.BodyPlain, textOnly, in.Account, optOut, unsubURL) + + if in.Account != nil { + out.From = &EmailPreviewFrom{Name: strings.TrimSpace(in.Account.Name), Email: in.Account.Email} + } + if in.Campaign != nil && s.attachmentRepo != nil { + atts, err := s.attachmentRepo.ListForStep(ctx, in.Campaign.ID, in.SequenceID) + if err != nil { + log.Warn().Err(err).Str("campaign_id", in.Campaign.ID.String()).Msg("preview: load campaign attachments failed") + } + for _, a := range atts { + out.Attachments = append(out.Attachments, EmailPreviewAttachment{ID: a.ID, Filename: a.Filename, Size: a.Size, MimeType: a.MimeType}) + } + } + return out +} + +// finishBody applies what the send path adds after rendering, in its order: +// derive the plain part, drop HTML for a plain-text campaign, add the mailbox +// signature, then the opt-out footer (nil settings skip it). Shared by the +// preview and the test send so both show what a recipient gets. +func finishBody(bodyHTML, bodyPlain string, textOnly bool, account *models.Email, optOut *models.UnsubscribeSettings, unsubURL string) (string, string) { + if bodyPlain == "" && bodyHTML != "" { + bodyPlain = ExtractPlainTextFromHTML(bodyHTML) + } + if textOnly { + bodyHTML = "" + } + if account != nil && account.SignatureSync { + if bodyHTML != "" { + bodyHTML = AddSignature(bodyHTML, account.SignatureHTML, true) + } + if bodyPlain != "" { + bodyPlain = AddSignature(bodyPlain, account.SignaturePlain, false) + } + } + if optOut != nil { + bodyHTML, bodyPlain = appendOptOut(bodyHTML, bodyPlain, *optOut, unsubURL) + } + return bodyHTML, bodyPlain +} + +// campaignAttachmentRefs lists the files one step's send carries, as the refs +// the worker resolves from object storage: the campaign-wide files plus the +// ones scoped to that step (uuid.Nil = campaign-wide only). A load failure +// sends without them rather than failing the send, and is logged. +func (s *tasksService) campaignAttachmentRefs(ctx context.Context, campaignID, sequenceID uuid.UUID) []models.AttachmentRef { + if s.attachmentRepo == nil { + return nil + } + atts, err := s.attachmentRepo.ListForStep(ctx, campaignID, sequenceID) + if err != nil { + log.Warn().Err(err).Str("campaign_id", campaignID.String()).Msg("Failed to load campaign attachments") + return nil + } + refs := make([]models.AttachmentRef, 0, len(atts)) + for _, a := range atts { + refs = append(refs, models.AttachmentRef{S3Key: a.S3Key, Filename: a.Filename, MimeType: a.MimeType}) + } + return refs +} diff --git a/internal/tasks/preview_test.go b/internal/tasks/preview_test.go new file mode 100644 index 00000000..ced6bc70 --- /dev/null +++ b/internal/tasks/preview_test.go @@ -0,0 +1,55 @@ +package tasks + +import ( + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/models" +) + +// finishBody is what the preview and the test send share with the campaign +// send: the parts land in send order (body, signature, opt-out) and a +// plain-text campaign loses its HTML part before the signature is added. +func TestFinishBodyMatchesSendOrder(t *testing.T) { + account := &models.Email{SignatureSync: true, SignatureHTML: "

Ana

", SignaturePlain: "Ana"} + optOut := &models.UnsubscribeSettings{Mode: models.UnsubscribeModeText, Text: "Reply stop to opt out."} + + h, p := finishBody("

Hi

", "", false, account, optOut, "") + sig, foot := strings.Index(h, "

Ana

"), strings.Index(h, "Reply stop to opt out.") + if sig < 0 || foot < 0 || sig > foot { + t.Fatalf("html parts out of send order: %q", h) + } + if !strings.HasPrefix(p, "Hi") || strings.Index(p, "Ana") > strings.Index(p, "Reply stop") { + t.Fatalf("plain part not derived from html or out of order: %q", p) + } + + h, p = finishBody("

Hi

", "", true, account, optOut, "") + if h != "" { + t.Fatalf("plain-text campaign kept an html part: %q", h) + } + if !strings.Contains(p, "Ana") || !strings.Contains(p, "Reply stop") { + t.Fatalf("plain-text campaign lost signature or footer: %q", p) + } + + // No mailbox and no campaign: templates only, nothing appended. + h, p = finishBody("

Hi

", "Hi", false, nil, nil, "") + if h != "

Hi

" || p != "Hi" { + t.Fatalf("bare preview was decorated: %q / %q", h, p) + } + + // Signature sync off leaves the body alone even with a signature stored. + off := &models.Email{SignatureSync: false, SignatureHTML: "

Ana

"} + if h, _ = finishBody("

Hi

", "Hi", false, off, nil, ""); h != "

Hi

" { + t.Fatalf("signature added while sync is off: %q", h) + } +} + +func TestPreviewTemplatesWithUsesTheGivenLink(t *testing.T) { + p := previewTemplatesWith("s", "x", "", models.Contact{}, "https://api.example.com/unsubscribe/tok") + if !strings.Contains(p.BodyHTML, "https://api.example.com/unsubscribe/tok") { + t.Fatalf("link variable did not resolve to the given link: %q", p.BodyHTML) + } + if q := PreviewTemplates("s", "{{.UnsubscribeLink}}", "", models.Contact{}); q.BodyHTML != PreviewUnsubscribeLink { + t.Fatalf("default preview link changed: %q", q.BodyHTML) + } +} diff --git a/internal/tasks/service.go b/internal/tasks/service.go index 4d53a63d..5fe05384 100644 --- a/internal/tasks/service.go +++ b/internal/tasks/service.go @@ -10,6 +10,7 @@ import ( "github.com/warmbly/warmbly/internal/app/cipher" "github.com/warmbly/warmbly/internal/app/credits" "github.com/warmbly/warmbly/internal/app/feature" + "github.com/warmbly/warmbly/internal/app/unsublink" warmupapp "github.com/warmbly/warmbly/internal/app/warmup" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/events" @@ -48,8 +49,10 @@ type TasksService interface { HandleEmailTask(task *proto.ProcessTask) *errx.Error HandleUserEmailTask(task *proto.ProcessTask) *errx.Error - // Test email support - SendTestEmail(ctx context.Context, userID string, accountID uuid.UUID, recipient string, campaign *models.Campaign, sequence *models.Sequence) *errx.Error + // Test email and preview support. Both are org-scoped: the mailbox has to + // belong to orgID, and the contact (optional) is resolved by the caller. + SendTestEmail(ctx context.Context, orgID uuid.UUID, accountID uuid.UUID, recipient string, campaign *models.Campaign, sequence *models.Sequence, contact *models.Contact) *errx.Error + PreviewEmail(ctx context.Context, orgID uuid.UUID, in EmailPreviewInput) *EmailPreview GetCampaignSequences(ctx context.Context, campaignID uuid.UUID) ([]models.Sequence, error) // Warmup scheduling lifecycle @@ -79,6 +82,9 @@ type TasksService interface { // stops warmup sends from a domain that has been failing SPF/DMARC past // the operator's grace window. Nil leaves the state observe-only. SetDomainAuthPolicy(p DomainAuthPolicy) + // SetUnsubscribeLinks wires the signer behind per-recipient unsubscribe + // links (the List-Unsubscribe header and the link-mode footer). + SetUnsubscribeLinks(signer *unsublink.Signer) // SetCloudLink wires the self-hosted side of the warmup pool link: a // mailbox the cloud warms gets no local warmup chain. SetCloudLink(r CloudLinkReader) @@ -165,6 +171,16 @@ type tasksService struct { domainAuth DomainAuthPolicy // cloudLink is nil on instances that are not linked to Warmbly Cloud. cloudLink CloudLinkReader + + // unsubLinks mints the signed per-recipient unsubscribe links. Nil or + // disabled means no link can be minted: the List-Unsubscribe header is + // left off and link-mode footers fall back to the text line. + unsubLinks *unsublink.Signer +} + +// SetUnsubscribeLinks wires the unsubscribe link signer. +func (s *tasksService) SetUnsubscribeLinks(signer *unsublink.Signer) { + s.unsubLinks = signer } // DomainAuthPolicy resolves whether the sending-domain authentication gate is diff --git a/internal/tasks/step_attachments_test.go b/internal/tasks/step_attachments_test.go new file mode 100644 index 00000000..7589f4e0 --- /dev/null +++ b/internal/tasks/step_attachments_test.go @@ -0,0 +1,55 @@ +package tasks + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// stubAttachmentRepo records the scope a send asked for and answers with the +// rows a step-scoped query would return. +type stubAttachmentRepo struct { + repository.AttachmentRepository + gotCampaign uuid.UUID + gotSequence uuid.UUID + rows []models.CampaignAttachment +} + +func (s *stubAttachmentRepo) ListForStep(_ context.Context, campaignID, sequenceID uuid.UUID) ([]models.CampaignAttachment, error) { + s.gotCampaign, s.gotSequence = campaignID, sequenceID + return s.rows, nil +} + +// The files a send carries are the step's, not the campaign's: sequence_id was +// stored on upload and honoured by the duplicate path, but the send path listed +// every attachment of the campaign, so a file attached to step 1 rode step 2, +// step 3 and every follow-up after them. +func TestCampaignAttachmentRefsAsksForTheStepsFiles(t *testing.T) { + campaignID, sequenceID := uuid.New(), uuid.New() + repo := &stubAttachmentRepo{rows: []models.CampaignAttachment{ + {S3Key: "k1", Filename: "brief.pdf", MimeType: "application/pdf"}, + }} + s := &tasksService{attachmentRepo: repo} + + refs := s.campaignAttachmentRefs(context.Background(), campaignID, sequenceID) + + if repo.gotCampaign != campaignID || repo.gotSequence != sequenceID { + t.Fatalf("listed (%s, %s), want the campaign and step being sent (%s, %s)", + repo.gotCampaign, repo.gotSequence, campaignID, sequenceID) + } + if len(refs) != 1 || refs[0].S3Key != "k1" || refs[0].Filename != "brief.pdf" { + t.Fatalf("refs = %#v, want the step's one file", refs) + } +} + +// Without an attachment store a send still goes out, carrying nothing. +func TestCampaignAttachmentRefsWithoutAStore(t *testing.T) { + s := &tasksService{} + if refs := s.campaignAttachmentRefs(context.Background(), uuid.New(), uuid.New()); refs != nil { + t.Fatalf("refs = %#v, want none", refs) + } +} diff --git a/internal/tasks/template.go b/internal/tasks/template.go index a52b3cae..7f29fb17 100644 --- a/internal/tasks/template.go +++ b/internal/tasks/template.go @@ -13,7 +13,6 @@ import ( "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/tmplfuncs" "github.com/warmbly/warmbly/internal/pkg/warmpersona" - "github.com/warmbly/warmbly/internal/repository" ) // Conversation represents a warmup conversation for AI generation @@ -162,21 +161,31 @@ func TemplateError(tmpl string) error { // intentionally left untouched here (single-brace {a|b} survives the template // pass) and expanded later in the pipeline where applicable. func RenderTemplate(tmpl string, contact models.Contact) string { + return RenderTemplateWith(tmpl, contact, nil) +} + +// RenderTemplateWith is RenderTemplate with per-send values that are not +// contact fields (today: the recipient's unsubscribe link). They win a name +// collision with a custom field, like the standard fields do. +func RenderTemplateWith(tmpl string, contact models.Contact, extra map[string]string) string { if tmpl == "" { return tmpl } data := buildTemplateData(contact) + for k, v := range extra { + data[k] = v + } prepared := rewriteSpacedFieldRefs(tmpl) t := compiledTemplate(prepared) if t == nil { - return naiveRenderTemplate(tmpl, contact) // known-bad -> legacy path + return naiveRenderTemplate(tmpl, contact, extra) // known-bad -> legacy path } var b strings.Builder if err := t.Execute(&b, data); err != nil { - return naiveRenderTemplate(tmpl, contact) + return naiveRenderTemplate(tmpl, contact, extra) } return b.String() } @@ -185,8 +194,11 @@ func RenderTemplate(tmpl string, contact models.Contact) string { // substitution for the standard fields and every custom field. It is the // graceful fallback when text/template parsing or execution fails, so a body // always renders even for malformed conditional syntax. -func naiveRenderTemplate(tmpl string, contact models.Contact) string { +func naiveRenderTemplate(tmpl string, contact models.Contact, extra map[string]string) string { result := tmpl + for k, v := range extra { + result = strings.ReplaceAll(result, fmt.Sprintf("{{.%s}}", k), v) + } result = strings.ReplaceAll(result, "{{.FirstName}}", contact.FirstName) result = strings.ReplaceAll(result, "{{.LastName}}", contact.LastName) result = strings.ReplaceAll(result, "{{.Email}}", contact.Email) @@ -210,18 +222,32 @@ type TemplatePreview struct { Unresolved []string `json:"unresolved,omitempty"` // literal {{…}} tokens left after render } +// PreviewUnsubscribeLink stands in for the per-recipient link in previews. +const PreviewUnsubscribeLink = "https://example.com/unsubscribe/preview" + // unresolvedToken matches a {{…}} token still present after rendering (i.e. one // that failed to parse and fell through to literal substitution). var unresolvedToken = regexp.MustCompile(`\{\{[^{}]*\}\}`) +// bodyClose matches a closing body tag in any case, since HTML tag names are +// case-insensitive and a pasted document may well carry . +var bodyClose = regexp.MustCompile(`(?i)`) + // PreviewTemplates renders subject/html/plain against contact EXACTLY as the // send path does (template render + spintax), and reports parse errors plus any // tokens that did not resolve. func PreviewTemplates(subject, bodyHTML, bodyPlain string, contact models.Contact) TemplatePreview { + return previewTemplatesWith(subject, bodyHTML, bodyPlain, contact, PreviewUnsubscribeLink) +} + +// previewTemplatesWith is PreviewTemplates with the unsubscribe link the +// {{unsubscribe_link}} variable resolves to. +func previewTemplatesWith(subject, bodyHTML, bodyPlain string, contact models.Contact, unsubscribeURL string) TemplatePreview { + extra := map[string]string{UnsubscribeLinkVar: unsubscribeURL} p := TemplatePreview{ - Subject: expandSpintax(RenderTemplate(subject, contact)), - BodyHTML: expandSpintax(RenderTemplate(bodyHTML, contact)), - BodyPlain: expandSpintax(RenderTemplate(bodyPlain, contact)), + Subject: expandSpintax(RenderTemplateWith(subject, contact, extra)), + BodyHTML: expandSpintax(RenderTemplateWith(bodyHTML, contact, extra)), + BodyPlain: expandSpintax(RenderTemplateWith(bodyPlain, contact, extra)), } for _, f := range []struct{ name, raw string }{{"subject", subject}, {"body", bodyHTML}, {"plain text", bodyPlain}} { if err := TemplateError(f.raw); err != nil { @@ -240,17 +266,25 @@ func PreviewTemplates(subject, bodyHTML, bodyPlain string, contact models.Contac return p } -// AddSignature adds signature to email body +// AddSignature places the mailbox signature under the body. HTML gets its own +// block with a top margin, not

: the breaks stacked against the body's +// own trailing margin and showed as blank lines in Apple Mail and Outlook. func AddSignature(body string, signature string, isHTML bool) string { if signature == "" { return body } - if isHTML { - return body + "

" + signature + if !isHTML { + return body + "\n\n" + signature } - return body + "\n\n" + signature + block := `
` + signature + `
` + // Trailing content belongs inside the document, as for the pixel and footer. + if loc := bodyClose.FindAllStringIndex(body, -1); loc != nil { + at := loc[len(loc)-1][0] + return body[:at] + block + body[at:] + } + return body + block } // AddOpenTrackingPixel adds an invisible tracking pixel to HTML email. @@ -277,62 +311,6 @@ func AddOpenTrackingPixel(htmlBody string, taskID uuid.UUID, trackingDomain stri return htmlBody + pixel } -// WrapLinksForTracking rewrites every external link to an opaque -// click-tracking ticket (https:///c/) and returns the minted -// rows. The destination never travels inside the link, so there is nothing -// to forge: the tracking service resolves tickets via the backend internal -// API and 404s anything it does not know. The caller MUST persist the -// returned rows before using the rewritten body (and fall back to the -// original body on failure) so an email can never ship dead tickets. -func WrapLinksForTracking(htmlBody string, taskID, campaignID uuid.UUID, trackingDomain string) (string, []repository.TrackedLink) { - // No tracking host means no ticket can be resolved, and a wrapped link - // would be a dead link in a real customer's email. Ship the originals. - trackingDomain = config.NormalizeTrackingHost(trackingDomain) - if trackingDomain == "" { - return htmlBody, nil - } - - // Regex to find href attributes - linkRegex := regexp.MustCompile(`href="([^"]+)"`) - var links []repository.TrackedLink - - result := linkRegex.ReplaceAllStringFunc(htmlBody, func(match string) string { - // Extract the original URL - originalURL := linkRegex.FindStringSubmatch(match)[1] - - // Skip if already a tracking link or anchor link - if strings.HasPrefix(originalURL, "#") || - strings.Contains(originalURL, trackingDomain) || - strings.HasPrefix(originalURL, "mailto:") || - strings.HasPrefix(originalURL, "tel:") { - return match - } - - // Skip data URLs and javascript links - if strings.HasPrefix(originalURL, "data:") || - strings.HasPrefix(originalURL, "javascript:") { - return match - } - - // Only http(s) destinations are storable redirect targets - if !strings.HasPrefix(originalURL, "http://") && !strings.HasPrefix(originalURL, "https://") { - return match - } - - id := uuid.New() - links = append(links, repository.TrackedLink{ - ID: id, - TaskID: taskID, - CampaignID: campaignID, - Destination: originalURL, - }) - - return fmt.Sprintf(`href="%s"`, config.TrackingURL(trackingDomain, "/c/"+id.String())) - }) - - return result, links -} - // personaPick chooses from a mailbox's preferred subset of phrasing options so // each mailbox keeps a consistent "voice" while still varying message to // message. Falls back gracefully for tiny option sets. diff --git a/internal/tasks/template_test.go b/internal/tasks/template_test.go index 52492dee..abf96f38 100644 --- a/internal/tasks/template_test.go +++ b/internal/tasks/template_test.go @@ -234,3 +234,51 @@ func TestGenerateMessageID_Format(t *testing.T) { t.Errorf("message ID should start with <, got %q", mid) } } + +// The HTML signature is a block with a top margin, not

stacked on the +// body's own trailing margin, and it goes inside the document when there is one. +func TestAddSignatureSpacing(t *testing.T) { + out := AddSignature("

Hi

", "

Ana

", true) + if strings.Contains(out, "
") { + t.Errorf("signature still separated by breaks: %q", out) + } + if !strings.Contains(out, `

Ana

`) { + t.Errorf("signature not wrapped in its own block: %q", out) + } + + doc := AddSignature("

Hi

", "

Ana

", true) + if !strings.HasSuffix(doc, "") || strings.Index(doc, "Ana") > strings.Index(doc, "") { + t.Errorf("signature landed outside the document: %q", doc) + } + + // HTML tag names are case-insensitive, so a pasted counts too. + upper := AddSignature("

Hi

", "

Ana

", true) + if !strings.HasSuffix(upper, "") || strings.Index(upper, "Ana") > strings.Index(upper, "") { + t.Errorf("signature landed outside an upper-case document: %q", upper) + } + spaced := AddSignature("

Hi

", "

Ana

", true) + if strings.Index(spaced, "Ana") > strings.Index(spaced, "") { + t.Errorf("signature landed outside a spaced closing tag: %q", spaced) + } + + if plain := AddSignature("Hi", "Ana", false); plain != "Hi\n\nAna" { + t.Errorf("plain-text spacing changed: %q", plain) + } + if none := AddSignature("

Hi

", "", true); none != "

Hi

" { + t.Errorf("empty signature altered the body: %q", none) + } +} + +// The opt-out footer still lands after the signature once both are applied to a +// body carrying a , which is the order a reader expects. +func TestSignatureThenOptOutOrderInsideDocument(t *testing.T) { + body := AddSignature("

Hi

", "

Ana

", true) + body, _ = appendOptOut(body, "", models.UnsubscribeSettings{Mode: models.UnsubscribeModeText, Text: "Reply stop."}, "") + sig, foot := strings.Index(body, "Ana"), strings.Index(body, "Reply stop.") + if sig < 0 || foot < 0 || sig > foot { + t.Fatalf("footer did not follow the signature: %q", body) + } + if strings.Index(body, "") < foot { + t.Fatalf("footer landed outside the document: %q", body) + } +} diff --git a/internal/tasks/test_email.go b/internal/tasks/test_email.go index 97f5acf0..fa7556e8 100644 --- a/internal/tasks/test_email.go +++ b/internal/tasks/test_email.go @@ -3,6 +3,7 @@ package tasks import ( "context" "fmt" + "time" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" @@ -14,58 +15,65 @@ func (s *tasksService) GetCampaignSequences(ctx context.Context, campaignID uuid return s.campaignRepo.GetSequencesByCampaignID(ctx, campaignID) } -// SendTestEmail renders a campaign email and sends it to a test recipient for preview -func (s *tasksService) SendTestEmail(ctx context.Context, userID string, accountID uuid.UUID, recipient string, campaign *models.Campaign, sequence *models.Sequence) *errx.Error { - // Load the email account - account, err := s.emailRepo.GetByID(ctx, accountID) - if err != nil || account == nil { - return errx.New(errx.NotFound, "email account not found") - } - - // Verify account belongs to user - if account.UserID != userID { - return errx.ErrForbidden - } - - // Create a dummy contact for template rendering - testContact := models.Contact{ +// testContact stands in when no real contact is chosen for a test send. +func testContact(recipient string) models.Contact { + return models.Contact{ ID: uuid.New(), FirstName: "Test", LastName: "Recipient", Email: recipient, Company: "Test Company", } +} - // Render templates with the test contact - subject := RenderTemplate(sequence.Subject, testContact) - bodyHTML := RenderTemplate(sequence.BodyHTML, testContact) - bodyPlain := RenderTemplate(sequence.BodyPlain, testContact) - - if bodyPlain == "" && bodyHTML != "" { - bodyPlain = ExtractPlainTextFromHTML(bodyHTML) +// SendTestEmail renders a campaign step as the send path would and mails it to +// recipient through one of the organization's mailboxes. contact, when given, +// is the real contact the copy is rendered for; nil uses a placeholder. The +// message carries the campaign's attachments, the mailbox signature and the +// opt-out footer, so what lands in the tester's inbox is what a lead gets. +func (s *tasksService) SendTestEmail(ctx context.Context, orgID uuid.UUID, accountID uuid.UUID, recipient string, campaign *models.Campaign, sequence *models.Sequence, contact *models.Contact) *errx.Error { + // Any member allowed to send may test from any of the organization's + // mailboxes, not only the ones they connected. GetByID is the full row + // (the org-scoped Get omits worker_id, which the send needs). + account, err := s.emailRepo.GetByID(ctx, accountID) + if err != nil || account == nil || account.OrganizationID == nil || *account.OrganizationID != orgID { + return errx.New(errx.NotFound, "email account not found") } - // Prepend [TEST] to subject - subject = "[TEST] " + subject - - // Add signature if enabled - if account.SignatureSync { - bodyHTML = AddSignature(bodyHTML, account.SignatureHTML, true) - bodyPlain = AddSignature(bodyPlain, account.SignaturePlain, false) + renderFor := testContact(recipient) + if contact != nil { + renderFor = *contact } - // Generate message ID - messageID := generateMessageID(account.Email) + // A test send carries the real opt-out footer and header so the sender + // sees exactly what a recipient will, but its link names no contact + // (uuid.Nil), so clicking it can never suppress anyone. + optOut := s.resolveOptOut(ctx, orgID, campaign) + var unsubscribeURL string + if s.unsubLinks != nil && s.unsubLinks.Enabled() { + unsubscribeURL = s.unsubLinks.URL(orgID, campaign.ID, uuid.Nil, time.Now()) + } - // Build tracking info (disabled for test emails) + rendered := previewTemplatesWith(sequence.Subject, sequence.BodyHTML, sequence.BodyPlain, renderFor, unsubscribeURL) + bodyHTML, bodyPlain := finishBody(rendered.BodyHTML, rendered.BodyPlain, campaign.TextOnly, account, &optOut, unsubscribeURL) + subject := "[TEST] " + rendered.Subject + + headerURL := "" + if campaign.UnsubscribeHeader { + headerURL = unsubscribeURL + } + + // Tracking is deliberately off: a test open or click must not count. emailMsg := EmailMessage{ - From: account.Email, - To: []string{recipient}, - Subject: subject, - BodyHTML: bodyHTML, - BodyPlain: bodyPlain, - MessageID: messageID, - IsWarmup: false, + From: account.Email, + To: []string{recipient}, + Subject: subject, + BodyHTML: bodyHTML, + BodyPlain: bodyPlain, + MessageID: generateMessageID(account.Email), + IsWarmup: false, + UnsubscribeURL: headerURL, + Attachments: s.campaignAttachmentRefs(ctx, campaign.ID, sequence.ID), } taskID := uuid.New() diff --git a/internal/tasks/tracking_links_test.go b/internal/tasks/tracking_links_test.go index 8461bec7..52f66827 100644 --- a/internal/tasks/tracking_links_test.go +++ b/internal/tasks/tracking_links_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" ) func TestAddOpenTrackingPixelUsesTheConfiguredHost(t *testing.T) { @@ -68,3 +69,126 @@ func TestWrapLinksForTrackingSkipsItsOwnHostAndNonHTTP(t *testing.T) { t.Fatalf("body should be unchanged: %s", out) } } + +func TestTrackLinksReadsTheAnchorTextAsTheLabel(t *testing.T) { + html := `

See our Pricing & plans or Logo

` + _, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true}) + if len(links) != 2 { + t.Fatalf("expected two tickets, got %d", len(links)) + } + if links[0].Label != "our Pricing & plans" { + t.Fatalf("label not read from the anchor text: %q", links[0].Label) + } + if links[1].Label != "Logo" { + t.Fatalf("image link should fall back to alt text: %q", links[1].Label) + } +} + +func TestTrackLinksTagsUTMAndStoresTheTaggedDestination(t *testing.T) { + html := `Pricing page` + utm := &UTMParams{Source: "warmbly", Medium: "email", Campaign: "q3_outbound"} + out, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: utm}) + if len(links) != 1 { + t.Fatalf("expected one ticket, got %d", len(links)) + } + want := "https://example.com/pricing?ref=1&utm_source=warmbly&utm_medium=email&utm_campaign=q3_outbound&utm_content=pricing_page#top" + if links[0].Destination != want { + t.Fatalf("destination = %q, want %q", links[0].Destination, want) + } + if strings.Contains(out, "utm_") { + t.Fatalf("the email must carry only the ticket, got: %s", out) + } +} + +func TestTrackLinksTagsUTMWithoutWrappingWhenLinkTrackingIsOff(t *testing.T) { + html := `Go` + out, links := TrackLinks(html, LinkTracking{UTM: &UTMParams{Source: "s", Medium: "m", Campaign: "c"}}) + if links != nil { + t.Fatalf("no tickets expected, got %v", links) + } + want := `Go` + if out != want { + t.Fatalf("got %s\nwant %s", out, want) + } +} + +func TestTrackLinksKeepsHandWrittenUTMValues(t *testing.T) { + html := `Hero` + _, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: &UTMParams{Source: "warmbly", Medium: "email", Campaign: "c"}}) + got := links[0].Destination + if strings.Count(got, "utm_source=") != 1 || !strings.Contains(got, "utm_source=newsletter") || !strings.Contains(got, "utm_content=hero") { + t.Fatalf("hand-written values must win: %s", got) + } + if !strings.Contains(got, "utm_medium=email") || !strings.Contains(got, "utm_campaign=c") { + t.Fatalf("missing values must still be added: %s", got) + } +} + +func TestTrackLinksNumbersLinksWithoutText(t *testing.T) { + html := `` + _, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: &UTMParams{Source: "s", Medium: "m", Campaign: "c"}}) + if !strings.HasSuffix(links[0].Destination, "utm_content=link_1") || !strings.HasSuffix(links[1].Destination, "utm_content=link_2") { + t.Fatalf("unlabelled links should be numbered: %s / %s", links[0].Destination, links[1].Destination) + } +} + +// A stylesheet or preload href in the head is not a link anyone clicks; +// redirecting it through a ticket would break it. +func TestTrackLinksOnlyTouchesAnchors(t *testing.T) { + html := `p` + out, links := WrapLinksForTracking(html, uuid.New(), uuid.New(), "t.acme.com") + if len(links) != 1 || links[0].Destination != "https://example.com/p" { + t.Fatalf("expected only the anchor wrapped: %v", links) + } + if !strings.Contains(out, `href="https://example.com/a.css"`) { + t.Fatalf("stylesheet href must be untouched: %s", out) + } +} + +func TestCampaignUTMDefaults(t *testing.T) { + c := &models.Campaign{Name: "Q3 Outbound: Fintech!", UTMTracking: true} + p := CampaignUTM(c) + if p == nil || p.Source != "warmbly" || p.Medium != "email" || p.Campaign != "q3_outbound_fintech" { + t.Fatalf("unexpected defaults: %+v", p) + } + c.UTMSource, c.UTMCampaign = " acme ", "launch" + p = CampaignUTM(c) + if p.Source != "acme" || p.Campaign != "launch" { + t.Fatalf("overrides not honoured: %+v", p) + } + if CampaignUTM(&models.Campaign{Name: "x"}) != nil { + t.Fatal("utm tagging off must yield nil") + } +} + +func TestTrackLinksReadsBareHrefAndIgnoresDataHref(t *testing.T) { + html := `Bare` + out, links := WrapLinksForTracking(html, uuid.New(), uuid.New(), "t.acme.com") + if len(links) != 1 || links[0].Destination != "https://example.com/bare" || links[0].Label != "Bare" { + t.Fatalf("expected the bare href wrapped: %+v", links) + } + if !strings.Contains(out, `data-href="https://tracker.example/x"`) || !strings.Contains(out, `target="_blank"`) { + t.Fatalf("other attributes must survive: %s", out) + } +} + +func TestTrackableURLComparesTheHost(t *testing.T) { + if trackableURL("https://t.acme.com/c/abc", "t.acme.com") { + t.Fatal("a link already on the tracking host must be left alone") + } + if !trackableURL("https://example.com/?next=t.acme.com", "t.acme.com") { + t.Fatal("a URL merely mentioning the host is a normal link") + } + if !trackableURL("https://not-t.acme.com/", "t.acme.com") { + t.Fatal("a different host that ends with the tracking host is a normal link") + } +} + +func TestTagPlainTextLinks(t *testing.T) { + body := "See https://example.com/pricing. Docs: https://example.com/docs?x=1\nSkip https://t.acme.com/c/abc" + out := TagPlainTextLinks(body, &UTMParams{Source: "s", Medium: "m", Campaign: "c"}, "t.acme.com") + want := "See https://example.com/pricing?utm_source=s&utm_medium=m&utm_campaign=c&utm_content=link_1. Docs: https://example.com/docs?x=1&utm_source=s&utm_medium=m&utm_campaign=c&utm_content=link_2\nSkip https://t.acme.com/c/abc" + if out != want { + t.Fatalf("got %s\nwant %s", out, want) + } +} diff --git a/internal/updater/api.go b/internal/updater/api.go new file mode 100644 index 00000000..e4e4f5e9 --- /dev/null +++ b/internal/updater/api.go @@ -0,0 +1,95 @@ +// Package updater is the host-side agent that applies an update to a +// self-hosted Warmbly: pull the checkout, rebuild, restart, and wait for the +// backend to answer again. It runs next to the stack (a compose sidecar that +// holds the docker socket, or a systemd unit on a bare-metal host) and the +// backend drives it over a token-authenticated HTTP API that is never exposed +// publicly. This file is the wire contract both sides compile against. +package updater + +import "time" + +// Mode selects how the runner rebuilds and restarts after the checkout moved. +type Mode string + +const ( + // ModeCompose rebuilds the images and recreates the containers of the + // compose project the checkout belongs to. + ModeCompose Mode = "compose" + // ModeCommand runs an operator-provided command (a build-and-restart + // script) and leaves the rest to it. + ModeCommand Mode = "command" + // ModeImage is the clone-free install: no git, no build. The release tag + // is pinned in the install's .env, images are pulled from the registry and + // the containers whose image changed are recreated. + ModeImage Mode = "image" +) + +// JobStatus is the lifecycle of one update run. +type JobStatus string + +const ( + JobRunning JobStatus = "running" + JobSucceeded JobStatus = "succeeded" + JobFailed JobStatus = "failed" +) + +// Checkout describes the git checkout the updater manages. +type Checkout struct { + Branch string `json:"branch"` + Detached bool `json:"detached"` + Commit string `json:"commit"` + Describe string `json:"describe"` + RemoteCommit string `json:"remote_commit"` + Behind int `json:"behind"` + Dirty bool `json:"dirty"` + FetchedAt time.Time `json:"fetched_at"` + FetchError string `json:"fetch_error,omitempty"` +} + +// Job is one update run, with the tail of its log. +type Job struct { + ID string `json:"id"` + Status JobStatus `json:"status"` + Target string `json:"target"` + Step string `json:"step"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Error string `json:"error,omitempty"` + FromCommit string `json:"from_commit"` + ToCommit string `json:"to_commit,omitempty"` + Log []string `json:"log"` +} + +// Release describes an image-mode install: which tag its compose file +// resolves and where the images come from. It is the image-mode counterpart +// of Checkout, and exactly one of the two is ever set. +type Release struct { + // Tag is the value of WARMBLY_TAG in the install's .env, or the compose + // default when it names none. + Tag string `json:"tag"` + // Prefix is the registry namespace every image is pulled from. + Prefix string `json:"prefix"` + // Pinned is whether Tag is a fixed version rather than a moving channel + // tag. A moving tag republishes under the same name, so re-pulling it is + // a real update; a pinned one only moves when a release is chosen. + Pinned bool `json:"pinned"` +} + +// Status is the updater's answer to GET /status. +type Status struct { + Mode Mode `json:"mode"` + RepoDir string `json:"repo_dir"` + Version string `json:"version"` + // Checkout is set in compose and command mode, Release in image mode. + Checkout *Checkout `json:"checkout,omitempty"` + Release *Release `json:"release,omitempty"` + Job *Job `json:"job,omitempty"` + LastJob *Job `json:"last_job,omitempty"` +} + +// UpdateRequest is the body of POST /update. An empty Tag means: pull the +// tracked branch when the checkout is on one, otherwise refuse. In image mode +// it means re-pull whatever tag the install is already pinned to. +type UpdateRequest struct { + Tag string `json:"tag"` +} diff --git a/internal/updater/compose.go b/internal/updater/compose.go new file mode 100644 index 00000000..18fc6d12 --- /dev/null +++ b/internal/updater/compose.go @@ -0,0 +1,148 @@ +package updater + +import ( + "context" + "fmt" + "os/exec" + "sort" + "strings" +) + +// selfService is the compose service the updater itself runs as. +const selfService = "updater" + +// composeArgs prefixes every compose invocation with the project and the +// profiles, so the sidecar addresses the same stack the operator started. +func (r *Runner) composeArgs(args ...string) []string { + out := []string{"compose", "-p", r.cfg.ComposeProject} + for _, p := range strings.Split(r.cfg.ComposeProfiles, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, "--profile", p) + } + } + return append(out, args...) +} + +func (r *Runner) composeOutput(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "docker", r.composeArgs(args...)...) + cmd.Dir = r.cfg.RepoDir + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("docker compose %s: %s", strings.Join(args, " "), strings.TrimSpace(string(out))) + } + return strings.TrimSpace(string(out)), nil +} + +// composeUpdate rebuilds every image and recreates the containers whose image +// or configuration changed. Postgres, Redis and NATS keep running: compose +// leaves a container alone when nothing about it changed. +func (r *Runner) composeUpdate(ctx context.Context, job *Job) error { + r.step(job, "build") + describe := r.git.describe(ctx) + env := []string{ + "WARMBLY_BUILD_VERSION=" + describe, + "WARMBLY_BUILD_COMMIT=" + job.ToCommit, + "DOCKER_CLI_HINTS=false", + } + r.logf(job, "building images (%s)", describe) + if err := r.exec(ctx, job, r.cfg.RepoDir, env, "docker", r.composeArgs("build")...); err != nil { + return err + } + + r.step(job, "restart") + services, err := r.servicesToRecreate(ctx) + if err != nil { + return err + } + r.logf(job, "recreating %s", strings.Join(services, ", ")) + args := append(r.composeArgs("up", "-d", "--no-build"), services...) + if err := r.exec(ctx, job, r.cfg.RepoDir, env, "docker", args...); err != nil { + return err + } + + if r.cfg.Prune { + r.step(job, "prune") + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", "image", "prune", "-f"); err != nil { + r.logf(job, "image prune failed (ignored): %v", err) + } + } + return nil +} + +// servicesToRecreate is every service the checkout's profiles define plus +// everything currently running, minus the updater itself. The union covers a +// service the new version added and one the operator started by hand. +func (r *Runner) servicesToRecreate(ctx context.Context) ([]string, error) { + defined, err := r.composeOutput(ctx, "config", "--services") + if err != nil { + return nil, err + } + running, _ := r.composeOutput(ctx, "ps", "--services", "--status", "running") + set := map[string]bool{} + for _, chunk := range []string{defined, running} { + for _, s := range strings.Split(chunk, "\n") { + if s = strings.TrimSpace(s); s != "" && s != selfService { + set[s] = true + } + } + } + out := make([]string, 0, len(set)) + for s := range set { + out = append(out, s) + } + sort.Strings(out) + if len(out) == 0 { + return nil, fmt.Errorf("no compose services found in %s", r.cfg.RepoDir) + } + return out, nil +} + +// selfImageID is the image id the compose file resolves for the updater +// service, whether that image was just built or just pulled. It is asked of +// compose rather than assembled by hand, because the service carries an +// image: key now and `project-updater` is no longer its tag. +func (r *Runner) selfImageID(ctx context.Context) string { + ref, err := r.composeOutput(ctx, "config", "--images", selfService) + if err != nil || ref == "" { + // Older compose, or a file with neither key: fall back to the name a + // build without an image: key produces. + ref = r.cfg.ComposeProject + "-" + selfService + } + // --images answers one line per service; only one service was asked for. + if i := strings.IndexByte(ref, '\n'); i >= 0 { + ref = ref[:i] + } + out, err := exec.CommandContext(ctx, "docker", "image", "inspect", "-f", "{{.Id}}", strings.TrimSpace(ref)).Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// recreateSelf moves the updater onto the image it just built or pulled. It +// cannot run `compose up updater` in-process (that stops this container half +// way), so a detached one-off container from the new image does it a few +// seconds later. +func (r *Runner) recreateSelf(ctx context.Context, job *Job) { + running, err := r.composeOutput(ctx, "ps", "-q", selfService) + if err != nil || running == "" { + return + } + currentImage, err := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.Image}}", running).Output() + if err != nil { + return + } + nextImage := r.selfImageID(ctx) + if nextImage == "" || strings.TrimSpace(string(currentImage)) == nextImage { + return + } + r.logf(job, "updater image changed; recreating the updater itself") + inner := fmt.Sprintf("sleep 3; docker %s", strings.Join(r.composeArgs("up", "-d", "--no-build", "--no-deps", selfService), " ")) + args := r.composeArgs("run", "-d", "--rm", "--no-deps", "--entrypoint", "sh", selfService, "-c", inner) + cmd := exec.Command("docker", args...) + cmd.Dir = r.cfg.RepoDir + if out, err := cmd.CombinedOutput(); err != nil { + r.logf(job, "could not schedule the updater's own recreate (ignored): %s", strings.TrimSpace(string(out))) + } + r.saveState() +} diff --git a/internal/updater/git.go b/internal/updater/git.go new file mode 100644 index 00000000..b7b65dba --- /dev/null +++ b/internal/updater/git.go @@ -0,0 +1,130 @@ +package updater + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strconv" + "strings" + "time" +) + +// gitTimeout bounds one git invocation so a hung remote cannot pin a job. +const gitTimeout = 3 * time.Minute + +type git struct { + dir string + remote string +} + +// run executes git in the checkout. safe.directory covers the compose case, +// where the sidecar runs as root against a checkout the operator owns. +func (g git) run(ctx context.Context, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, gitTimeout) + defer cancel() + full := append([]string{"-c", "safe.directory=*", "-C", g.dir}, args...) + cmd := exec.CommandContext(ctx, "git", full...) + var out, stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return "", fmt.Errorf("git %s: %s", args[0], msg) + } + return strings.TrimSpace(out.String()), nil +} + +func (g git) head(ctx context.Context) (string, error) { + return g.run(ctx, "rev-parse", "HEAD") +} + +// branch returns the current branch, or "" when HEAD is detached. +func (g git) branch(ctx context.Context) (string, error) { + out, err := g.run(ctx, "rev-parse", "--abbrev-ref", "HEAD") + if err != nil { + return "", err + } + if out == "HEAD" { + return "", nil + } + return out, nil +} + +func (g git) describe(ctx context.Context) string { + out, err := g.run(ctx, "describe", "--tags", "--always", "--dirty") + if err != nil { + return "" + } + return out +} + +func (g git) dirty(ctx context.Context) (bool, error) { + out, err := g.run(ctx, "status", "--porcelain", "--untracked-files=no") + if err != nil { + return false, err + } + return out != "", nil +} + +func (g git) fetch(ctx context.Context) error { + _, err := g.run(ctx, "fetch", "--tags", "--prune", g.remote) + return err +} + +func (g git) remoteHead(ctx context.Context, branch string) (string, error) { + return g.run(ctx, "rev-parse", g.remote+"/"+branch) +} + +// behind counts commits on the remote branch that HEAD does not have. +func (g git) behind(ctx context.Context, branch string) (int, error) { + out, err := g.run(ctx, "rev-list", "--count", "HEAD.."+g.remote+"/"+branch) + if err != nil { + return 0, err + } + return strconv.Atoi(out) +} + +func (g git) pull(ctx context.Context, branch string) error { + _, err := g.run(ctx, "merge", "--ff-only", g.remote+"/"+branch) + return err +} + +func (g git) checkoutTag(ctx context.Context, tag string) error { + _, err := g.run(ctx, "checkout", "--detach", "refs/tags/"+tag) + return err +} + +func (g git) tagExists(ctx context.Context, tag string) bool { + _, err := g.run(ctx, "rev-parse", "--verify", "-q", "refs/tags/"+tag) + return err == nil +} + +// inspect reads the checkout state without touching the network. fetch runs +// separately so a slow remote never delays a status answer. +func (g git) inspect(ctx context.Context) (*Checkout, error) { + head, err := g.head(ctx) + if err != nil { + return nil, err + } + branch, err := g.branch(ctx) + if err != nil { + return nil, err + } + c := &Checkout{Commit: head, Branch: branch, Detached: branch == "", Describe: g.describe(ctx)} + if dirty, err := g.dirty(ctx); err == nil { + c.Dirty = dirty + } + if branch != "" { + if remote, err := g.remoteHead(ctx, branch); err == nil { + c.RemoteCommit = remote + } + if n, err := g.behind(ctx, branch); err == nil { + c.Behind = n + } + } + return c, nil +} diff --git a/internal/updater/image.go b/internal/updater/image.go new file mode 100644 index 00000000..a959dfed --- /dev/null +++ b/internal/updater/image.go @@ -0,0 +1,226 @@ +package updater + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Image mode is the updater for an install that never had a checkout: the one +// `curl warmbly.com/install.sh | sh` creates, which holds a generated compose +// file and a .env and runs the published release images. There is nothing to +// pull with git and nothing to build, so an update is: pin the tag in .env, +// `docker compose pull`, recreate what changed. + +// tagVar is the .env key the generated compose file reads every image tag +// from, so pinning a release is one line the operator can also edit by hand. +const tagVar = "WARMBLY_TAG" + +// envFileName is the environment file next to the compose file. Compose reads +// it automatically, so writing the tag there is all it takes to move. +const envFileName = ".env" + +// composeFileNames are the compose files an image install may carry, in the +// order docker compose itself resolves them. +var composeFileNames = []string{"docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"} + +// composeFile finds the install's compose file, or reports that there is none. +func composeFile(dir string) (string, bool) { + for _, name := range composeFileNames { + p := filepath.Join(dir, name) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p, true + } + } + return "", false +} + +// readTag returns the tag the install is pinned to, or "prod" when .env does +// not name one (the compose default). +func (r *Runner) readTag() string { + if v := readEnvVar(filepath.Join(r.cfg.RepoDir, envFileName), tagVar); v != "" { + return v + } + return "prod" +} + +// releaseState is the image-mode answer to "what is this install running". +func (r *Runner) releaseState() *Release { + tag := r.readTag() + return &Release{ + Tag: tag, + Prefix: r.imagePrefix(), + // A moving tag republishes under the same name, so "pull again" is a + // real update for it and a no-op for a pinned version. + Pinned: tag != "prod" && tag != "dev" && tag != "latest", + } +} + +func (r *Runner) imagePrefix() string { + if v := readEnvVar(filepath.Join(r.cfg.RepoDir, envFileName), "WARMBLY_IMAGE_PREFIX"); v != "" { + return v + } + return "ghcr.io/warmbly/warmbly" +} + +// imageUpdate is the image-mode counterpart of composeUpdate: no git, no +// build. The tag is written to .env FIRST so the pull, the recreate and every +// later `docker compose up` by hand all resolve the same images. +func (r *Runner) imageUpdate(ctx context.Context, job *Job, tag string) error { + r.step(job, "resolve") + if tag == "" { + tag = r.readTag() + r.logf(job, "no release named; re-pulling the pinned tag %s", tag) + } + if err := validTag(tag); err != nil { + return err + } + from := r.readTag() + envPath := filepath.Join(r.cfg.RepoDir, envFileName) + if err := setEnvVar(envPath, tagVar, tag); err != nil { + return fmt.Errorf("could not pin %s=%s in %s: %w", tagVar, tag, envPath, err) + } + if from == tag { + r.logf(job, "already pinned to %s; pulling it again", tag) + } else { + r.logf(job, "moved %s -> %s in %s", from, tag, envFileName) + } + + r.step(job, "pull") + r.logf(job, "pulling %s/*:%s", r.imagePrefix(), tag) + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", r.composeArgs("pull")...); err != nil { + // A pull that fails leaves .env pointing at a tag with no images, and + // the next `docker compose up` by hand would then fail the same way. + if rerr := setEnvVar(envPath, tagVar, from); rerr == nil { + r.logf(job, "pull failed; %s put back to %s", tagVar, from) + } + return err + } + + r.step(job, "restart") + services, err := r.servicesToRecreate(ctx) + if err != nil { + return err + } + r.logf(job, "recreating %s", strings.Join(services, ", ")) + args := append(r.composeArgs("up", "-d", "--no-build", "--remove-orphans"), services...) + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", args...); err != nil { + return err + } + + if r.cfg.Prune { + r.step(job, "prune") + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", "image", "prune", "-f"); err != nil { + r.logf(job, "image prune failed (ignored): %v", err) + } + } + return nil +} + +// validTag rejects anything that is not shaped like a release tag. +// +// The tag is written into .env as a whole line and then into an image +// reference, so a newline in it would append arbitrary environment to the +// install and a space would split the reference. The set below is what a +// container tag may hold anyway (OCI: alphanumerics, then any of ._-), so this +// refuses nothing legitimate. +func validTag(tag string) error { + if tag == "" || len(tag) > 128 { + return fmt.Errorf("%q is not a usable release tag", tag) + } + for _, r := range tag { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.' || r == '_' || r == '-': + default: + return fmt.Errorf("release tag %q contains %q, which cannot appear in an image tag", tag, r) + } + } + return nil +} + +// env file editing +// +// The file is read and rewritten whole rather than appended to, so a second +// update does not leave two WARMBLY_TAG lines with the loser still readable. + +// readEnvVar returns the last assignment of key in a KEY=VALUE file, or "". +// Last wins because that is what docker compose itself does. +func readEnvVar(path, key string) string { + b, err := os.ReadFile(path) //nolint:gosec // operator-owned install directory + if err != nil { + return "" + } + out := "" + for _, line := range strings.Split(string(b), "\n") { + k, v, ok := splitEnvLine(line) + if ok && k == key { + out = v + } + } + return out +} + +// setEnvVar rewrites key in place, keeping every other line and the file's +// position, and appends it when it is absent. The write is atomic and 0600: +// this file holds every secret the instance has. +func setEnvVar(path, key, value string) error { + raw, err := os.ReadFile(path) //nolint:gosec // operator-owned install directory + if err != nil && !os.IsNotExist(err) { + return err + } + lines := []string{} + if len(raw) > 0 { + lines = strings.Split(strings.TrimRight(string(raw), "\n"), "\n") + } + replaced := false + for i, line := range lines { + k, _, ok := splitEnvLine(line) + if !ok || k != key { + continue + } + if replaced { + // A duplicate earlier in the file would win nothing but confuse + // the next person to read it. + lines[i] = "# " + line + " # superseded by the updater" + continue + } + lines[i] = key + "=" + value + replaced = true + } + if !replaced { + lines = append(lines, key+"="+value) + } + body := strings.Join(lines, "\n") + "\n" + + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// splitEnvLine parses one KEY=VALUE line, skipping comments and blanks and +// unwrapping the quotes compose accepts. +func splitEnvLine(line string) (string, string, bool) { + s := strings.TrimSpace(line) + if s == "" || strings.HasPrefix(s, "#") { + return "", "", false + } + s = strings.TrimPrefix(s, "export ") + key, value, ok := strings.Cut(s, "=") + if !ok { + return "", "", false + } + key = strings.TrimSpace(key) + if key == "" { + return "", "", false + } + value = strings.TrimSpace(value) + if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] { + value = value[1 : len(value)-1] + } + return key, value, true +} diff --git a/internal/updater/owner_other.go b/internal/updater/owner_other.go new file mode 100644 index 00000000..30fb4214 --- /dev/null +++ b/internal/updater/owner_other.go @@ -0,0 +1,5 @@ +//go:build !unix + +package updater + +func ownerOf(string) (uid, gid int, ok bool) { return 0, 0, false } diff --git a/internal/updater/owner_unix.go b/internal/updater/owner_unix.go new file mode 100644 index 00000000..3597b13a --- /dev/null +++ b/internal/updater/owner_unix.go @@ -0,0 +1,21 @@ +//go:build unix + +package updater + +import ( + "os" + "syscall" +) + +// ownerOf returns the uid and gid owning path. +func ownerOf(path string) (uid, gid int, ok bool) { + info, err := os.Stat(path) + if err != nil { + return 0, 0, false + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, false + } + return int(st.Uid), int(st.Gid), true +} diff --git a/internal/updater/runner.go b/internal/updater/runner.go new file mode 100644 index 00000000..314cf551 --- /dev/null +++ b/internal/updater/runner.go @@ -0,0 +1,521 @@ +package updater + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +// Config is everything the runner reads from the environment. +type Config struct { + Mode Mode + RepoDir string + Remote string + // Command runs in ModeCommand after the checkout moved. It is passed to + // sh -c and has to build and restart everything itself. + Command string + // ComposeProject is the -p the stack was started with (the Makefile pins + // "warmbly"), ComposeProfiles the extra profiles to activate on top of the + // checkout's .env so the updater's own image is rebuilt too. + ComposeProject string + ComposeProfiles string + // BackendHealthURL is polled after the restart until it answers 200. + BackendHealthURL string + StateDir string + FetchInterval time.Duration + Prune bool + AllowDirty bool + Version string +} + +// maxLogLines caps a job's retained log so the status answer stays small. +const maxLogLines = 1500 + +// healthWait bounds how long the restart step waits for the backend. +const healthWait = 6 * time.Minute + +// Runner owns the checkout, the one job at a time, and the persisted history. +type Runner struct { + cfg Config + git git + + mu sync.Mutex + checkout *Checkout + job *Job + lastJob *Job + cancel context.CancelFunc +} + +// ErrJobRunning is returned when an update is asked for while one runs. +var ErrJobRunning = errors.New("an update is already running") + +func NewRunner(cfg Config) (*Runner, error) { + if cfg.RepoDir == "" { + return nil, errors.New("UPDATER_REPO_DIR is required") + } + if cfg.Mode == ModeImage { + // The clone-free install has no checkout to inspect; what it must have + // is the compose file the pull and the recreate address. + if _, ok := composeFile(cfg.RepoDir); !ok { + return nil, fmt.Errorf("%s holds no compose file; UPDATER_MODE=image needs the install directory", cfg.RepoDir) + } + } else if _, err := os.Stat(filepath.Join(cfg.RepoDir, ".git")); err != nil { + return nil, fmt.Errorf("%s is not a git checkout: %w", cfg.RepoDir, err) + } + if cfg.Mode == ModeCommand && strings.TrimSpace(cfg.Command) == "" { + return nil, errors.New("UPDATER_MODE=command needs UPDATER_COMMAND") + } + if cfg.Remote == "" { + cfg.Remote = "origin" + } + if cfg.ComposeProject == "" { + cfg.ComposeProject = "warmbly" + } + if cfg.FetchInterval <= 0 { + cfg.FetchInterval = 30 * time.Minute + } + r := &Runner{cfg: cfg, git: git{dir: cfg.RepoDir, remote: cfg.Remote}} + r.loadState() + return r, nil +} + +// Start refreshes the checkout state now and then on the fetch interval. +func (r *Runner) Start(ctx context.Context) { + go func() { + r.Refresh(ctx) + t := time.NewTicker(r.cfg.FetchInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + r.Refresh(ctx) + } + } + }() +} + +// Refresh fetches the remote and re-reads the checkout. In image mode there is +// nothing to fetch: what a release exists is the backend's GitHub check, and +// what is installed is one line of .env, read live in Status. +func (r *Runner) Refresh(ctx context.Context) *Checkout { + if r.cfg.Mode == ModeImage { + return nil + } + fetchErr := r.git.fetch(ctx) + c, err := r.git.inspect(ctx) + if err != nil { + log.Printf("updater: inspect checkout: %v", err) + c = &Checkout{} + } + c.FetchedAt = time.Now() + if fetchErr != nil { + c.FetchError = fetchErr.Error() + } + r.mu.Lock() + r.checkout = c + r.mu.Unlock() + return c +} + +// Status is the current snapshot. The checkout is re-read without a fetch so +// a job that just moved HEAD is reflected immediately. +func (r *Runner) Status(ctx context.Context) Status { + r.mu.Lock() + prev := r.checkout + job := cloneJob(r.job) + last := cloneJob(r.lastJob) + r.mu.Unlock() + + if r.cfg.Mode == ModeImage { + return Status{ + Mode: r.cfg.Mode, + RepoDir: r.cfg.RepoDir, + Version: r.cfg.Version, + Release: r.releaseState(), + Job: job, + LastJob: last, + } + } + + c, err := r.git.inspect(ctx) + if err == nil && prev != nil { + c.FetchedAt = prev.FetchedAt + c.FetchError = prev.FetchError + } else if err != nil { + c = prev + } + return Status{ + Mode: r.cfg.Mode, + RepoDir: r.cfg.RepoDir, + Version: r.cfg.Version, + Checkout: c, + Job: job, + LastJob: last, + } +} + +// StartUpdate begins a job and returns immediately. Progress is read from +// Status; the log is appended as the steps run. +func (r *Runner) StartUpdate(req UpdateRequest) (*Job, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.job != nil && r.job.Status == JobRunning { + return nil, ErrJobRunning + } + target := strings.TrimSpace(req.Tag) + if target == "" { + // What "no tag" means differs per mode, and the job label is what the + // admin panel shows while it runs. + target = "branch" + if r.cfg.Mode == ModeImage { + target = r.readTag() + } + } + job := &Job{ + ID: uuid.NewString(), + Status: JobRunning, + Target: target, + Step: "starting", + StartedAt: time.Now(), + } + r.job = job + ctx, cancel := context.WithCancel(context.Background()) + r.cancel = cancel + go r.execute(ctx, job, req) + return cloneJob(job), nil +} + +// Stop aborts a running job, used on process shutdown. +func (r *Runner) Stop() { + r.mu.Lock() + cancel := r.cancel + r.mu.Unlock() + if cancel != nil { + cancel() + } +} + +func (r *Runner) execute(ctx context.Context, job *Job, req UpdateRequest) { + err := r.runSteps(ctx, job, req) + r.mu.Lock() + now := time.Now() + job.FinishedAt = &now + if err != nil { + job.Status = JobFailed + job.Error = err.Error() + r.appendLocked(job, fmt.Sprintf("update failed: %v", err)) + } else { + job.Status = JobSucceeded + r.appendLocked(job, "update finished") + } + r.lastJob = job + r.job = nil + r.cancel = nil + r.mu.Unlock() + r.saveState() + + if err == nil && (r.cfg.Mode == ModeCompose || r.cfg.Mode == ModeImage) { + // Last, because it may replace this very process: the outcome above is + // already on disk for the successor to report. + r.recreateSelf(ctx, job) + } +} + +func (r *Runner) runSteps(ctx context.Context, job *Job, req UpdateRequest) error { + if r.cfg.Mode == ModeImage { + if err := r.imageUpdate(ctx, job, strings.TrimSpace(req.Tag)); err != nil { + return err + } + return r.waitForBackend(ctx, job) + } + + from, err := r.git.head(ctx) + if err != nil { + return err + } + r.mu.Lock() + job.FromCommit = from + r.mu.Unlock() + + r.step(job, "fetch") + r.logf(job, "fetching %s", r.cfg.Remote) + if err := r.git.fetch(ctx); err != nil { + return err + } + + r.step(job, "checkout") + if dirty, err := r.git.dirty(ctx); err == nil && dirty && !r.cfg.AllowDirty { + return errors.New("the checkout has local modifications; commit or stash them, or set UPDATER_ALLOW_DIRTY=true") + } + tag := strings.TrimSpace(req.Tag) + switch { + case tag != "": + if !r.git.tagExists(ctx, tag) { + return fmt.Errorf("tag %s does not exist on %s", tag, r.cfg.Remote) + } + r.logf(job, "checking out %s", tag) + if err := r.git.checkoutTag(ctx, tag); err != nil { + return err + } + default: + branch, err := r.git.branch(ctx) + if err != nil { + return err + } + if branch == "" { + return errors.New("the checkout is detached (pinned to a tag); choose a release to move to") + } + r.logf(job, "fast-forwarding %s to %s/%s", branch, r.cfg.Remote, branch) + if err := r.git.pull(ctx, branch); err != nil { + return err + } + } + to, err := r.git.head(ctx) + if err != nil { + return err + } + r.mu.Lock() + job.ToCommit = to + r.mu.Unlock() + if to == from { + r.logf(job, "already at %s; rebuilding anyway", short(to)) + } else { + r.logf(job, "moved %s -> %s", short(from), short(to)) + } + r.restoreOwnership(ctx, job) + + switch r.cfg.Mode { + case ModeCommand: + r.step(job, "command") + r.logf(job, "running UPDATER_COMMAND") + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "sh", "-c", r.cfg.Command); err != nil { + return err + } + default: + if err := r.composeUpdate(ctx, job); err != nil { + return err + } + } + + return r.waitForBackend(ctx, job) +} + +// waitForBackend is the last step of every mode: the update is only finished +// once the API answers again. +func (r *Runner) waitForBackend(ctx context.Context, job *Job) error { + if r.cfg.BackendHealthURL == "" { + return nil + } + r.step(job, "wait") + r.logf(job, "waiting for the backend at %s", r.cfg.BackendHealthURL) + if err := waitHealthy(ctx, r.cfg.BackendHealthURL, healthWait); err != nil { + return err + } + r.logf(job, "backend is answering") + return nil +} + +// restoreOwnership gives files git wrote as root back to the checkout's owner, +// so the operator's own git pull keeps working after the sidecar moved HEAD. +func (r *Runner) restoreOwnership(ctx context.Context, job *Job) { + if os.Geteuid() != 0 { + return + } + uid, gid, ok := ownerOf(r.cfg.RepoDir) + if !ok || uid == 0 { + return + } + spec := fmt.Sprintf("%d:%d", uid, gid) + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "chown", "-R", spec, filepath.Join(r.cfg.RepoDir, ".git")); err != nil { + r.logf(job, "could not restore ownership of .git: %v", err) + } + // Only files git touched need it; a full recursive chown over node_modules + // would take longer than the build. + out, err := r.git.run(ctx, "diff", "--name-only", job.FromCommit, "HEAD") + if err != nil || out == "" { + return + } + args := []string{spec} + for _, f := range strings.Split(out, "\n") { + if f = strings.TrimSpace(f); f != "" { + args = append(args, filepath.Join(r.cfg.RepoDir, f)) + } + } + _ = exec.CommandContext(ctx, "chown", args...).Run() +} + +// exec runs a command with its output streamed into the job log. +func (r *Runner) exec(ctx context.Context, job *Job, dir string, env []string, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), env...) + pr, pw := io.Pipe() + cmd.Stdout = pw + cmd.Stderr = pw + done := make(chan struct{}) + go func() { + defer close(done) + sc := bufio.NewScanner(pr) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimRight(sc.Text(), "\r") + if strings.TrimSpace(line) == "" { + continue + } + r.logf(job, "%s", line) + } + }() + err := cmd.Run() + _ = pw.Close() + <-done + if err != nil { + return fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return nil +} + +func (r *Runner) step(job *Job, name string) { + r.mu.Lock() + job.Step = name + r.mu.Unlock() + r.saveState() +} + +func (r *Runner) logf(job *Job, format string, args ...any) { + r.mu.Lock() + r.appendLocked(job, fmt.Sprintf(format, args...)) + r.mu.Unlock() +} + +// appendLocked records one log line. The caller holds r.mu; the mutex is not +// reentrant, so the completion path in execute must use this, never logf. +func (r *Runner) appendLocked(job *Job, msg string) { + line := time.Now().UTC().Format("15:04:05") + " " + msg + job.Log = append(job.Log, line) + if len(job.Log) > maxLogLines { + job.Log = job.Log[len(job.Log)-maxLogLines:] + } + log.Printf("updater: %s", line) +} + +// persisted state + +type stateFile struct { + LastJob *Job `json:"last_job,omitempty"` +} + +func (r *Runner) statePath() string { + if r.cfg.StateDir == "" { + return "" + } + return filepath.Join(r.cfg.StateDir, "state.json") +} + +func (r *Runner) loadState() { + p := r.statePath() + if p == "" { + return + } + b, err := os.ReadFile(p) + if err != nil { + return + } + var st stateFile + if err := json.Unmarshal(b, &st); err != nil { + return + } + if st.LastJob != nil && st.LastJob.Status == JobRunning { + // The process died mid-job (most likely it recreated itself after a + // successful compose run, or the host rebooted). Say so rather than + // showing a job that runs forever. + now := time.Now() + st.LastJob.Status = JobFailed + st.LastJob.FinishedAt = &now + st.LastJob.Error = "the updater restarted before the job finished" + } + r.lastJob = st.LastJob +} + +func (r *Runner) saveState() { + p := r.statePath() + if p == "" { + return + } + r.mu.Lock() + st := stateFile{LastJob: cloneJob(r.lastJob)} + if r.job != nil { + st.LastJob = cloneJob(r.job) + } + r.mu.Unlock() + b, err := json.MarshalIndent(st, "", " ") + if err != nil { + return + } + if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { + return + } + tmp := p + ".tmp" + if err := os.WriteFile(tmp, b, 0o640); err != nil { + return + } + _ = os.Rename(tmp, p) +} + +// helpers + +func cloneJob(j *Job) *Job { + if j == nil { + return nil + } + c := *j + c.Log = append([]string(nil), j.Log...) + return &c +} + +func short(sha string) string { + if len(sha) > 12 { + return sha[:12] + } + return sha +} + +func waitHealthy(ctx context.Context, url string, max time.Duration) error { + deadline := time.Now().Add(max) + client := &http.Client{Timeout: 3 * time.Second} + for { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("the backend did not answer at %s within %s; check its logs", url, max) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(3 * time.Second): + } + } +} diff --git a/internal/updater/server.go b/internal/updater/server.go new file mode 100644 index 00000000..bdaaccf6 --- /dev/null +++ b/internal/updater/server.go @@ -0,0 +1,88 @@ +package updater + +import ( + "bytes" + "crypto/subtle" + "encoding/json" + "errors" + "io" + "net/http" + "strings" +) + +// Server is the HTTP face of the runner. Every route except /health needs +// the bearer token; the backend is the only intended caller. +type Server struct { + runner *Runner + token string +} + +func NewServer(runner *Runner, token string) (*Server, error) { + if strings.TrimSpace(token) == "" { + return nil, errors.New("UPDATER_TOKEN is required") + } + return &Server{runner: runner, token: token}, nil +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + }) + mux.HandleFunc("GET /status", s.auth(s.status)) + mux.HandleFunc("POST /check", s.auth(s.check)) + mux.HandleFunc("POST /update", s.auth(s.update)) + return mux +} + +func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + got := strings.TrimSpace(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer")) + if subtle.ConstantTimeCompare([]byte(got), []byte(s.token)) != 1 { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + next(w, r) + } +} + +func (s *Server) status(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, s.runner.Status(r.Context())) +} + +func (s *Server) check(w http.ResponseWriter, r *http.Request) { + s.runner.Refresh(r.Context()) + writeJSON(w, http.StatusOK, s.runner.Status(r.Context())) +} + +func (s *Server) update(w http.ResponseWriter, r *http.Request) { + var req UpdateRequest + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4096)) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid body"}) + return + } + // A bare POST means "the tracked branch"; only a non-empty body is decoded. + if len(bytes.TrimSpace(raw)) > 0 { + if err := json.Unmarshal(raw, &req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid body"}) + return + } + } + job, err := s.runner.StartUpdate(req) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, ErrJobRunning) { + code = http.StatusConflict + } + writeJSON(w, code, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusAccepted, job) +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/internal/utils/paging/paging.go b/internal/utils/paging/paging.go index 1ffe0260..09df4688 100644 --- a/internal/utils/paging/paging.go +++ b/internal/utils/paging/paging.go @@ -159,3 +159,54 @@ func DecodeCursor(token string) (*string, *errx.Error) { s := id.String() return &s, nil } + +// mergedPrefix versions the merged-feed keyset token: (at, source, id). Used by +// feeds that merge several tables into one chronological list, where two rows +// can share a timestamp across tables and even within one, so the position +// needs the source and that source's row id to be unambiguous. +const mergedPrefix = "m1_" + +// EncodeMerged wraps an (at, source, id) keyset position in an opaque token. +// Returns nil for the zero id ("no next page") so the JSON field serializes as +// null. +func EncodeMerged(at time.Time, source int, id uuid.UUID) *string { + if id == uuid.Nil { + return nil + } + payload := at.UTC().Format(time.RFC3339Nano) + "|" + strconv.Itoa(source) + "|" + id.String() + tok := mergedPrefix + base64.RawURLEncoding.EncodeToString([]byte(payload)) + return &tok +} + +// DecodeMergedCursor reverses EncodeMerged. An empty token yields (zeroTime, 0, +// uuid.Nil, nil) (start from the beginning); an invalid token returns a 400. +func DecodeMergedCursor(token string) (time.Time, int, uuid.UUID, *errx.Error) { + if token == "" { + return time.Time{}, 0, uuid.Nil, nil + } + invalid := errx.New(errx.BadRequest, "invalid cursor") + if !strings.HasPrefix(token, mergedPrefix) { + return time.Time{}, 0, uuid.Nil, invalid + } + raw, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(token, mergedPrefix)) + if err != nil { + return time.Time{}, 0, uuid.Nil, invalid + } + parts := strings.SplitN(string(raw), "|", 3) + if len(parts) != 3 { + return time.Time{}, 0, uuid.Nil, invalid + } + at, err := time.Parse(time.RFC3339Nano, parts[0]) + if err != nil { + return time.Time{}, 0, uuid.Nil, invalid + } + source, err := strconv.Atoi(parts[1]) + if err != nil || source < 0 { + return time.Time{}, 0, uuid.Nil, invalid + } + id, err := uuid.Parse(parts[2]) + if err != nil { + return time.Time{}, 0, uuid.Nil, invalid + } + return at, source, id, nil +} diff --git a/internal/utils/paging/paging_test.go b/internal/utils/paging/paging_test.go new file mode 100644 index 00000000..42abc95f --- /dev/null +++ b/internal/utils/paging/paging_test.go @@ -0,0 +1,43 @@ +package paging + +import ( + "testing" + "time" + + "github.com/google/uuid" +) + +func TestMergedCursorRoundTrip(t *testing.T) { + at := time.Date(2026, 6, 9, 11, 42, 0, 123456000, time.FixedZone("CEST", 2*3600)) + id := uuid.New() + tok := EncodeMerged(at, 7, id) + if tok == nil { + t.Fatal("want a token for a real position") + } + gotAt, gotSource, gotID, xerr := DecodeMergedCursor(*tok) + if xerr != nil { + t.Fatalf("decode: %v", xerr) + } + if !gotAt.Equal(at) || gotSource != 7 || gotID != id { + t.Fatalf("round trip lost data: %v %d %s", gotAt, gotSource, gotID) + } + if EncodeMerged(at, 7, uuid.Nil) != nil { + t.Fatal("the zero id means no next page and must encode as nil") + } +} + +func TestMergedCursorRejectsMalformedTokens(t *testing.T) { + if _, _, _, xerr := DecodeMergedCursor(""); xerr != nil { + t.Fatalf("an empty token is the first page, got %v", xerr) + } + for _, tok := range []string{ + "2026-06-09T11:42:00Z", // a bare timestamp is not a cursor + "t1_" + (*EncodeTime(time.Now(), uuid.New()))[3:], // wrong version + "m1_!!!", // not base64 + "m1_" + (*EncodeMerged(time.Now(), 1, uuid.New()))[3:] + "x", // trailing garbage + } { + if _, _, _, xerr := DecodeMergedCursor(tok); xerr == nil { + t.Fatalf("%q must be rejected", tok) + } + } +} diff --git a/internal/utils/validate/campaign.go b/internal/utils/validate/campaign.go index 8da82b60..1866b656 100644 --- a/internal/utils/validate/campaign.go +++ b/internal/utils/validate/campaign.go @@ -3,6 +3,7 @@ package validate import ( "fmt" "time" + "unicode/utf8" "github.com/warmbly/warmbly/internal/bitmask" "github.com/warmbly/warmbly/internal/config" @@ -157,3 +158,18 @@ func CampaignTrackingDomain(host string) *errx.Error { } return nil } + +// CampaignUTMValue validates one of the campaign's UTM overrides. Empty means +// "use the default". Values are query-string parameters, so they must be +// short, single-line and printable; encoding is the send path's job. +func CampaignUTMValue(v string) *errx.Error { + if utf8.RuneCountInString(v) > 128 { + return errx.New(errx.BadRequest, "utm values must be 128 characters or fewer") + } + for _, r := range v { + if r < 0x20 || r == 0x7f { + return errx.New(errx.BadRequest, "utm values cannot contain control characters") + } + } + return nil +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 00000000..4d2b068c --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,55 @@ +// Package version holds the build identity stamped into every Warmbly binary. +// +// The values are injected at link time (see deploy/docker/*.Dockerfile and the +// Makefile), so a binary knows which release or commit it was built from +// without reading the checkout. .git is excluded from the docker build context, +// which is why runtime/debug's vcs data cannot be used instead. +package version + +import ( + "os" + "strings" +) + +var ( + // Version is the release tag (v1.4.0) or a git describe string + // (v1.4.0-3-gabc1234). Empty when the build did not stamp one. + Version = "" + // Commit is the full git sha the binary was built from. + Commit = "" + // BuiltAt is the RFC 3339 build time. + BuiltAt = "" +) + +// String is the version to display: the stamped value, WARMBLY_VERSION from +// the environment (the pre-existing override), or "dev". +func String() string { + if v := strings.TrimSpace(Version); v != "" { + return v + } + if v := strings.TrimSpace(os.Getenv("WARMBLY_VERSION")); v != "" { + return v + } + return "dev" +} + +// ShortCommit is the first 12 characters of the commit, or empty. +func ShortCommit() string { + c := strings.TrimSpace(Commit) + if len(c) > 12 { + return c[:12] + } + return c +} + +// Info is the JSON shape every surface reports the running build as. +type Info struct { + Version string `json:"version"` + Commit string `json:"commit,omitempty"` + BuiltAt string `json:"built_at,omitempty"` +} + +// Current returns the running build's identity. +func Current() Info { + return Info{Version: String(), Commit: strings.TrimSpace(Commit), BuiltAt: strings.TrimSpace(BuiltAt)} +} diff --git a/scripts/build-cli.sh b/scripts/build-cli.sh new file mode 100755 index 00000000..806bb0ab --- /dev/null +++ b/scripts/build-cli.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# +# Builds the `warmbly` CLI for every platform we publish, packages each one, +# and writes the manifests the package managers read. +# +# Run by the release workflow and by `make cli-dist`, so a release artifact can +# be reproduced locally byte for byte given the same VERSION and COMMIT. +# +# ./scripts/build-cli.sh dist +# +# Assets are named without the version on purpose: the install script resolves +# https://github.com/warmbly/warmbly/releases/latest/download/warmbly__.tar.gz +# with no GitHub API call, and the unauthenticated API's rate limit is exactly +# what breaks a curl installer on a shared CI runner. +set -euo pipefail + +cd "$(dirname "$0")/.." + +OUT=${1:-dist} +REPO=warmbly/warmbly +MODULE=github.com/warmbly/warmbly + +VERSION=${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo dev)} +COMMIT=${COMMIT:-$(git rev-parse HEAD 2>/dev/null || echo "")} +BUILT_AT=${BUILT_AT:-$(date -u +%Y-%m-%dT%H:%M:%SZ)} + +# Every platform the install script and the package managers know how to ask +# for. Keep this list and the one in site/public/cli.sh in step; the installer +# check verifies they agree. +PLATFORMS="darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 windows/amd64 windows/arm64" + +LDFLAGS="-s -w + -X ${MODULE}/internal/version.Version=${VERSION} + -X ${MODULE}/internal/version.Commit=${COMMIT} + -X ${MODULE}/internal/version.BuiltAt=${BUILT_AT}" + +# macOS ships shasum and not GNU sha256sum, and this script is meant to be +# reproducible on a maintainer's laptop as well as on the release runner. +sha256_all() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$@" + else + shasum -a 256 "$@" + fi +} + +rm -rf "$OUT" +mkdir -p "$OUT" + +# Completions ship inside every archive so the install script can drop them in +# without running the binary it just downloaded, which it cannot do for a +# cross-platform install anyway. +stage_completions() { + local host_bin=$1 dest=$2 + mkdir -p "$dest" + for shell in bash zsh fish powershell; do + "$host_bin" completion "$shell" > "$dest/warmbly.$shell" 2>/dev/null || true + done +} + +echo "building warmbly ${VERSION}" + +host_bin="$OUT/.host/warmbly" +mkdir -p "$OUT/.host" +# shellcheck disable=SC2086 +go build -ldflags="$LDFLAGS" -o "$host_bin" ./cmd/cli + +completions="$OUT/.completions" +stage_completions "$host_bin" "$completions" + +for target in $PLATFORMS; do + os=${target%/*} + arch=${target#*/} + ext="" + if [ "$os" = "windows" ]; then ext=".exe"; fi + + stage="$OUT/.stage/warmbly_${os}_${arch}" + mkdir -p "$stage" + echo " $os/$arch" + # shellcheck disable=SC2086 + CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" \ + go build -ldflags="$LDFLAGS" -o "$stage/warmbly${ext}" ./cmd/cli + + cp LICENSE README.md "$stage/" + cp -r "$completions" "$stage/completions" + + if [ "$os" = "windows" ]; then + (cd "$stage" && zip -qr "../../warmbly_${os}_${arch}.zip" .) + else + tar -czf "$OUT/warmbly_${os}_${arch}.tar.gz" -C "$stage" . + fi +done + +rm -rf "$OUT/.stage" "$OUT/.host" "$OUT/.completions" + +(cd "$OUT" && sha256_all warmbly_* > checksums.txt) +echo +cat "$OUT/checksums.txt" + +# ───────────────────────────────────────────────────────────────────────── +# Package manager manifests +# +# Written here rather than by hand so the checksums in them can never drift +# from the archives they describe, which is the failure mode that makes a tap +# install fail for everyone at once. +# ───────────────────────────────────────────────────────────────────────── + +sum_for() { awk -v f="$1" '$2 == f { print $1 }' "$OUT/checksums.txt"; } + +BASE="https://github.com/${REPO}/releases/download/${VERSION}" + +cat > "$OUT/warmbly.rb" < "warmbly" + zsh_completion.install "completions/warmbly.zsh" => "_warmbly" + fish_completion.install "completions/warmbly.fish" => "warmbly.fish" + end + + test do + assert_match "warmbly", shell_output("#{bin}/warmbly version") + end +end +EOF + +cat > "$OUT/warmbly.json" <&2; exit 1; } +pass() { printf '\033[32m✓\033[0m %s\n' "$*"; } + +# macOS ships shasum and not GNU sha256sum; this check has to run there too. +sha256_all() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@"; else shasum -a 256 "$@"; fi +} +sha256_verify() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum -c "$@"; else shasum -a 256 -c "$@"; fi +} + +[[ -f $SCRIPT ]] || fail "$SCRIPT is missing" +[[ -f $PS_SCRIPT ]] || fail "$PS_SCRIPT is missing" + +# The script is executed by whatever /bin/sh is on the machine, which on Debian +# and Ubuntu is dash. Checking with bash alone would let a bashism through to +# exactly the hosts this is aimed at. +if command -v dash >/dev/null 2>&1; then + dash -n "$SCRIPT" || fail "the installer is not valid POSIX sh (dash -n)" + pass "parses as POSIX sh" +else + sh -n "$SCRIPT" || fail "the installer does not parse" + pass "parses (dash not installed; POSIX check was approximate)" +fi + +if command -v shellcheck >/dev/null 2>&1; then + shellcheck -s sh "$SCRIPT" || fail "shellcheck found problems in the installer" + pass "shellcheck clean ($(shellcheck --version | awk '/^version:/ {print $2}'))" +else + echo "· shellcheck not installed; skipped" +fi + +# --help must work before anything is set up, which is where an unbound +# variable under set -u would otherwise hide. +sh "$SCRIPT" --help >/dev/null || fail "--help failed" +pass "--help works" + +# A rejected flag has to print its message. This used to abort with +# "C_RED: unbound variable" instead, because parse_args runs before the colours +# are set and `set -u` turns an unset variable into a fatal error. Only --help +# and --dry-run were exercised, so nothing caught it. +for bad in --nonsense --dir; do + # The exit status matters as much as the message: a script that explains the + # problem and then exits 0 tells every caller the install succeeded. + if out=$(sh "$SCRIPT" "$bad" 2>&1); then + fail "$bad exited 0; a rejected flag has to fail" + fi + case "$out" in + *"unbound variable"*) fail "$bad aborted with an unbound variable instead of an error message" ;; + esac + case "$out" in + *"unknown option"*|*"needs a path"*) ;; + *) fail "$bad did not explain itself: +$out" ;; + esac +done +pass "a rejected flag explains itself" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# --dry-run reaches its end with no network and, above all, writes nothing. +HOME="$work/dryhome" SHELL=/bin/bash sh "$SCRIPT" --dry-run --no-color --dir "$work/dryhome/bin" >/dev/null 2>&1 \ + || fail "--dry-run failed" +[[ ! -e "$work/dryhome" ]] || fail "--dry-run created $work/dryhome; it must write nothing" +pass "--dry-run runs and writes nothing" + +# Every platform the installer will ask for has to be one we build, and the +# other way round. A mismatch is a 404 for whoever runs it on that machine. +script_platforms=$(sed -n 's/^PLATFORMS="\(.*\)"$/\1/p' "$SCRIPT" | tr ' ' '\n' | sort) +build_platforms=$(sed -n 's/^PLATFORMS="\(.*\)"$/\1/p' scripts/build-cli.sh | tr ' ' '\n' | sed 's|/|_|' | grep -v '^windows' | sort) +if [[ "$script_platforms" != "$build_platforms" ]]; then + fail "cli.sh and scripts/build-cli.sh disagree about platforms: +installer builds for: +$script_platforms +release builds: +$build_platforms" +fi +pass "installer and release agree on platforms" + +# ───────────────────────────────────────────────────────────────────────── +# A real install, against a mirror on disk. file:// keeps this offline, which +# is what lets it run in CI without reaching GitHub. +# ───────────────────────────────────────────────────────────────────────── + +mirror="$work/mirror" +mkdir -p "$mirror" "$work/stage/completions" + +# A stand-in for the real binary: the installer only needs something that runs +# and answers `version`, and building the real CLI here would make this check +# a minute slower for nothing. +cat > "$work/stage/warmbly" <<'STUB' +#!/bin/sh +[ "${1:-}" = version ] && echo "warmbly v0.0.0-test (test)" && exit 0 +exit 0 +STUB +chmod +x "$work/stage/warmbly" +echo "# completions" > "$work/stage/completions/warmbly.bash" +echo "# completions" > "$work/stage/completions/warmbly.zsh" +echo "# completions" > "$work/stage/completions/warmbly.fish" +cp LICENSE "$work/stage/" 2>/dev/null || echo license > "$work/stage/LICENSE" + +host_os=$(uname -s | tr '[:upper:]' '[:lower:]') +case "$(uname -m)" in + x86_64|amd64) host_arch=amd64 ;; + arm64|aarch64) host_arch=arm64 ;; + *) host_arch=amd64 ;; +esac +asset="warmbly_${host_os}_${host_arch}.tar.gz" +tar -czf "$mirror/$asset" -C "$work/stage" . +( cd "$mirror" && sha256_all "$asset" > checksums.txt ) + +home="$work/home" +mkdir -p "$home" +HOME="$home" SHELL=/bin/bash sh "$SCRIPT" \ + --base-url "file://$mirror" --dir "$home/bin" --no-color >/dev/null 2>&1 \ + || fail "installing from a local mirror failed" + +[[ -x "$home/bin/warmbly" ]] || fail "the installer did not produce $home/bin/warmbly" +[[ "$("$home/bin/warmbly" version)" == "warmbly v0.0.0-test (test)" ]] || fail "the installed binary does not run" +pass "installs a working binary from a mirror" + +grep -q 'warmbly CLI installer' "$home/.bash_profile" 2>/dev/null || grep -q 'warmbly CLI installer' "$home/.bashrc" 2>/dev/null \ + || fail "the installer did not put the install directory on PATH" +pass "puts the install directory on PATH" + +[[ -f "$home/.local/share/bash-completion/completions/warmbly" ]] || fail "no bash completions were written" +pass "writes shell completions" + +# A second run must not append the PATH line again. +HOME="$home" SHELL=/bin/bash sh "$SCRIPT" \ + --base-url "file://$mirror" --dir "$home/bin" --no-color >/dev/null 2>&1 \ + || fail "the second install run failed" +occurrences=$(grep -c 'warmbly CLI installer' "$home/.bash_profile" 2>/dev/null || true) +[[ "${occurrences:-0}" -le 1 ]] || fail "re-running appended the PATH line again ($occurrences times)" +pass "re-running is idempotent" + +# A tampered archive must stop the install, not warn about it. +bad="$work/badmirror" +mkdir -p "$bad" +cp "$mirror/$asset" "$bad/" +sed 's/^[0-9a-f]\{64\}/0000000000000000000000000000000000000000000000000000000000000000/' \ + "$mirror/checksums.txt" > "$bad/checksums.txt" +badhome="$work/badhome" +if HOME="$badhome" sh "$SCRIPT" --base-url "file://$bad" --dir "$badhome/bin" --no-color >/dev/null 2>&1; then + fail "a checksum mismatch did not stop the install" +fi +[[ ! -e "$badhome/bin/warmbly" ]] || fail "a checksum mismatch still installed the binary" +pass "refuses to install on a checksum mismatch" + +# --uninstall removes what it wrote, and leaves the credentials alone. +mkdir -p "$home/.config/warmbly" +echo "token" > "$home/.config/warmbly/hosts.yml" +HOME="$home" SHELL=/bin/bash sh "$SCRIPT" --uninstall --dir "$home/bin" --no-color >/dev/null 2>&1 \ + || fail "--uninstall failed" +[[ ! -e "$home/bin/warmbly" ]] || fail "--uninstall left the binary behind" +[[ ! -e "$home/.local/share/bash-completion/completions/warmbly" ]] || fail "--uninstall left completions behind" +[[ -f "$home/.config/warmbly/hosts.yml" ]] || fail "--uninstall removed the credentials; it must not" +pass "--uninstall removes the binary and completions, and keeps credentials" + +# ───────────────────────────────────────────────────────────────────────── +# The Windows half. Ubuntu runners ship pwsh, so this is a real parse there. +# ───────────────────────────────────────────────────────────────────────── + +if command -v pwsh >/dev/null 2>&1; then + pwsh -NoProfile -Command " + \$errors = \$null + [System.Management.Automation.Language.Parser]::ParseFile('$PWD/$PS_SCRIPT', [ref]\$null, [ref]\$errors) | Out-Null + if (\$errors) { \$errors | ForEach-Object { Write-Host \$_ }; exit 1 } + " || fail "$PS_SCRIPT does not parse as PowerShell" + pass "cli.ps1 parses as PowerShell" +else + echo "· pwsh not installed; skipped the PowerShell parse" +fi + +# ───────────────────────────────────────────────────────────────────────── +# The published checksum, which is what makes "download, verify, read, run" a +# real alternative to piping into a shell. +# ───────────────────────────────────────────────────────────────────────── + +[[ -f $SUMFILE ]] || fail "$SUMFILE is missing. Run: make cli-sha" +( cd site/public && sha256_verify "$(basename "$SUMFILE")" >/dev/null ) \ + || fail "$SUMFILE does not match $SCRIPT. Run: make cli-sha" +pass "published checksum matches" + +printf '\n\033[32mAll CLI installer checks passed.\033[0m\n' diff --git a/scripts/check-installer.sh b/scripts/check-installer.sh new file mode 100755 index 00000000..2c9d0e4f --- /dev/null +++ b/scripts/check-installer.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# Checks the one-command installer served at https://warmbly.com/install.sh. +# +# It is served verbatim out of site/public, so this runs against the exact +# bytes a `curl -fsSL https://warmbly.com/install.sh | sh` executes: +# +# * it parses as POSIX sh, in dash and not only in bash +# * shellcheck has nothing to say about it +# * --help, --print-env and --dry-run work without a terminal, a docker or a +# network, because that is how someone reads it before trusting it +# * every compose file it can generate is one docker compose accepts +# * the published checksum matches, so the documented +# "download, verify, read, run" path actually verifies +set -euo pipefail + +cd "$(dirname "$0")/.." +SCRIPT=site/public/install.sh +SUMFILE=site/public/install.sh.sha256 + +fail() { printf '\n\033[31m✗\033[0m %s\n' "$*" >&2; exit 1; } +pass() { printf '\033[32m✓\033[0m %s\n' "$*"; } + +[[ -f $SCRIPT ]] || fail "$SCRIPT is missing" + +# The script is executed by whatever /bin/sh is on the operator's box, which on +# Debian and Ubuntu is dash. Checking it with bash alone would let a bashism +# through to exactly the hosts this is aimed at. +if command -v dash >/dev/null 2>&1; then + dash -n "$SCRIPT" || fail "the installer is not valid POSIX sh (dash -n)" + pass "parses as POSIX sh" +else + sh -n "$SCRIPT" || fail "the installer does not parse" + pass "parses (dash not installed; POSIX check was approximate)" +fi + +# The version is printed because it matters: SC2015 and friends move between +# releases, so a local run that passes on a newer shellcheck than CI's is not +# the same check. When CI disagrees with you, this line is why. +if command -v shellcheck >/dev/null 2>&1; then + shellcheck -s sh "$SCRIPT" || fail "shellcheck found problems in the installer" + pass "shellcheck clean ($(shellcheck --version | awk '/^version:/ {print $2}'))" +else + echo "· shellcheck not installed; skipped" +fi + +# --help must work before anything is set up, which is where an unbound +# variable under set -u would otherwise hide. +sh "$SCRIPT" --help >/dev/null || fail "--help failed" +pass "--help works" + +# --demo has to reach its end with no terminal, no Docker and no network, and +# above all it must not create the install directory it talks about. +demo_dir=$(mktemp -d)/opt-warmbly +sh "$SCRIPT" --demo --no-color --dir "$demo_dir" >/dev/null 2>&1 || fail "--demo failed" +[[ ! -e $demo_dir ]] || fail "--demo created $demo_dir; it must write nothing" +pass "--demo runs and writes nothing" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +sh "$SCRIPT" --print-env --no-color --dir "$work/inst" --version v0.0.0-test >"$work/env" || + fail "--print-env failed" +for key in WARMBLY_TAG AUTH_SECRET CREDENTIALS_ENCRYPTION_KEY KMS_LOCAL_MASTER_KEY \ + INTERNAL_API_TOKEN SECRET_KEY_BASE PRIMARY_DB WARMBLY_SETTINGS_BOOTSTRAP; do + grep -q "^${key}=." "$work/env" || fail "--print-env wrote no $key" +done +grep -q '^WARMBLY_TAG=v0.0.0-test$' "$work/env" || fail "--version was not pinned into .env" +pass "--print-env writes a complete .env" + +# Every shape of answer has to produce a compose file compose accepts. These +# are the four that change the file's structure rather than its values. +check_shape() { + local label=$1; shift + local dir="$work/shape" + rm -rf "$dir"; mkdir -p "$dir" + sh "$SCRIPT" --dry-run --no-color --dir "$dir/inst" --version v0.0.0-test "$@" >"$dir/out" || + fail "--dry-run failed for: $label" + python3 - "$dir" <<'PY' +import sys, os, re +d = sys.argv[1] +lines = open(os.path.join(d, "out")).read().split("\n") +def extract(marker, out): + idx = [i for i, l in enumerate(lines) if l.strip().startswith("── ") and marker in l] + if not idx: + return + body = [] + for l in lines[idx[0] + 1:]: + if l.strip().startswith("── ") and "(mode" in l: + break + body.append(l[2:] if l.startswith(" ") else l) + while body and (body[-1].strip() == "" or body[-1][:1] in "╭│╰"): + body.pop() + open(os.path.join(d, out), "w").write("\n".join(body) + "\n") +extract("/.env", ".env") +extract("docker-compose.yml", "docker-compose.yml") +PY + [[ -f "$dir/docker-compose.yml" ]] || fail "--dry-run printed no compose file for: $label" + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + ( cd "$dir" && docker compose config >/dev/null ) || fail "invalid compose file for: $label" + fi + pass "generates a valid stack: $label" +} + +check_shape "defaults" +check_shape "bundled TLS" --tls caddy --host warmbly.example.com +check_shape "core only" --components core +check_shape "named volumes" --data-root volumes +check_shape "external stores" --database-url postgres://u:p@db:5432/w --redis-url redis://cache:6379 --blobs s3 + +# Nothing drawn inside a redraw loop may be wider than the terminal. A wrapped +# line is two physical rows, every cursor-up counts logical ones, and the menu +# then draws over itself and over whatever was on screen before it. This is the +# regression that check exists for. +if command -v python3 >/dev/null 2>&1; then + python3 - "$SCRIPT" <<'PYEOF' || fail "the installer drew past the terminal width" +import fcntl, os, pty, re, select, struct, sys, termios, time + +script = sys.argv[1] +failures = [] +for cols in (80, 100): + master, slave = pty.openpty() + fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 45, cols, 0, 0)) + pid = os.fork() + if pid == 0: + os.setsid() + fcntl.ioctl(slave, termios.TIOCSCTTY, 0) + os.dup2(slave, 0); os.dup2(slave, 1); os.dup2(slave, 2) + os.close(master); os.close(slave) + os.environ["TERM"] = "xterm-256color" + os.environ["WARMBLY_DEMO_FAST"] = "1" + os.execvp("sh", ["sh", script, "--demo"]) + os._exit(1) + os.close(slave) + buf = b"" + start = last = time.time() + sent = 0 + while time.time() - start < 90: + r, _, _ = select.select([master], [], [], 0.25) + if r: + try: + chunk = os.read(master, 65536) + except OSError: + break + if not chunk: + break + buf += chunk + last = time.time() + elif time.time() - last > 0.4 and sent < 80: + tail = re.sub(r"\x1b\[[0-9;?]*[a-zA-Z]", "", buf.decode("utf-8", "replace"))[-400:] + os.write(master, b"copied\r" if "Type 'copied'" in tail else b"\r") + sent += 1 + last = time.time() + if b"That was the demo" in buf: + break + for fn in (lambda: os.close(master), lambda: os.waitpid(pid, 0)): + try: + fn() + except OSError: + pass + text = buf.decode("utf-8", "replace") + if "command not found" in text or "syntax error" in text: + failures.append(f"{cols} columns: the run produced shell errors") + if "That was the demo" not in text: + failures.append(f"{cols} columns: the demo did not reach its end") + plain = re.sub(r"\x1b\[[0-9;?]*[a-zA-Z]", "", text) + over = [l for l in plain.replace("\r", "\n").split("\n") if len(l) > cols] + if over: + failures.append(f"{cols} columns: {len(over)} line(s) too wide, first: {over[0][:cols + 20]!r}") + +for f in failures: + print(" " + f, file=sys.stderr) +sys.exit(1 if failures else 0) +PYEOF + pass "draws inside the terminal at 80 and 100 columns" +else + echo "· python3 not installed; the width check was skipped" +fi + +# The checksum is the whole answer to "why would I pipe this into a shell", so +# a stale one is a failure, not a warning. +if [[ ! -f $SUMFILE ]]; then + fail "$SUMFILE is missing. Regenerate it with: make installer-sha" +fi +expected=$(awk '{print $1}' "$SUMFILE") +actual=$(sha256sum "$SCRIPT" | awk '{print $1}') +if [[ $expected != "$actual" ]]; then + fail "$SUMFILE is stale. + published $expected + actual $actual + Regenerate it with: make installer-sha" +fi +pass "published checksum matches" + +printf '\n\033[32mThe installer is good.\033[0m\n' diff --git a/scripts/upgrade-bare-metal.sh b/scripts/upgrade-bare-metal.sh new file mode 100755 index 00000000..36e3f7ee --- /dev/null +++ b/scripts/upgrade-bare-metal.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Rebuild a Docker-free Warmbly install after the checkout moved, then hand the +# artifacts to the privileged installer. This is what the updater runs in +# UPDATER_MODE=command (deploy/systemd/warmbly-updater.service), and what you +# run by hand after `git pull`. It is the "Upgrading" section of +# docs/content/docs/development/bare-metal.mdx as a script. +# +# scripts/upgrade-bare-metal.sh # build, then install + restart +# scripts/upgrade-bare-metal.sh --pull # git pull --ff-only first +# +# Run it as the user who owns the checkout. Everything here runs unprivileged; +# the only root step is the fixed-path installer, which is the single command +# that user may run through sudo (see deploy/systemd/warmbly-install-release.sh). +set -euo pipefail + +SRC="${WARMBLY_SRC:-/opt/warmbly/src}" +PREFIX="${WARMBLY_PREFIX:-/opt/warmbly}" +INSTALLER="${WARMBLY_INSTALLER:-/usr/local/sbin/warmbly-install-release}" + +log() { printf '==> %s\n' "$*"; } + +if [[ "${1:-}" == "--pull" ]]; then + log "pulling" + git -C "$SRC" pull --ff-only +fi + +[[ -x "$INSTALLER" ]] || { + echo "$INSTALLER is missing. Install it root-owned first:" >&2 + echo " sudo install -o root -g root -m 0755 $SRC/deploy/systemd/warmbly-install-release.sh $INSTALLER" >&2 + exit 1 +} + +cd "$SRC" +VERSION="$(git describe --tags --always --dirty 2>/dev/null || echo dev)" +COMMIT="$(git rev-parse HEAD 2>/dev/null || true)" +BUILT_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +LDFLAGS="-s -w -X github.com/warmbly/warmbly/internal/version.Version=$VERSION -X github.com/warmbly/warmbly/internal/version.Commit=$COMMIT -X github.com/warmbly/warmbly/internal/version.BuiltAt=$BUILT_AT" + +# A service is built only if this host runs it: no Rust toolchain means no +# tracking rebuild, and a unit that is not installed is skipped. +has_unit() { systemctl list-unit-files "warmbly-$1.service" --no-legend 2>/dev/null | grep -q .; } + +log "building Go services ($VERSION)" +export CGO_ENABLED=0 +mkdir -p out +for cmd in backend forms consumer worker migrate warmblyctl updater; do + go build -ldflags="$LDFLAGS" -o "out/$cmd" "./cmd/$cmd" +done + +if has_unit tracking && command -v cargo >/dev/null 2>&1; then + log "building tracking" + (cd tracking && cargo build --release) +fi + +if has_unit realtime && command -v mix >/dev/null 2>&1; then + log "building realtime" + (cd realtime && MIX_ENV=prod mix deps.get --only prod && MIX_ENV=prod mix compile && MIX_ENV=prod mix release --overwrite) +fi + +if command -v pnpm >/dev/null 2>&1; then + for app in web admin forms; do + if [[ -d "$PREFIX/$app" ]]; then + log "building $app" + (cd "$app" && pnpm install --frozen-lockfile && pnpm build) + fi + done +fi + +log "installing and restarting (sudo $INSTALLER)" +if [[ "$(id -u)" -eq 0 ]]; then + "$INSTALLER" +else + sudo -n "$INSTALLER" +fi + +log "done: $VERSION" diff --git a/site/README.md b/site/README.md index a44fa923..920ce2d7 100644 --- a/site/README.md +++ b/site/README.md @@ -16,6 +16,32 @@ From the repo root you can also use `make site`, which is a shortcut for `cd site && pnpm dev`. `make app` does **not** start this site — it lives outside the docker compose stack and ships on its own cadence. +## The installer + +`public/install.sh` is the one-command self-host installer, served verbatim at +`https://warmbly.com/install.sh`. It is a static asset, so what ships is exactly +what a `curl | sh` executes. + +Two things the host has to get right: + +- serve it as `text/plain` (or `application/x-sh`), never `text/html`, so + reading it in a browser works and nothing tries to render it +- serve `public/install.sh.sha256` alongside it, unmodified + +Editing it means regenerating the checksum, and CI fails when the two disagree: + +```sh +make installer-sha # regenerate site/public/install.sh.sha256 +make installer-check # everything CI runs: POSIX parse, shellcheck, dry runs +make installer-demo # walk the wizard; installs nothing, needs no Docker +``` + +`make installer-demo` is the fastest way to see a UI change: it runs the real +questions and the real review with the pull, the container creation and the +health wait played, and writes nothing anywhere. It takes about half a minute; +`WARMBLY_DEMO_FAST=1 make installer-demo` cuts that to under ten seconds while +you iterate. + ## Layout ``` diff --git a/site/public/cli.ps1 b/site/public/cli.ps1 new file mode 100644 index 00000000..dcc6033e --- /dev/null +++ b/site/public/cli.ps1 @@ -0,0 +1,268 @@ +<# +.SYNOPSIS + Installs the warmbly CLI on Windows. + +.DESCRIPTION + One static binary, no toolchain, no admin rights. Downloads the archive for + this machine's architecture from the GitHub release, checks it against the + published checksum, unpacks it into a per-user directory and puts that + directory on the user PATH. + + Re-running it upgrades in place. + +.EXAMPLE + irm https://warmbly.com/cli.ps1 | iex + +.EXAMPLE + & ([scriptblock]::Create((irm https://warmbly.com/cli.ps1))) -Version v1.4.0 + +.EXAMPLE + & ([scriptblock]::Create((irm https://warmbly.com/cli.ps1))) -Uninstall + +.LINK + https://docs.warmbly.com/api/cli/ +#> +[CmdletBinding()] +param( + # Where the binary goes. Defaults to a per-user directory so nothing here + # needs an elevated shell. + [string]$Dir = $env:WARMBLY_INSTALL_DIR, + + # A release tag to pin, for example v1.4.0. Defaults to the newest release. + [string]$Version = $env:WARMBLY_CLI_VERSION, + + # Download from a mirror of the release assets instead of GitHub, for an + # egress-restricted network. + [string]$BaseUrl = $env:WARMBLY_CLI_BASE_URL, + + # Leave the user PATH alone. + [switch]$NoModifyPath, + + # Print what would happen and change nothing. + [switch]$DryRun, + + # Remove the binary and its PATH entry. + [switch]$Uninstall, + + # Reinstall even when the version already matches. + [switch]$Force +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$Repo = 'warmbly/warmbly' +$Releases = "https://github.com/$Repo/releases" +$Docs = 'https://docs.warmbly.com/api/cli/' + +function Write-Step { param($m) Write-Host "> $m" -ForegroundColor Cyan } +function Write-Ok { param($m) Write-Host "✓ $m" -ForegroundColor Green } +function Write-Warn { param($m) Write-Host "! $m" -ForegroundColor Yellow } +function Write-Fail { param($m) Write-Host "✗ $m" -ForegroundColor Red; exit 1 } + +# Windows on ARM runs amd64 binaries under emulation, but a native build is +# published, so the architecture is read rather than assumed. +function Get-Arch { + $arch = $env:PROCESSOR_ARCHITECTURE + if ($env:PROCESSOR_ARCHITEW6432) { $arch = $env:PROCESSOR_ARCHITEW6432 } + switch ($arch) { + 'AMD64' { return 'amd64' } + 'ARM64' { return 'arm64' } + default { + Write-Fail @" +No published build for $arch. +We publish amd64 and arm64. Build from source with: + go install github.com/$Repo/cmd/cli@latest +"@ + } + } +} + +function Get-AssetUrl { + param($Name) + if ($BaseUrl) { return "$($BaseUrl.TrimEnd('/'))/$Name" } + if ($Version) { return "$Releases/download/$Version/$Name" } + return "$Releases/latest/download/$Name" +} + +function Get-InstallDir { + if ($Dir) { return $Dir } + return (Join-Path $env:LOCALAPPDATA 'Warmbly\bin') +} + +# The user PATH is read from the registry rather than from $env:PATH, because +# the session copy already has machine entries merged in and writing that back +# would move machine-wide entries into the user scope. +function Add-ToUserPath { + param($Target) + + $current = [Environment]::GetEnvironmentVariable('Path', 'User') + if ($null -eq $current) { $current = '' } + $entries = $current -split ';' | Where-Object { $_ -ne '' } + + if ($entries -contains $Target) { + Write-Ok "$Target is already on your PATH" + return + } + if ($NoModifyPath) { + Write-Warn "$Target is not on your PATH. Add it yourself, or re-run without -NoModifyPath." + return + } + if ($DryRun) { + Write-Host " would add $Target to the user PATH" + return + } + + $updated = (@($entries) + $Target) -join ';' + [Environment]::SetEnvironmentVariable('Path', $updated, 'User') + # The registry change reaches new processes only, so this session gets the + # entry too. Without it, the very next command in this window fails. + $env:Path = "$env:Path;$Target" + Write-Ok "added $Target to your PATH" + Write-Warn 'Open a new terminal for other programs to see it.' +} + +function Install-Completions { + param($Source) + + $profilePath = $PROFILE.CurrentUserAllHosts + $marker = '# Added by the warmbly CLI installer' + + # Reported before any path is built: the dry run has no unpacked archive, + # and Join-Path on an empty path is a terminating error under Stop. + if ($DryRun) { + Write-Host " would add completions to $profilePath" + return + } + + $completion = Join-Path $Source 'completions\warmbly.powershell' + if (-not (Test-Path $completion)) { return } + if ((Test-Path $profilePath) -and (Select-String -Path $profilePath -Pattern ([regex]::Escape($marker)) -Quiet)) { + return + } + + $dest = Join-Path (Get-InstallDir) 'warmbly.completion.ps1' + Copy-Item $completion $dest -Force + + New-Item -ItemType Directory -Force -Path (Split-Path $profilePath) | Out-Null + Add-Content -Path $profilePath -Value "`n$marker`n. `"$dest`"" + Write-Ok "wrote completions and referenced them from $profilePath" +} + +function Invoke-Uninstall { + $target = Get-InstallDir + $exe = Join-Path $target 'warmbly.exe' + $removed = $false + + if (Test-Path $exe) { + if ($DryRun) { Write-Host "would remove $exe" } + else { Remove-Item $exe -Force; Write-Ok "removed $exe" } + $removed = $true + } + + $completion = Join-Path $target 'warmbly.completion.ps1' + if (Test-Path $completion) { + if (-not $DryRun) { Remove-Item $completion -Force } + $removed = $true + } + + if (-not $DryRun) { + $current = [Environment]::GetEnvironmentVariable('Path', 'User') + if ($current) { + $kept = $current -split ';' | Where-Object { $_ -ne '' -and $_ -ne $target } + [Environment]::SetEnvironmentVariable('Path', ($kept -join ';'), 'User') + } + } + + if (-not $removed) { Write-Warn "nothing to remove: no warmbly.exe in $target" } + + $config = Join-Path $env:APPDATA 'warmbly' + if (Test-Path $config) { + Write-Host '' + Write-Host "Your sign-ins are still in $config." + Write-Host "Remove them with: Remove-Item -Recurse '$config'" + } +} + +function Invoke-Install { + $arch = Get-Arch + $target = Get-InstallDir + $asset = "warmbly_windows_$arch.zip" + + Write-Step 'Installing the warmbly CLI' + Write-Host " platform: windows/$arch" + Write-Host " version: $(if ($Version) { $Version } else { 'latest' })" + Write-Host " into: $target" + Write-Host '' + + $exe = Join-Path $target 'warmbly.exe' + if ((Test-Path $exe) -and $Version -and -not $Force) { + $current = (& $exe version 2>$null | Select-Object -First 1) -split ' ' | Select-Object -Index 1 + if ($current -eq $Version) { + Write-Ok "warmbly $current is already installed in $target" + return + } + } + + if ($DryRun) { + Write-Host "would download $(Get-AssetUrl $asset)" + Write-Host "would verify it against $(Get-AssetUrl 'checksums.txt')" + Write-Host "would install $exe" + Install-Completions '' + Add-ToUserPath $target + return + } + + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("warmbly-" + [guid]::NewGuid()) + New-Item -ItemType Directory -Force -Path $tmp | Out-Null + try { + Write-Step "Downloading $asset" + $zip = Join-Path $tmp $asset + try { + Invoke-WebRequest -Uri (Get-AssetUrl $asset) -OutFile $zip -UseBasicParsing + } catch { + Write-Fail "could not download $(Get-AssetUrl $asset)`nIf you pinned -Version, check the tag exists: $Releases" + } + + # The checksum is why this is safer than a bare download: a truncated + # transfer and a tampered one are indistinguishable to Expand-Archive. + try { + $sums = Join-Path $tmp 'checksums.txt' + Invoke-WebRequest -Uri (Get-AssetUrl 'checksums.txt') -OutFile $sums -UseBasicParsing + $want = (Select-String -Path $sums -Pattern ([regex]::Escape($asset)) | Select-Object -First 1).Line -split '\s+' | Select-Object -First 1 + $got = (Get-FileHash $zip -Algorithm SHA256).Hash.ToLower() + if (-not $want) { + Write-Warn "checksums.txt has no entry for $asset; continuing without verification" + } elseif ($want.ToLower() -ne $got) { + Write-Fail "checksum mismatch for $asset.`n expected $want`n got $got`nNothing was installed." + } else { + Write-Ok 'checksum verified' + } + } catch { + Write-Warn 'could not fetch checksums.txt; continuing without verification' + } + + Write-Step 'Unpacking' + $unpacked = Join-Path $tmp 'x' + Expand-Archive -Path $zip -DestinationPath $unpacked -Force + $source = Join-Path $unpacked 'warmbly.exe' + if (-not (Test-Path $source)) { Write-Fail 'the archive did not contain warmbly.exe' } + + New-Item -ItemType Directory -Force -Path $target | Out-Null + Copy-Item $source $exe -Force + + $installed = (& $exe version 2>$null | Select-Object -First 1) + Write-Ok "installed $installed to $exe" + + Install-Completions $unpacked + Add-ToUserPath $target + + Write-Host '' + Write-Host 'Next: warmbly auth login' + Write-Host "Docs: $Docs" + } finally { + Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue + } +} + +if ($Uninstall) { Invoke-Uninstall } else { Invoke-Install } diff --git a/site/public/cli.sh b/site/public/cli.sh new file mode 100644 index 00000000..4ba50455 --- /dev/null +++ b/site/public/cli.sh @@ -0,0 +1,555 @@ +#!/bin/sh +# +# curl -fsSL https://warmbly.com/cli.sh | sh +# +# Installs the `warmbly` CLI on this machine: one static binary, no Go +# toolchain, no package manager, no root. It downloads the archive for your +# platform from the GitHub release, checks it against the published checksum, +# and puts the binary somewhere on your PATH. +# +# sh cli.sh --help every flag, and the environment variable for each +# sh cli.sh --dry-run print exactly what it would do, touch nothing +# sh cli.sh --uninstall remove the binary and the completions it wrote +# +# What it does, in full: +# +# * detects your OS and CPU, and stops with a real message if we publish no +# build for it rather than downloading something that cannot run +# * resolves the newest release (or the one you pin with --version) +# * downloads warmbly__.tar.gz and checksums.txt, and REFUSES to +# install if the two disagree +# * installs to ~/.local/bin by default, which needs no sudo. Nothing else on +# your system is touched +# * writes shell completions, and tells you the one line to add to your shell +# profile if the install directory is not already on PATH +# +# Re-running it upgrades in place and says so when there is nothing to do. +# +# Verify before running, if you would rather: +# +# curl -fsSLO https://warmbly.com/cli.sh +# curl -fsSLO https://warmbly.com/cli.sh.sha256 +# sha256sum -c cli.sh.sha256 +# less cli.sh && sh cli.sh +# +# https://docs.warmbly.com/api/cli/ + +set -eu + +# ───────────────────────────────────────────────────────────────────────── +# Constants +# ───────────────────────────────────────────────────────────────────────── + +REPO="warmbly/warmbly" +BIN="warmbly" +DOCS="https://docs.warmbly.com/api/cli/" +RELEASES="https://github.com/${REPO}/releases" + +# Every platform scripts/build-cli.sh publishes. The two lists have to agree: +# a platform here with no archive downloads a 404, and one missing here is a +# build nobody can install. +PLATFORMS="darwin_amd64 darwin_arm64 linux_amd64 linux_arm64" + +# ───────────────────────────────────────────────────────────────────────── +# Options. Every one is also an environment variable, so the same install runs +# from Ansible, cloud-init, a Dockerfile or an agent with no keyboard. +# ───────────────────────────────────────────────────────────────────────── + +DIR=${WARMBLY_INSTALL_DIR:-} +VERSION=${WARMBLY_CLI_VERSION:-} +# Where the archives come from. Overridable so an air-gapped or +# egress-restricted network can mirror the release assets internally and still +# use this exact script. +BASE_URL=${WARMBLY_CLI_BASE_URL:-} +NO_MODIFY_PATH=${WARMBLY_NO_MODIFY_PATH:-} +NO_COMPLETIONS=${WARMBLY_NO_COMPLETIONS:-} +DRY_RUN="" +UNINSTALL="" +FORCE="" +QUIET="" +USE_COLOR=1 + +# ───────────────────────────────────────────────────────────────────────── +# Output +# +# The colour variables are defined empty here rather than only in +# setup_colors, because parse_args runs first and can call die: under `set -u` +# an unset C_RED turns a "unknown option" message into an unbound-variable +# error, which is what a mistyped flag would have printed. +# ───────────────────────────────────────────────────────────────────────── + +C_RESET=""; C_DIM=""; C_BOLD=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" + +setup_colors() { + if [ -n "$USE_COLOR" ] && [ -t 2 ] && [ "${TERM:-dumb}" != "dumb" ] && [ -z "${NO_COLOR:-}" ]; then + C_RESET=$(printf '\033[0m') + C_DIM=$(printf '\033[2m') + C_BOLD=$(printf '\033[1m') + C_RED=$(printf '\033[31m') + C_GREEN=$(printf '\033[32m') + C_YELLOW=$(printf '\033[33m') + C_CYAN=$(printf '\033[36m') + else + C_RESET=""; C_DIM=""; C_BOLD=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" + fi +} + +say() { [ -n "$QUIET" ] || printf '%s\n' "$*" >&2; } +step() { [ -n "$QUIET" ] || printf '%s>%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +ok() { [ -n "$QUIET" ] || printf '%s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +warn() { printf '%s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +die() { printf '%s✗%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; } + +usage() { + cat <&2; die "unknown option $1" ;; + esac + shift + done +} + +# ───────────────────────────────────────────────────────────────────────── +# Platform +# ───────────────────────────────────────────────────────────────────────── + +detect_platform() { + os=$(uname -s 2>/dev/null || echo unknown) + arch=$(uname -m 2>/dev/null || echo unknown) + + case $os in + Linux) OS=linux ;; + Darwin) OS=darwin ;; + MINGW*|MSYS*|CYGWIN*) + die "this script installs the Unix build. +On Windows run this in PowerShell instead: + irm https://warmbly.com/cli.ps1 | iex" ;; + *) die "no published build for $os. Build from source with: go install github.com/${REPO}/cmd/cli@latest" ;; + esac + + case $arch in + x86_64|amd64) ARCH=amd64 ;; + arm64|aarch64) ARCH=arm64 ;; + *) die "no published build for $arch on $OS. +We publish amd64 and arm64. Build from source with: + go install github.com/${REPO}/cmd/cli@latest" ;; + esac + + TARGET="${OS}_${ARCH}" + for known in $PLATFORMS; do + if [ "$known" = "$TARGET" ]; then + return 0 + fi + done + die "no published build for $TARGET" +} + +# fetch writes a URL to a file. curl and wget are both accepted because a +# minimal container image has exactly one of them and it is never the one you +# assumed. +fetch() { + url=$1 + dest=$2 + if [ -n "$DOWNLOADER" ] && [ "$DOWNLOADER" = curl ]; then + curl -fsSL --retry 3 --retry-delay 1 -o "$dest" "$url" + else + wget -q -O "$dest" "$url" + fi +} + +require_downloader() { + if command -v curl >/dev/null 2>&1; then + DOWNLOADER=curl + elif command -v wget >/dev/null 2>&1; then + DOWNLOADER=wget + else + die "neither curl nor wget is installed, so there is nothing to download with" + fi +} + +# sha256_of prints a file's checksum with whichever tool the host has. macOS +# ships shasum, Linux ships sha256sum, Alpine ships both or neither. +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + elif command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 "$1" | awk '{print $NF}' + else + echo "" + fi +} + +# ───────────────────────────────────────────────────────────────────────── +# Install directory +# ───────────────────────────────────────────────────────────────────────── + +# A literal tilde is what this matches: someone who typed --dir '~/bin' inside +# quotes meant their home directory, not a folder called "~". +# shellcheck disable=SC2088 +expand_tilde() { + case $DIR in + "~/"*) DIR="${HOME}/${DIR#\~/}" ;; + esac +} + +# resolve_dir picks where the binary goes. ~/.local/bin is the default because +# it needs no sudo and is on PATH by default on most modern distributions; +# piping an installer into a shell should never need root. +resolve_dir() { + if [ -z "$DIR" ]; then + DIR="${HOME:-/root}/.local/bin" + fi + expand_tilde +} + +# on_path answers whether DIR is already searched, so we only talk about shell +# profiles when there is a real problem to solve. +on_path() { + case ":${PATH}:" in + *":${DIR}:"*) return 0 ;; + *) return 1 ;; + esac +} + +# profile_file is the file a login shell reads, chosen from $SHELL rather than +# the shell running this script: this runs under sh no matter what the person +# actually uses. +profile_file() { + shell_name=$(basename "${SHELL:-sh}") + case $shell_name in + zsh) printf '%s' "${ZDOTDIR:-$HOME}/.zshrc" ;; + bash) + if [ -f "$HOME/.bashrc" ]; then + printf '%s' "$HOME/.bashrc" + else + printf '%s' "$HOME/.bash_profile" + fi ;; + fish) printf '%s' "$HOME/.config/fish/config.fish" ;; + *) printf '%s' "$HOME/.profile" ;; + esac +} + +# The single quotes below are the point: $PATH has to reach the profile +# unexpanded, so it still resolves every time the shell reads it. +# shellcheck disable=SC2016 +path_line() { + shell_name=$(basename "${SHELL:-sh}") + case $shell_name in + fish) printf 'fish_add_path %s' "$DIR" ;; + *) printf 'export PATH="%s:$PATH"' "$DIR" ;; + esac +} + +# ensure_on_path appends the PATH line to the right profile, once. The marker +# comment is what makes a second run a no-op instead of a growing file. +ensure_on_path() { + if on_path; then + return 0 + fi + line=$(path_line) + if [ -n "$NO_MODIFY_PATH" ]; then + warn "$DIR is not on your PATH. Add this yourself:" + say " $line" + return 0 + fi + + profile=$(profile_file) + if [ -n "$DRY_RUN" ]; then + say " would add to $profile: $line" + return 0 + fi + + if [ -f "$profile" ] && grep -q "warmbly CLI" "$profile" 2>/dev/null; then + ok "$profile already has the PATH line" + else + mkdir -p "$(dirname "$profile")" + { + printf '\n# Added by the warmbly CLI installer\n' + printf '%s\n' "$line" + } >> "$profile" + ok "added $DIR to your PATH in $profile" + fi + warn "open a new terminal, or run: $line" +} + +# ───────────────────────────────────────────────────────────────────────── +# Completions +# ───────────────────────────────────────────────────────────────────────── + +# completion_dir is where the shell looks without any configuration. When there +# is no such place we say nothing rather than writing a file that is never read. +completion_dir() { + shell_name=$(basename "${SHELL:-sh}") + case $shell_name in + bash) + if [ -d "$HOME/.local/share/bash-completion/completions" ] || [ "$1" = create ]; then + printf '%s' "$HOME/.local/share/bash-completion/completions/warmbly" + fi ;; + zsh) + printf '%s' "${ZDOTDIR:-$HOME}/.zfunc/_warmbly" ;; + fish) + printf '%s' "$HOME/.config/fish/completions/warmbly.fish" ;; + *) printf '' ;; + esac +} + +install_completions() { + if [ -n "$NO_COMPLETIONS" ]; then + return 0 + fi + shell_name=$(basename "${SHELL:-sh}") + src="" + case $shell_name in + bash) src="$1/completions/warmbly.bash" ;; + zsh) src="$1/completions/warmbly.zsh" ;; + fish) src="$1/completions/warmbly.fish" ;; + *) return 0 ;; + esac + + dest=$(completion_dir create) + [ -n "$dest" ] || return 0 + + # The dry run has no unpacked archive to copy from, so it reports the + # destination rather than testing for a source that cannot exist yet. + if [ -n "$DRY_RUN" ]; then + say " would write $shell_name completions to $dest" + return 0 + fi + [ -f "$src" ] || return 0 + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + ok "wrote $shell_name completions to $dest" + if [ "$shell_name" = zsh ]; then + say " ${C_DIM}zsh needs ~/.zfunc on its fpath: add \`fpath+=~/.zfunc\` above compinit${C_RESET}" + fi + return 0 +} + +# ───────────────────────────────────────────────────────────────────────── +# Uninstall +# ───────────────────────────────────────────────────────────────────────── + +do_uninstall() { + resolve_dir + target="$DIR/$BIN" + removed="" + + if [ -f "$target" ]; then + if [ -n "$DRY_RUN" ]; then + say "would remove $target" + else + rm -f "$target" + ok "removed $target" + fi + removed=1 + fi + + for c in "$HOME/.local/share/bash-completion/completions/warmbly" \ + "${ZDOTDIR:-$HOME}/.zfunc/_warmbly" \ + "$HOME/.config/fish/completions/warmbly.fish"; do + if [ -f "$c" ]; then + if [ -n "$DRY_RUN" ]; then + say "would remove $c" + else + rm -f "$c" + ok "removed $c" + fi + removed=1 + fi + done + + if [ -z "$removed" ]; then + warn "nothing to remove: no warmbly found in $DIR" + fi + + # Deliberately left alone: it holds the credentials, and someone + # reinstalling in a minute should not have to sign in again. + if [ -d "${XDG_CONFIG_HOME:-$HOME/.config}/warmbly" ]; then + say "" + say "Your sign-ins are still in ${XDG_CONFIG_HOME:-$HOME/.config}/warmbly." + say "Remove them with: rm -rf ${XDG_CONFIG_HOME:-$HOME/.config}/warmbly" + fi + return 0 +} + +# ───────────────────────────────────────────────────────────────────────── +# Install +# ───────────────────────────────────────────────────────────────────────── + +# archive_url builds the download URL. The version-less asset names are what +# let "latest" resolve with no GitHub API call, so the install works on a CI +# runner whose IP has already spent the unauthenticated rate limit. +archive_url() { + name=$1 + if [ -n "$BASE_URL" ]; then + printf '%s/%s' "${BASE_URL%/}" "$name" + elif [ -n "$VERSION" ]; then + printf '%s/download/%s/%s' "$RELEASES" "$VERSION" "$name" + else + printf '%s/latest/download/%s' "$RELEASES" "$name" + fi +} + +installed_version() { + if [ -x "$DIR/$BIN" ]; then + "$DIR/$BIN" version 2>/dev/null | head -1 | awk '{print $2}' + fi +} + +do_install() { + detect_platform + resolve_dir + + archive="warmbly_${TARGET}.tar.gz" + url=$(archive_url "$archive") + sums_url=$(archive_url "checksums.txt") + + step "Installing the warmbly CLI" + say " platform: ${OS}/${ARCH}" + say " version: ${VERSION:-latest}" + say " into: ${DIR}" + say "" + + current=$(installed_version) + if [ -n "$current" ] && [ -z "$FORCE" ] && [ -n "$VERSION" ] && [ "$current" = "$VERSION" ]; then + ok "warmbly $current is already installed in $DIR" + say " ${C_DIM}--force reinstalls it anyway${C_RESET}" + return 0 + fi + + if [ -n "$DRY_RUN" ]; then + say "would download $url" + say "would verify it against $sums_url" + say "would install $DIR/$BIN" + install_completions "" || true + ensure_on_path + return 0 + fi + + tmp=$(mktemp -d 2>/dev/null || mktemp -d -t warmbly) + # The trap is set before the first write, so an interrupted install leaves + # nothing behind in /tmp. + trap 'rm -rf "$tmp"' EXIT INT TERM + + step "Downloading $archive" + if ! fetch "$url" "$tmp/$archive"; then + die "could not download $url +If you pinned --version, check the tag exists: ${RELEASES}" + fi + + # The checksum is the whole reason this is safer than a bare curl into tar: + # a truncated download and a tampered one look the same to tar. + if fetch "$sums_url" "$tmp/checksums.txt" 2>/dev/null; then + want=$(awk -v f="$archive" '$2 == f || $2 == "*"f { print $1 }' "$tmp/checksums.txt" | head -1) + got=$(sha256_of "$tmp/$archive") + if [ -z "$want" ]; then + warn "checksums.txt has no entry for $archive; continuing without verification" + elif [ -z "$got" ]; then + warn "no sha256 tool on this machine, so the download was not verified" + elif [ "$want" != "$got" ]; then + die "checksum mismatch for $archive. + expected $want + got $got +Nothing was installed. Try again, and if it happens twice report it: ${RELEASES}" + else + ok "checksum verified" + fi + else + warn "could not fetch checksums.txt; continuing without verification" + fi + + step "Unpacking" + mkdir -p "$tmp/x" + tar -xzf "$tmp/$archive" -C "$tmp/x" || die "the archive could not be unpacked" + [ -f "$tmp/x/$BIN" ] || die "the archive did not contain $BIN" + + mkdir -p "$DIR" 2>/dev/null || die "could not create $DIR. +Pick somewhere writable with --dir, for example: --dir \$HOME/bin" + + # install(1) is not on every minimal image, so this is cp plus chmod, done + # to a temporary name and moved into place: replacing a running binary with + # a rename is atomic, overwriting one in place is not. + cp "$tmp/x/$BIN" "$DIR/.$BIN.new" || die "could not write to $DIR. +Pick somewhere writable with --dir, or re-run with sudo if $DIR is system-owned." + chmod 0755 "$DIR/.$BIN.new" + mv -f "$DIR/.$BIN.new" "$DIR/$BIN" + + version_now=$("$DIR/$BIN" version 2>/dev/null | head -1 || echo "") + ok "installed ${version_now:-warmbly} to $DIR/$BIN" + + install_completions "$tmp/x" || true + ensure_on_path + + say "" + say "${C_BOLD}Next:${C_RESET} warmbly auth login" + say "${C_DIM}Docs: ${DOCS}${C_RESET}" + return 0 +} + +main() { + parse_args "$@" + setup_colors + require_downloader + + if [ -n "$UNINSTALL" ]; then + do_uninstall + return 0 + fi + do_install +} + +main "$@" diff --git a/site/public/cli.sh.sha256 b/site/public/cli.sh.sha256 new file mode 100644 index 00000000..74d87d0e --- /dev/null +++ b/site/public/cli.sh.sha256 @@ -0,0 +1 @@ +bc129c17774a4c45bb08c5d6228beedf199ce384aa3a16cb798ebe5c75e1861b cli.sh diff --git a/site/public/install.sh b/site/public/install.sh new file mode 100644 index 00000000..2a250be9 --- /dev/null +++ b/site/public/install.sh @@ -0,0 +1,3081 @@ +#!/bin/sh +# +# curl -fsSL https://warmbly.com/install.sh | sh +# +# Installs Warmbly on this machine: pull the published release images, write a +# real .env and a compose file, start the stack, print the link that claims it. +# No git clone, no Go, Rust, Node or Elixir toolchain, nothing compiled here. +# +# sh install.sh --wizard ask where the data lives, what is kept and for +# how long, and how it is backed up +# sh install.sh --dry-run print the exact files it would write, touch nothing +# sh install.sh --help every flag, and the environment variable for each +# +# On reading this before running it: that is the right instinct, and it is why +# the checksum is published next to it. The short version of what it does: +# +# * checks for docker, and offers to install it rather than doing so quietly +# * creates one directory (default /opt/warmbly) and writes only inside it +# * generates five secrets with openssl, writes them 0600, prints the two +# that are unrecoverable and waits for you to say you have copied them +# * resolves the newest release once and PINS it in .env, so every later +# `docker compose up` in that directory is the same version. Never :latest +# * runs `docker compose pull` and `docker compose up -d` +# +# It asks for sudo only for the steps that need it, names them when it does, +# and never wants to be piped into sudo itself. Re-running it reconfigures in +# place: it does not regenerate secrets, does not touch an existing data root, +# and refuses a directory it did not create unless you pass --force. +# +# Verify before running, if you would rather: +# +# curl -fsSLO https://warmbly.com/install.sh +# curl -fsSLO https://warmbly.com/install.sh.sha256 +# sha256sum -c install.sh.sha256 +# less install.sh && sh install.sh +# +# https://docs.warmbly.com/development/install/ + +set -eu + +# ───────────────────────────────────────────────────────────────────────── +# Constants +# ───────────────────────────────────────────────────────────────────────── + +SCRIPT_NAME="Warmbly installer" +REPO="warmbly/warmbly" +DEFAULT_REGISTRY="ghcr.io/warmbly/warmbly" +DEFAULT_DIR="/opt/warmbly" +DOCS="https://docs.warmbly.com" +RELEASES_API="https://api.github.com/repos/${REPO}/releases" + +# The marker file that says this directory is ours. Its absence in a non-empty +# directory is what --force overrides, so the installer can never take over a +# path that belongs to something else by accident. +MARKER=".warmbly-install" + +# Ports the stack publishes, in the order they are checked for collisions. +# 3000 is the one that actually collides: half of self-hosted software wants it. +PORT_BACKEND=8080 +PORT_WEB=5173 +PORT_ADMIN=5174 +PORT_TRACKING=3000 +PORT_REALTIME=4000 +PORT_FORMS=8090 + +# ───────────────────────────────────────────────────────────────────────── +# Answers. Every one is a flag, an environment variable and a wizard question, +# and the defaults here are exactly what the fast path installs. +# ───────────────────────────────────────────────────────────────────────── + +DIR="${WARMBLY_DIR:-$DEFAULT_DIR}" +HOSTNAME_ANSWER="${WARMBLY_HOST:-localhost}" +TLS="${WARMBLY_TLS:-none}" +DATA_ROOT="${WARMBLY_DATA_ROOT:-}" # empty = /data; "volumes" = docker named volumes +BLOBS="${WARMBLY_BLOBS:-filesystem}" # filesystem | s3 +COMPONENTS="${WARMBLY_COMPONENTS:-full}" # core | full +VERSION="${WARMBLY_VERSION:-}" # empty = resolve the newest release +CHANNEL="${WARMBLY_CHANNEL:-stable}" +REGISTRY="${WARMBLY_IMAGE_PREFIX:-$DEFAULT_REGISTRY}" +EXTERNAL_DB="${WARMBLY_DATABASE_URL:-}" +EXTERNAL_REDIS="${WARMBLY_REDIS_URL:-}" +MAIL_MODE="${WARMBLY_MAIL:-log}" # log | smtp +REGISTRATION="${WARMBLY_REGISTRATION:-invite_only}" +UPDATE_CHECK="${WARMBLY_UPDATE_CHECK:-true}" +BOOTSTRAP_EMAIL="${WARMBLY_BOOTSTRAP_EMAIL:-}" +BOOTSTRAP_HASH="${WARMBLY_BOOTSTRAP_PASSWORD_HASH:-}" +BACKUP_DIR="${WARMBLY_BACKUP_DIR:-}" # empty = no scheduled backup +BACKUP_KEEP="${WARMBLY_BACKUP_KEEP:-14}" +BACKUP_SCHEDULE="${WARMBLY_BACKUP_SCHEDULE:-daily}" +BACKUP_KEYS="${WARMBLY_BACKUP_KEYS:-true}" +BACKUP_S3="${WARMBLY_BACKUP_S3:-}" + +# Retention and sync, the data-control answers. They are written into .env as +# one WARMBLY_SETTINGS_BOOTSTRAP document, applied on first boot and editable +# in Instance > Instance settings from then on. +SYNC_BACKFILL_DAYS="${WARMBLY_SYNC_BACKFILL_DAYS:-90}" +SYNC_BACKFILL_MESSAGES="${WARMBLY_SYNC_BACKFILL_MESSAGES:-5000}" +SYNC_DAILY_MAILBOX="${WARMBLY_SYNC_DAILY_MAILBOX:-2000}" +SYNC_DAILY_ORG="${WARMBLY_SYNC_DAILY_ORG:-25000}" +RET_ENGAGEMENT="${WARMBLY_RETENTION_ENGAGEMENT_DAYS:-365}" +RET_FORMS="${WARMBLY_RETENTION_FORM_DAYS:-180}" +RET_AUDIT="${WARMBLY_RETENTION_AUDIT_DAYS:-90}" + +# S3 blob answers, only read when BLOBS=s3. +S3_BUCKET="${WARMBLY_S3_BUCKET:-}" +S3_ENDPOINT="${WARMBLY_S3_ENDPOINT:-}" +S3_REGION="${WARMBLY_S3_REGION:-auto}" +S3_KEY="${WARMBLY_S3_ACCESS_KEY_ID:-}" +S3_SECRET="${WARMBLY_S3_SECRET_ACCESS_KEY:-}" + +# SMTP answers, only read when MAIL_MODE=smtp. +SMTP_HOST=""; SMTP_PORT=""; SMTP_USER=""; SMTP_PASS=""; SMTP_SECURITY="starttls" +SMTP_FROM="" + +# Reverse-proxy answer, only read when TLS=proxy. +PROXY_CIDRS="${WARMBLY_TRUSTED_PROXIES:-172.16.0.0/12,10.0.0.0/8,192.168.0.0/16}" + +# ───────────────────────────────────────────────────────────────────────── +# Modes +# ───────────────────────────────────────────────────────────────────────── + +MODE=install # install | uninstall +WIZARD=0 +DEMO=0 +ASSUME_YES=0 +DRY_RUN=0 +PRINT_ENV=0 +FORCE=0 +PURGE_DATA=0 +USE_COLOR=1 +# Appending is the default. The wizard reads as a transcript, nothing above it +# is touched, and there is no way for a redraw to land on top of something +# else. --clear opts into the full-screen version. +USE_CLEAR=0 +INTERACTIVE=0 +TTY=/dev/tty + +# Filled in as the run proceeds. +RESOLVED_TAG="" +COMPOSE="docker compose" +SUDO="" +EXISTING=0 + +# The palette starts empty so that anything reachable before setup_term (--help, +# an unknown flag) still prints under set -u. setup_term fills it in. +ESC=""; R=""; B=""; DIM=""; SKY=""; GREEN=""; AMBER="" +RED=""; GREY=""; WHITE=""; HIDE=""; SHOW="" +COLS=80; WIDTH=76 + +# ───────────────────────────────────────────────────────────────────────── +# Terminal: colour, cursor, and the drawing primitives everything else uses. +# +# Every escape sequence goes through these, so NO_COLOR, a pipe, a dumb +# terminal and --no-color all degrade to plain text in one place instead of +# leaking half-rendered ANSI into a log file. +# ───────────────────────────────────────────────────────────────────────── + +setup_term() { + # A wizard needs a keyboard. Piped into sh, stdin is the script itself, so + # the terminal is reopened explicitly; without one, only the answered + # (flag or environment) path can run. + if [ -t 0 ] && [ -t 1 ]; then + INTERACTIVE=1 + TTY=/dev/stdin + elif [ -c /dev/tty ] && ( exec 9<>/dev/tty ) 2>/dev/null && [ -t 1 ]; then + INTERACTIVE=1 + TTY=/dev/tty + fi + + if [ -n "${NO_COLOR:-}" ] || [ "${TERM:-dumb}" = "dumb" ] || [ ! -t 1 ]; then + USE_COLOR=0 + fi + if [ "$USE_COLOR" = 1 ]; then + ESC=$(printf '\033') + R="${ESC}[0m"; B="${ESC}[1m"; DIM="${ESC}[2m" + SKY="${ESC}[38;5;39m" + GREEN="${ESC}[38;5;42m"; AMBER="${ESC}[38;5;214m"; RED="${ESC}[38;5;203m" + GREY="${ESC}[38;5;245m"; WHITE="${ESC}[38;5;255m" + HIDE="${ESC}[?25l"; SHOW="${ESC}[?25h" + else + ESC=""; R=""; B=""; DIM=""; SKY=""; GREEN=""; AMBER="" + RED=""; GREY=""; WHITE=""; HIDE=""; SHOW="" + USE_CLEAR=0 + fi + + # Clearing a terminal that is not ours to clear is never right. + [ "$INTERACTIVE" = 1 ] || USE_CLEAR=0 + + # Width. Everything drawn inside a redraw loop is fitted to this, so + # getting it wrong is what makes a menu draw over itself. Three sources, + # because tput is missing on a minimal image and stty needs the terminal: + # an exported COLUMNS wins, since that is how a caller says so explicitly. + COLS="" + case "${COLUMNS:-}" in ''|*[!0-9]*) ;; *) COLS=$COLUMNS ;; esac + if [ -z "$COLS" ] && have tput; then + COLS=$(tput cols 2>/dev/null || true) + fi + if [ -z "$COLS" ] && [ "$INTERACTIVE" = 1 ]; then + COLS=$(stty size <"$TTY" 2>/dev/null | cut -d' ' -f2 || true) + fi + case "$COLS" in ''|*[!0-9]*) COLS=80 ;; esac + # Clamped: a 300 column terminal should not get a 300 column rule, and + # below the floor the boxes stop being boxes. + [ "$COLS" -lt 44 ] && COLS=44 + [ "$COLS" -gt 92 ] && COLS=92 + WIDTH=$((COLS - 4)) +} + +say() { printf '%s\n' "$*"; } +out() { printf '%b\n' "$*"; } +outn() { printf '%b' "$*"; } + +# Restore the cursor whatever happens: a Ctrl-C inside a spinner would +# otherwise leave the operator's terminal without one. +cleanup_term() { + [ "$USE_COLOR" = 1 ] && printf '%b' "$SHOW" + stty_sane +} +stty_sane() { + [ "$INTERACTIVE" = 1 ] || return 0 + stty sane <"$TTY" 2>/dev/null || true +} +trap 'cleanup_term' EXIT +trap 'cleanup_term; printf "\n"; die "cancelled"' INT + +# clear_screen is opt-in (--clear). Wiping someone's terminal is a thing to be +# asked for, not assumed, and a clear that the terminal declines leaves the +# wizard drawing over whatever was already there. ESC[3J is deliberately not +# sent: it erases the scrollback buffer, and the conversation above this +# command is not ours to delete. +clear_screen() { + [ "$USE_CLEAR" = 1 ] || return 0 + printf '%b' "${ESC}[H${ESC}[2J" +} + +repeat_char() { + # $1 char, $2 count. printf pads with spaces and sed swaps them for the + # character: no loop and no fork per character, which is what keeps the + # redraws smooth. sed rather than tr because tr maps byte to byte and every + # box-drawing character here is three bytes of UTF-8. + _n=$2 + [ "$_n" -lt 1 ] && { printf ''; return; } + printf "%${_n}s" "" | sed "s/ /$1/g" +} + +rule() { out "${DIM}$(repeat_char '─' "$WIDTH")${R}"; } + +# fit truncates plain text to n columns. +# +# It is what keeps the in-place redraws honest: a line longer than the terminal +# wraps onto a second physical row, and every cursor-up in this script counts +# LOGICAL rows, so one long option hint is enough to make a menu redraw over +# itself and over whatever was on screen before it. Applied to everything drawn +# inside a loop; prose printed once is free to wrap. +fit() { + _t=$1; _w=$2 + [ "$_w" -lt 12 ] && _w=12 + if [ "${#_t}" -le "$_w" ]; then + printf '%s' "$_t" + return 0 + fi + printf '%.*s...' "$((_w - 3))" "$_t" +} + +# A rounded box. Content comes on stdin, one line per row, already coloured. +# The border colour is $1. +box() { + _c="${1:-$DIM}" + out "${_c}╭$(repeat_char '─' $((WIDTH - 2)))╮${R}" + while IFS= read -r _line; do + _plain=$(strip_ansi "$_line") + # A content line wider than the box turns the border into a ragged + # edge. Truncating costs the line its colour, which is a better trade + # than a box that is not one. + if [ "${#_plain}" -gt $((WIDTH - 4)) ]; then + _line=$(fit "$_plain" $((WIDTH - 4))) + _plain=$_line + fi + _pad=$((WIDTH - 4 - ${#_plain})) + [ "$_pad" -lt 0 ] && _pad=0 + printf '%b %s%s %b\n' "${_c}│${R}" "$_line" "$(repeat_char ' ' "$_pad")" "${_c}│${R}" + done + out "${_c}╰$(repeat_char '─' $((WIDTH - 2)))╯${R}" +} + +# strip_ansi is what makes the borders line up: the padding is computed on the +# printable text, not on the byte length of a coloured string. ${#} counts +# bytes in a POSIX shell, so box CONTENT is kept ASCII; the border itself is +# printed rather than measured, and may be anything. +strip_ansi() { + printf '%s' "$1" | sed "s/$(printf '\033')\[[0-9;]*m//g" +} + +# ───────────────────────────────────────────────────────────────────────── +# The wordmark. Drawn once at the top of the run, one row at a time so it +# arrives rather than appears. Falls back to a single bold line when the +# terminal is not ours to animate. +# ───────────────────────────────────────────────────────────────────────── + +banner() { + if [ "$USE_COLOR" = 0 ] || [ "$COLS" -lt 60 ]; then + # The block wordmark is 56 columns wide, so on anything narrower it + # wraps into noise. One line says the same thing. + say "" + out " ${B}${SKY}WARMBLY${R} ${DIM}installer${R}" + say "" + return + fi + printf '%b' "$HIDE" + say "" + _i=0 + printf '%s\n' \ +"██ ██ █████ ██████ ███ ███ ██████ ██ ██ ██" \ +"██ █ ██ ██ ██ ██ ██ ████ ████ ██ ██ ██ ██ ██" \ +"██ ███ ██ ███████ ██████ ██ ████ ██ ██████ ██ ███ " \ +"████ ████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ " \ +" ██ ██ ██ ██ ██ ██ ██ ██ ██████ ██████ ██ " \ + | while IFS= read -r _row; do + _i=$((_i + 1)) + case "$_i" in + 1|2) _tone="${ESC}[38;5;33m" ;; + 3) _tone="${ESC}[38;5;39m" ;; + *) _tone="${ESC}[38;5;74m" ;; + esac + printf ' %b%s%b\n' "$_tone" "$_row" "$R" + nap 0.03 + done + printf ' %b%s%b\n\n' "$DIM" "email warmup and cold outreach, on your own machine" "$R" + printf '%b' "$SHOW" +} + +# nap sleeps a fraction of a second where the shell can, and not at all where +# it cannot. Animation is a nicety; a busybox sleep that rejects "0.03" must +# not turn into an error under set -e. +nap() { + [ "$USE_COLOR" = 1 ] || return 0 + sleep "$1" 2>/dev/null || true +} + +# ───────────────────────────────────────────────────────────────────────── +# Messages +# ───────────────────────────────────────────────────────────────────────── + +info() { out " ${SKY}·${R} $*"; } +ok() { out " ${GREEN}✓${R} $*"; } +warn() { out " ${AMBER}!${R} $*"; } +# note is the explanatory prose under a question. It is wrapped to the +# terminal rather than hand-wrapped to 72 columns, because the hand-wrapped +# version is ragged on anything narrower and this is the only text in the +# script long enough to care. +note() { + if [ "$COLS" -ge 78 ] || ! have fold; then + out " ${DIM}$*${R}" + return 0 + fi + printf '%s\n' "$*" | fold -s -w $((COLS - 6)) | while IFS= read -r _l; do + out " ${DIM}${_l}${R}" + done +} +step() { out "\n ${B}$*${R}"; } + +die() { + cleanup_term + out "\n ${RED}✗${R} ${B}$*${R}\n" >&2 + exit 1 +} + +# fail_with prints the error and, under it, what to do about it. An installer +# that stops without a next step is a support ticket. +fail_with() { + _msg=$1; shift + cleanup_term + out "\n ${RED}✗${R} ${B}${_msg}${R}" >&2 + for _l in "$@"; do out " ${DIM}${_l}${R}" >&2; done + out "" >&2 + exit 1 +} + +# ───────────────────────────────────────────────────────────────────────── +# The stepper: where the wizard is, drawn at the top of every question. +# ───────────────────────────────────────────────────────────────────────── + +STEP_TITLES="Where it lives|How it is reached|Where the data sits|Keys and secrets|What is kept|Backups|Who gets in|Footprint|Review" +STEP_TOTAL=9 +STEP_CURRENT=0 + +stepper() { + STEP_CURRENT=$1 + if [ "$USE_CLEAR" = 1 ]; then + clear_screen + else + # Appending, so the steps need their own separator or they run into + # each other and into whatever was on screen before the installer. + say "" + rule + fi + if [ "$USE_COLOR" = 0 ]; then + say "" + say "Step ${STEP_CURRENT} of ${STEP_TOTAL}: $(step_title "$STEP_CURRENT")" + say "" + return + fi + say "" + _line=" " + _i=1 + while [ "$_i" -le "$STEP_TOTAL" ]; do + if [ "$_i" -lt "$STEP_CURRENT" ]; then + _line="${_line}${GREEN}●${R}" + elif [ "$_i" -eq "$STEP_CURRENT" ]; then + _line="${_line}${SKY}◉${R}" + else + _line="${_line}${DIM}○${R}" + fi + [ "$_i" -lt "$STEP_TOTAL" ] && _line="${_line}${DIM}──${R}" + _i=$((_i + 1)) + done + out "$_line ${DIM}step ${STEP_CURRENT}/${STEP_TOTAL}${R}" + out " ${B}${WHITE}$(step_title "$STEP_CURRENT")${R}" + say "" +} + +step_title() { + printf '%s' "$STEP_TITLES" | cut -d'|' -f"$1" +} + +# ───────────────────────────────────────────────────────────────────────── +# Spinner. Runs a command in the background and animates until it finishes, +# then leaves one settled line behind: a tick, or a cross and the log. +# ───────────────────────────────────────────────────────────────────────── + +SPIN_FRAMES="⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" + +# spin
Triggers

- Nine moments it can fire. + Twelve moments it can fire.

Every automation starts from one event. Replies arrive already classified, meetings arrive matched to the contact, and a campaign step can launch an automation for each contact that reaches it. diff --git a/site/src/pages/campaigns.astro b/site/src/pages/campaigns.astro index 367d9eb2..ea90e672 100644 --- a/site/src/pages/campaigns.astro +++ b/site/src/pages/campaigns.astro @@ -72,7 +72,7 @@ const autoStop = [ ['Replied', 'The reply pulls the contact off the sequence the instant it lands. Everyone else keeps going.'], ['Goal met', 'A booked meeting or a won deal closes the sequence for that contact automatically.'], ['Hard bounce', 'The recipient is suppressed across the whole workspace. Nothing else ever queues to them.'], - ['Unsubscribed', 'One-click unsubscribe, RFC 8058. Suppression applies before the confirmation page loads.'], + ['Unsubscribed', 'A reply-to-opt-out line or unsubscribe link in every email, plus RFC 8058 one-click. Suppression is workspace-wide.'], ['Reply: STOP', 'Opt-out phrases in a reply are caught and suppressed without anyone in the loop.'], ['Manual suppress', 'An operator or an uploaded list suppresses the contact across every sequence at once.'], ]; diff --git a/site/src/pages/faq.astro b/site/src/pages/faq.astro index 0ecba6da..fb26dfb8 100644 --- a/site/src/pages/faq.astro +++ b/site/src/pages/faq.astro @@ -22,7 +22,7 @@ const groups = [ items: [ ['What is the per-mailbox cap?', 'Default 50 cold emails / day with a 10-minute minimum gap. You can lower it. You can raise it, but anything above 50 / day requires positive reputation signals.'], ['How do you spread sending across IPs?', 'One worker per machine, each with its own IP. We distribute mailboxes across workers and assign campaign sends round-robin across the chosen sender pool.'], - ['Do you offer dedicated IPs?', 'Yes, on Business. A dedicated worker comes with its own IP and is allocated to a single organisation.'], + ['Do you offer dedicated IPs?', 'Business and Enterprise run your sending on a worker machine allocated to your organisation alone, so your reputation is not shared with other customers. We do not sell or manage IPs as a product, and we do not let you bring your own.'], ['What providers do you support?', 'Google Workspace / Gmail, Microsoft 365 / Outlook, and any SMTP / IMAP provider. iCloud and Zoho are first-class too.'], ], }, diff --git a/site/src/pages/pricing.astro b/site/src/pages/pricing.astro index 5b99facf..b4dca6fd 100644 --- a/site/src/pages/pricing.astro +++ b/site/src/pages/pricing.astro @@ -53,7 +53,9 @@ const plans = [ features: [ 'Everything in Grow', '15,000 emails / day', - 'Dedicated IPs', + // Not an IP product: the mechanism is a worker machine bound to one org, + // and what it buys is reputation isolation. index.astro words it the same. + 'Sending kept apart from other customers', 'Team roles + audit log', 'Priority support', ], @@ -94,7 +96,7 @@ const matrix = [ ['Cold sends per day', '150', '3,000', '15,000', '15,000+'], ['Per-mailbox cold cap', '100 / day', '100 / day', '100 / day', 'Custom'], ['Distributed workers', 'Shared', 'Shared', 'Dedicated', 'Dedicated'], - ['Dedicated sending IPs', '—', '—', '✓', '✓'], + ['Sending kept apart from others', '—', '—', '✓', '✓'], ], }, { @@ -417,6 +419,7 @@ const headers = ['Starter', 'Grow', 'Business', 'Enterprise']; {[ ['Do you charge per email sent?', 'No. Each plan includes a fixed daily send budget. Unused volume does not carry over.'], ['Is warmup metered separately?', 'No. Warmup is unlimited on every hosted plan and runs against the premium pool.'], + ['Is there really no mailbox limit?', 'Mailboxes are unlimited on every paid plan under fair use: one mailbox for every send a day your plan includes, so Grow holds 3,000 and Business holds 15,000. That is far more than safe sending ever needs. Need more? Ask from the connect dialog and we usually answer within a business day. Connect them one at a time or import thousands from a CSV.'], ['What does self-hosting cost?', 'Nothing. The platform is Apache 2.0 and free to run on your own servers, with unlimited mailboxes. The only paid part is the warmup pool link: free for 10 mailboxes, $15 a month for unlimited.'], ['Can a self-hosted instance use the warmup pool?', 'Yes, once Warmbly Cloud launches. Open Settings, Warmbly Cloud on your instance, approve the code here, and pick the mailboxes to enroll. Warmbly runs their warmup in the shared pool; sending, contacts and inbox stay on your server.'], ['Is there a per-seat charge?', 'No. Add as many teammates as you want. Roles and audit log unlock on Business.'], diff --git a/site/src/pages/use-cases/agencies.astro b/site/src/pages/use-cases/agencies.astro index 081902fd..2e860a9c 100644 --- a/site/src/pages/use-cases/agencies.astro +++ b/site/src/pages/use-cases/agencies.astro @@ -77,8 +77,8 @@ const faq = [ 'As many as you subscribe to. Each client is its own workspace with its own subscription, and workspaces stack, so capacity scales with the roster instead of with a seat count.'], ['Do I have to manage billing for every client?', 'You own every workspace subscription as the agency. Clients do not see a Warmbly invoice unless you transfer ownership. Mark up whatever you want at the agency level.'], - ['Can a client get its own dedicated IPs?', - 'Yes. By default every paid workspace shares the premium warmup and worker pool. For a client that needs its own reputation surface, you can move it to dedicated workers, each its own machine and IP, carrying only that client\'s mail.'], + ['Can a client get its own sending infrastructure?', + 'Yes. By default every paid workspace shares the premium warmup and worker pool. For a client that needs its own reputation surface, you can move it onto sending infrastructure allocated to that client alone, carrying only their mail, so their reputation is not shared with anyone else. We do not sell or manage IPs as a product.'], ['What happens when a client\'s mailbox starts landing in spam?', 'It steps down the health ladder (Healthy, Watch, Quarantined, Blocked) and is pulled from the shared pool well before mailbox providers penalize it, acting around 10% spam placement rather than 80%, then recovers inside its own workspace.'], ['Can clients log in to see their own data?', @@ -262,7 +262,7 @@ const faq = [

Sending infrastructure

- Shared premium pool, or dedicated IPs per client. + Shared premium pool, or sending kept apart per client.

Every paid workspace warms and sends on the shared premium pool. When a client grows big enough to want its own IPs, move it onto dedicated workers, each on its own machine, sending only that client's mail. diff --git a/skills/warmbly-cli/SKILL.md b/skills/warmbly-cli/SKILL.md new file mode 100644 index 00000000..1022c003 --- /dev/null +++ b/skills/warmbly-cli/SKILL.md @@ -0,0 +1,153 @@ +--- +name: warmbly-cli +description: Use the `warmbly` CLI to drive Warmbly as a signed-in user - sign in with `warmbly auth login`, then work with campaigns, contacts, mailboxes, the unified inbox, analytics, webhooks and the live event stream on the hosted service or any self-hosted instance. Use whenever the task is to operate Warmbly the product from a terminal or a script and a `warmbly` binary is available. For instance recovery and accounts (database-level), use warmbly-ops instead. +--- + +# Driving Warmbly through the `warmbly` CLI + +`warmbly` is the customer CLI: it signs in as a person, holds one credential +per host in `~/.config/warmbly`, and speaks only the public REST API. Every +command is bounded by the scopes the sign-in approved. + +If the binary is not on PATH, install it without a toolchain or root: + +```bash +curl -fsSL https://warmbly.com/cli.sh | sh # macOS, Linux +irm https://warmbly.com/cli.ps1 | iex # Windows +``` + +Add `-s -- --dir ` to place it somewhere specific. It is also inside the +backend image on a self-hosted instance (`docker compose -p warmbly exec +backend warmbly ...`). + +It is not `warmblyctl`. That one talks to Postgres and exists for recovery and +accounts (the `warmbly-ops` skill). If both are available and the task is +product work, use `warmbly`. + +## Getting authenticated + +Check first, because a non-interactive agent cannot complete a browser flow: + +```bash +warmbly auth status +``` + +- **Signed in** (exit 0): carry on. +- **Not signed in** (exit 4): you need a credential. In order of preference: + 1. `WARMBLY_TOKEN` in the environment. It overrides everything and is never + written to disk, so it is the right answer for a script or a CI job. + 2. `echo "$KEY" | warmbly auth login --with-token`, when a key was supplied. + 3. `warmbly auth login`, which needs a human at a browser. Print the code and + the URL it shows and hand back to the user; do not sit in the poll loop + waiting for something only they can do. + +Set `WARMBLY_HOST` (or `--host`) for a self-hosted instance, and +`WARMBLY_API_URL` only when the API is not at `api.`. + +Exit code 4 always means the credential: missing, rejected, or short a scope. +`warmbly auth status` names which, and where the token came from. + +## Output: always ask for JSON + +Tables are for humans. Every command takes `--json`, and output is JSON +automatically when stdout is not a terminal, but pass it explicitly so the +shape does not depend on how you were invoked. + +```bash +warmbly campaign list --json +warmbly campaign list --json --all # every page, cursor followed +warmbly contact list --json --limit 100 +``` + +Lists are `{"data": [...], "pagination": {"next_cursor", "has_more"}}`. Page +with `--cursor `, or let `--all` do it. Cursors are opaque, never +construct one. + +## Command map + +`warmbly --help` lists subcommands; `warmbly --help` +gives the arguments and flags. Ids are positional, not flags. + +| Command | Covers | +|---|---| +| `status` | one call for "what is happening": mailboxes needing attention, what is sending, what is unread | +| `campaign` | list, view, create, edit, delete, steps, senders, segments, preflight, test, start, stop, logs | +| `contact` | list, view, create, edit, delete, lookup, timeline, emails, notes, import, export, verify | +| `mailbox` | list, view, edit, check, sync, behavior, warmup, hold, release, send | +| `inbox` | list, view, thread, read, reply, compose, drafts, scheduled, snooze | +| `suppression` | the list of addresses and domains that get no campaign mail | +| `segment`, `template`, `automation`, `form` | audiences, reply templates, automations, lead capture | +| `deal`, `pipeline`, `task` | the CRM | +| `analytics`, `audit`, `advisor` | numbers, the audit trail, recommendations | +| `webhook`, `key`, `oauth-app`, `integration` | the developer surface | +| `org`, `team`, `settings`, `warmup-routing` | the workspace surface a key can reach | +| `tool` | the AI tool registry, listed and called | +| `events tail` | the live event stream | +| `api` | any endpoint at all | + +Anything without a command is reachable through the passthrough. Paths are +relative to `/v1`: + +```bash +warmbly api "/campaigns?limit=10" --paginate +warmbly api /contacts -f email=jane@example.com -f first_name=Jane +warmbly api /campaigns/CAMPAIGN_ID -X PATCH -F daily_limit=40 +warmbly api /contacts/search -X POST --input filter.json +``` + +`-f` keeps a string, `-F` guesses the type (`true`, `null`, numbers, `@file`), +`key[sub]=v` nests, repeated `key[]=v` builds an array. + +## Sending safety, read before anything that sends + +These put real mail on the wire and prompt before doing so: +`campaign start`, `campaign test`, `mailbox send`, `inbox reply`, +`inbox compose`, `inbox approve-draft`. Everything else is safe to run freely. + +- With no terminal they refuse rather than send. `--yes` is what proceeds, so + **only pass `--yes` when the user asked for that specific send.** Never add + it globally to be rid of prompts. +- Run `warmbly campaign preflight CAMPAIGN_ID` before `campaign start` and act + on what it reports. It costs nothing and catches missing senders, empty + audiences and broken tracking. +- Never raise a mailbox's daily cap casually. The default is 50 campaign + emails per mailbox per day with 600 seconds between sends; a fresh mailbox + starts around 10-20. Do not go above 50 unless the user asked and the + mailbox has the history to justify it. +- Keep warmup running on mailboxes that campaign. Do not stop warmup because a + campaign started. +- If deliverability shows rising bounces or complaints, stop the campaign and + report. Do not push volume into a degrading mailbox. + +## Errors + +Failures print the API's `code` and `request_id` to stderr. Branch on `code` +(`not_found`, `forbidden`, `rate_limit_exceeded`, ...) and quote `request_id` +when reporting. On `rate_limit_exceeded`, wait the `Retry-After` it names. + +For a write you retry, pass `--idempotency-key ` so a retry cannot +double-apply. Any unique string works; reuse it only for the identical retry. + +Exit codes: `0` worked, `1` failed, `2` bad command line or a prompt with no +terminal, `4` credential. + +## Watching what happens + +```bash +warmbly events tail --json --intent EMAIL +``` + +Streams the live event stream as newline-delimited JSON. Needs a key with +`REALTIME_SUBSCRIBE`. Useful for confirming a send actually went out; give it +`--count N` so it terminates rather than running forever. + +## What this CLI cannot do + +- Connect a mailbox. That needs OAuth consent or a credential form in a + browser: `warmbly browse mailboxes --no-browser` prints the URL to hand over. +- Manage members, roles, invitations, workspace exports or billing. Every + `/organization/*` and `/subscription/*` route is session-only and refuses an + API key, so there is no command for them. `warmbly browse settings + --no-browser` prints the URL to hand over. +- Create accounts, reset passwords, grant platform admin, back up or restore an + instance. That is `warmblyctl` and the `warmbly-ops` skill. diff --git a/skills/warmbly-install/SKILL.md b/skills/warmbly-install/SKILL.md new file mode 100644 index 00000000..ff7f7fb3 --- /dev/null +++ b/skills/warmbly-install/SKILL.md @@ -0,0 +1,165 @@ +--- +name: warmbly-install +description: Install, reconfigure, back up, restore, move or uninstall a self-hosted Warmbly instance with install.sh and warmblyctl backup/restore - the one-command install, its non-interactive flags, generating a .env or compose file without installing, scheduled backups, and moving an instance to another host. Use when the task is to stand up, migrate or tear down an instance, rather than to operate one that is already running. +--- + +# Installing and moving a Warmbly instance + +`install.sh` is the front door: it pulls the published release images, writes +`docker-compose.yml` and `.env`, starts the stack and prints the claim link. +Nothing is compiled. `warmblyctl backup` and `warmblyctl restore` are the +other half: one bundle that moves a whole instance to another host. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh # the fast path +curl -fsSL https://warmbly.com/install.sh | sh -s -- --wizard # asks everything +``` + +## Rule zero: agents should not run the interactive forms + +The wizard, the review screen and the demo all need a terminal and read raw +keypresses. Driving them from a script means feeding a pty, and there is no +reason to: every answer is also a flag and a `WARMBLY_*` variable. Use +`--yes` with flags, and the run is silent, deterministic and idempotent. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --yes \ + --dir /opt/warmbly --host warmbly.example.com --tls caddy \ + --data-root /mnt/data/warmbly --retention-preset minimal \ + --backup-dir /mnt/backups/warmbly --version v1.4.2 +``` + +Without a terminal the script says so and falls back to defaults plus +whatever flags it was given, so a forgotten `--yes` degrades rather than +hanging. + +## Inspect before you install + +Three flags produce output and change nothing. Prefer them over reasoning +about what the install would do. + +| Command | Prints | +|---|---| +| `sh install.sh --print-env [flags]` | The exact `.env`, to stdout. Nothing else | +| `sh install.sh --dry-run [flags]` | Every file it would write, each under a `── (mode NNNN)` header | +| `sh install.sh --demo` | The whole thing played, installing nothing. Needs a terminal; not for agents | + +`--dry-run` output is greppable but is a display format, not a contract. When +you need one file, use `--print-env` for the `.env` and read the compose file +off disk after a real run. + +## The flags + +| Flag | Variable | Default | +|---|---|---| +| `--dir` | `WARMBLY_DIR` | `/opt/warmbly` | +| `--host` | `WARMBLY_HOST` | `localhost` | +| `--tls` | `WARMBLY_TLS` | `none`, or `caddy`, `proxy` | +| `--data-root` | `WARMBLY_DATA_ROOT` | `

/data`, or the word `volumes` | +| `--blobs` | `WARMBLY_BLOBS` | `filesystem`, or `s3` | +| `--database-url` | `WARMBLY_DATABASE_URL` | bundled Postgres | +| `--redis-url` | `WARMBLY_REDIS_URL` | bundled Redis | +| `--components` | `WARMBLY_COMPONENTS` | `full`, or `core` | +| `--version` | `WARMBLY_VERSION` | the newest release | +| `--channel` | `WARMBLY_CHANNEL` | `stable`, or `dev` | +| `--registry` | `WARMBLY_IMAGE_PREFIX` | `ghcr.io/warmbly/warmbly` | +| `--backup-dir` | `WARMBLY_BACKUP_DIR` | no scheduled backup | +| `--retention-preset` | | `default`, or `minimal` | +| `--force` | | Install into a directory the script did not create | + +## Mechanics that bite + +- **Re-running is reconfiguration, not reinstallation.** The script adopts the + existing `.env`, never regenerates a secret and never moves an existing data + root. Regenerating `CREDENTIALS_ENCRYPTION_KEY` would make every stored + mailbox credential permanently unreadable, so it will not do it and neither + should you. +- **It refuses a non-empty directory it did not create.** That is `--force`, + and `--force` is a decision about someone else's files. +- **Pin the version in automation.** Without `--version` it resolves the newest + release at run time, so two hosts provisioned a week apart are two versions. + The resolved tag is written to `WARMBLY_TAG` in `.env` either way. +- **Verify what landed.** After the pull the script compares the images against + the release's published `images.json` and stops on a mismatch. That check is + skipped for a custom `--registry`, and it says so. +- **The keys are the backup.** `keys-backup.txt` in the install directory holds + the two unrecoverable ones. It sits on the same disk as the database, so it + is not a backup until it is copied off the host. +- **`--uninstall` never touches data.** Data removal is `--purge-data`, which + also deletes the keys that could have opened a backup. + +## Checking an install + +```bash +cd /opt/warmbly +docker compose -p warmbly ps +docker compose -p warmbly exec backend warmblyctl status --json +``` + +`status --json` has stable keys and always exits 0; read `.summary.error`. +Everything else about a running instance is the `warmbly-ops` skill. + +## Backing up + +One bundle holds the three things that only restore together: the database, +the blob root, and the encryption keys. Any two of the three restore an +instance that looks fine and cannot read its own mailboxes. + +```bash +cd /opt/warmbly +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz \ + && docker compose -p warmbly exec -T backend rm -f /data/blobs/warmbly.tar.gz +``` + +`/data/blobs` is used as the hand-off because it is the one path the container +and the host both see. Delete it there afterwards, as above: `backup` leaves +its own output out of the archive, but a bundle left in the blob root is swept +into the next one. Keep the `&&`: an unguarded cleanup after a failed `cp` +deletes the only copy of the backup. The bundle is 0600 and holds every mailbox credential on the +instance plus the keys that open them; never write it somewhere world-readable +and never print its contents. + +`install.sh --wizard` can schedule exactly this (a `backup.sh` and a systemd +timer); `--backup-dir` sets it up non-interactively. + +## Moving an instance to another host + +```bash +# 1. On the new host +curl -fsSL https://warmbly.com/install.sh | sh -s -- --yes --host + +# 2. Copy CREDENTIALS_ENCRYPTION_KEY and KMS_LOCAL_MASTER_KEY from the old +# .env into the new one, then recreate so they take effect +cd /opt/warmbly && docker compose -p warmbly up -d + +# 3. Restore +docker compose -p warmbly cp ./warmbly.tar.gz backend:/data/blobs/warmbly.tar.gz +docker compose -p warmbly exec -T backend warmblyctl restore --file /data/blobs/warmbly.tar.gz --yes +docker compose -p warmbly restart + +# 4. Confirm +docker compose -p warmbly exec backend warmblyctl status --json +``` + +`restore` replaces everything on the target instance, so `--yes` is skipping a +typed confirmation about destroying data. Do not pass it to a command whose +target you have not just checked. + +It refuses outright when the host's keys are not the ones the bundle was +sealed with, and prints the two lines to add to `.env`. Do that rather than +reaching for `--force`: `--force` accepts losing every stored mailbox +credential, permanently, and it is never the right answer to an error message. + +## One workspace, not the instance + +Moving a single organization between two running instances is +`warmblyctl org export` / `org import`, in the `warmbly-ops` skill. The two are +not interchangeable: a bundle cannot be applied to one workspace, and a +workspace archive cannot restore an instance. + +## Reference + +- Install and every flag: https://docs.warmbly.com/development/install/ +- Where data lives, retention, backups, migration: https://docs.warmbly.com/development/data-control/ +- Building from source instead: https://docs.warmbly.com/development/deployment-guide/ diff --git a/skills/warmbly-ops/SKILL.md b/skills/warmbly-ops/SKILL.md index 7ea78e2f..3a72f9a7 100644 --- a/skills/warmbly-ops/SKILL.md +++ b/skills/warmbly-ops/SKILL.md @@ -41,6 +41,8 @@ for. Prefer parsing that over reasoning from the prose. | List workspaces | `warmblyctl org list` | | Move a workspace out | `warmblyctl org export --org --out file.zip` | | Move a workspace in | `warmblyctl org import --org ... --file file.zip --dry-run` first | +| Back the instance up | `warmblyctl backup --out /data/blobs/warmbly.tar.gz`, then copy it out and delete it | +| Restore one onto this instance | `warmblyctl restore --file /data/blobs/warmbly.tar.gz` | ## Mechanics that bite @@ -60,6 +62,21 @@ for. Prefer parsing that over reasoning from the prose. product has (every mailbox password and refresh token, sealed only by the passphrase you supply). Never write it anywhere world-readable, and never echo the passphrase. +- `backup` and `restore` are the INSTANCE-level pair and are not + interchangeable with `org export`/`org import`: a bundle cannot be applied to + one workspace, and a workspace archive cannot restore an instance. +- `restore` replaces every organization, user and mailbox on the target. It + asks for a typed `restore` first; `--yes` skips that, so only pass it to a + target you have just checked with `status --json`. +- `restore` refuses when the host's `CREDENTIALS_ENCRYPTION_KEY` or + `KMS_LOCAL_MASTER_KEY` differ from the bundle's, and prints the lines to put + in `.env`. Do that. `--force` accepts losing every stored mailbox credential + permanently and is never the right answer to that message. + +## Standing an instance up, or moving one + +Installing, reconfiguring, backing up on a schedule and moving an instance to +another host is the `warmbly-install` skill. ## Interacting with the product itself diff --git a/tracking/src/config.rs b/tracking/src/config.rs index d8f97949..3f1f81c3 100644 --- a/tracking/src/config.rs +++ b/tracking/src/config.rs @@ -53,6 +53,10 @@ pub struct Config { /// Shared bearer token for the backend internal API (required; same /// INTERNAL_API_TOKEN the workers use). pub internal_api_token: String, + /// Secret the source-address token is keyed with. An unkeyed hash of an + /// IPv4 address is reversible by enumeration; a keyed one is only a + /// stable name for one source. Defaults to the internal API token. + pub ip_hash_key: String, /// Per-source request budget for both tracking endpoints (default 300/min). pub rate_limit_per_min: u32, /// Page-view ingest budget per source per minute. Lower than the pixel @@ -198,6 +202,11 @@ impl Config { trusted_proxies, client_ip_header ); + let ip_hash_key = env::var("TRACKING_IP_HASH_KEY") + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| internal_api_token.clone()); + Ok(Self { env: env_name, host, @@ -213,6 +222,7 @@ impl Config { schema_registry_key, schema_registry_secret, backend_internal_url, + ip_hash_key, internal_api_token, rate_limit_per_min, pagehit_rate_limit_per_min, @@ -290,6 +300,10 @@ impl Config { schema_registry_key, schema_registry_secret, backend_internal_url, + ip_hash_key: env::var("TRACKING_IP_HASH_KEY") + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| internal_api_token.clone()), internal_api_token, rate_limit_per_min: 300, pagehit_rate_limit_per_min: 60, diff --git a/tracking/src/events.rs b/tracking/src/events.rs index 4e6c513a..fd6499b8 100644 --- a/tracking/src/events.rs +++ b/tracking/src/events.rs @@ -8,7 +8,13 @@ pub struct TrackingEvent { pub event_type: String, pub task_id: String, pub original_url: Option, + /// Click ticket id, so the consumer can name the link (destination and + /// anchor text) without matching URLs. + pub link_id: Option, pub timestamp: String, pub user_agent: Option, pub ip_hash: Option, + /// The source network (last IPv4 octet zeroed, IPv6 cut to 48 bits), + /// enough for the consumer's location lookup without naming a host. + pub client_ip: Option, } diff --git a/tracking/src/handlers.rs b/tracking/src/handlers.rs index 68ea44b8..6e68b0d2 100644 --- a/tracking/src/handlers.rs +++ b/tracking/src/handlers.rs @@ -46,6 +46,8 @@ pub struct AppState { /// Proxies whose forwarded-IP header is believed, and which header pub trusted_proxies: Arc>, pub client_ip_header: Arc, + /// Key for the source-address token (see `hash_ip`) + pub ip_hash_key: Arc, } impl AppState { @@ -75,6 +77,7 @@ impl AppState { hit_rate_limiter: Arc::new(RateLimiter::new(config.pagehit_rate_limit_per_min)), trusted_proxies: Arc::new(config.trusted_proxies.clone()), client_ip_header: Arc::new(config.client_ip_header.clone()), + ip_hash_key: Arc::new(config.ip_hash_key.clone()), } } @@ -124,13 +127,15 @@ pub async fn track_open( return pixel_response(); } - // Extract IP hash for deduplication + rate limiting - let ip_hash = Some(hash_ip(&client_ip( + // The address is hashed for deduplication + rate limiting; only its + // network travels with the event, for the location lookup downstream. + let ip = client_ip( peer, &headers, &state.trusted_proxies, &state.client_ip_header, - ))); + ); + let ip_hash = Some(hash_ip(&state.ip_hash_key, &ip)); // Anti-flood: over-budget sources still get the pixel (real mail clients // must never see a broken image), but nothing is published. @@ -164,9 +169,11 @@ pub async fn track_open( event_type: "EMAIL_OPENED".to_string(), task_id, original_url: None, + link_id: None, timestamp: Utc::now().to_rfc3339(), user_agent, ip_hash, + client_ip: Some(anonymize_ip(&ip)).filter(|n| !n.is_empty()), }) .await; }); @@ -191,13 +198,15 @@ pub async fn track_click( return (StatusCode::NOT_FOUND, "Unknown link").into_response(); } - // Anti-flood: cap total request rate per source - let ip_hash = Some(hash_ip(&client_ip( + // Anti-flood: cap total request rate per source. Only the address's + // network rides along, for the location lookup downstream. + let ip = client_ip( peer, &headers, &state.trusted_proxies, &state.client_ip_header, - ))); + ); + let ip_hash = Some(hash_ip(&state.ip_hash_key, &ip)); let source = ip_hash.clone().unwrap_or_else(|| "unknown".to_string()); if !state.rate_limiter.allow(&source).await { return (StatusCode::TOO_MANY_REQUESTS, "Slow down").into_response(); @@ -245,15 +254,18 @@ pub async fn track_click( // Publish event asynchronously (fire and forget) let producer = state.producer.clone(); let destination = link.destination.clone(); + let ticket = link_id.clone(); tokio::spawn(async move { producer .publish(TrackingEvent { event_type: "EMAIL_CLICKED".to_string(), task_id: link.task_id, original_url: Some(destination), + link_id: Some(ticket), timestamp: Utc::now().to_rfc3339(), user_agent, ip_hash, + client_ip: Some(anonymize_ip(&ip)).filter(|n| !n.is_empty()), }) .await; }); @@ -317,7 +329,7 @@ pub async fn track_page_hit( &state.trusted_proxies, &state.client_ip_header, ); - let source = hash_ip(&ip); + let source = hash_ip(&state.ip_hash_key, &ip); // Anti-flood: page views have their own, tighter budget on top of the // shared one, and the shared one counts too so a flood here also @@ -464,9 +476,32 @@ fn client_ip( } } -fn hash_ip(ip: &str) -> String { - // Hash the IP for privacy +/// The network an address belongs to, for the location lookup downstream: +/// the last IPv4 octet zeroed, an IPv6 address cut to its first 48 bits. +/// City-level resolution survives; a single host is no longer named, so the +/// bus can retain the event without retaining the address. +fn anonymize_ip(ip: &str) -> String { + match ip.parse::() { + Ok(IpAddr::V4(v4)) => { + let o = v4.octets(); + format!("{}.{}.{}.0", o[0], o[1], o[2]) + } + Ok(IpAddr::V6(v6)) => { + let s = v6.segments(); + std::net::Ipv6Addr::new(s[0], s[1], s[2], 0, 0, 0, 0, 0).to_string() + } + Err(_) => String::new(), + } +} + +/// A stable, keyed token for a source address: the same source gets the same +/// token (dedupe, rate limits, the burst rule), and nobody holding the token +/// can enumerate IPv4 space to get the address back, because the key is +/// secret. The key goes in first so the digest is not one of a public value. +fn hash_ip(key: &str, ip: &str) -> String { let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hasher.update([0u8]); hasher.update(ip.as_bytes()); let result = hasher.finalize(); format!("{:x}", result)[..16].to_string() // Take first 16 chars @@ -533,6 +568,23 @@ mod tests { ); } + #[test] + fn hash_ip_is_keyed_and_stable() { + assert_eq!(hash_ip("k", "203.0.113.9"), hash_ip("k", "203.0.113.9")); + assert_ne!(hash_ip("k", "203.0.113.9"), hash_ip("other", "203.0.113.9")); + assert_eq!(hash_ip("k", "203.0.113.9").len(), 16); + } + + #[test] + fn anonymize_ip_keeps_only_the_network() { + assert_eq!(anonymize_ip("203.0.113.9"), "203.0.113.0"); + assert_eq!( + anonymize_ip("2001:db8:abcd:1234:5678::1"), + "2001:db8:abcd::" + ); + assert_eq!(anonymize_ip("not an ip"), ""); + } + #[test] fn host_of_strips_scheme_and_path() { assert_eq!(host_of("https://WWW.Example.com/a?b#c"), "www.example.com"); diff --git a/tracking/src/kafka.rs b/tracking/src/kafka.rs index 7a39925e..7ff43b59 100644 --- a/tracking/src/kafka.rs +++ b/tracking/src/kafka.rs @@ -25,9 +25,11 @@ pub const TRACKING_EVENT_SCHEMA: &str = r#" {"name": "event_type", "type": "string", "avro.java.string": "String"}, {"name": "task_id", "type": "string", "avro.java.string": "String"}, {"name": "original_url", "type": ["null", "string"], "default": null}, + {"name": "link_id", "type": ["null", "string"], "default": null}, {"name": "timestamp", "type": "string", "avro.java.string": "String"}, {"name": "user_agent", "type": ["null", "string"], "default": null}, - {"name": "ip_hash", "type": ["null", "string"], "default": null} + {"name": "ip_hash", "type": ["null", "string"], "default": null}, + {"name": "client_ip", "type": ["null", "string"], "default": null} ] } "#; @@ -58,6 +60,13 @@ impl ToAvroValue for TrackingEvent { None => Value::Union(0, Box::new(Value::Null)), }, ), + ( + "link_id", + match &self.link_id { + Some(id) => Value::Union(1, Box::new(Value::String(id.clone()))), + None => Value::Union(0, Box::new(Value::Null)), + }, + ), ("timestamp", Value::String(self.timestamp.clone())), ( "user_agent", @@ -73,6 +82,13 @@ impl ToAvroValue for TrackingEvent { None => Value::Union(0, Box::new(Value::Null)), }, ), + ( + "client_ip", + match &self.client_ip { + Some(net) => Value::Union(1, Box::new(Value::String(net.clone()))), + None => Value::Union(0, Box::new(Value::Null)), + }, + ), ] } } diff --git a/web/Dockerfile b/web/Dockerfile index b646aadb..741632c8 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -5,6 +5,8 @@ # heavy pnpm build runs only once. FROM --platform=$BUILDPLATFORM node:22-alpine AS build WORKDIR /app +# No TTY in a build: CI=true makes pnpm reinstall instead of prompting. +ENV CI=true RUN corepack enable && corepack prepare pnpm@11.9.0 --activate COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ RUN pnpm install --frozen-lockfile diff --git a/web/src/app/app/analytics/page.tsx b/web/src/app/app/analytics/page.tsx index a3dfd435..2f011f49 100644 --- a/web/src/app/app/analytics/page.tsx +++ b/web/src/app/app/analytics/page.tsx @@ -29,6 +29,9 @@ import type { DitherTone } from "@/components/ui/dither"; import AnalyticsShareButton from "@/components/app/analytics/AnalyticsShareButton"; import useDashboard from "@/lib/api/hooks/app/analytics/useDashboard"; +const AUTO_OPENS_TIP = "Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail) or within seconds of sending, not a person reading"; +const AUTO_CLICKS_TIP = "Auto-clicks: links followed by a security gateway scanning the email, not a person; not counted as clicks"; + type Range = "7d" | "30d" | "90d"; type Metric = "sent" | "opens" | "clicks" | "replies"; @@ -82,8 +85,8 @@ export default function AnalyticsPage() { const breakdown = [ { label: "Sent", value: os?.total_emails_sent, icon: SendIcon, dot: "bg-slate-400" }, - { label: "Opens", value: os?.total_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: os?.machine_opens ? `${num(os.machine_opens)} auto` : undefined }, - { label: "Clicks", value: os?.total_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500" }, + { label: "Opens", value: os?.total_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: os?.machine_opens ? `${num(os.machine_opens)} auto` : undefined, noteTitle: AUTO_OPENS_TIP }, + { label: "Clicks", value: os?.total_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500", note: os?.machine_clicks ? `${num(os.machine_clicks)} auto` : undefined, noteTitle: AUTO_CLICKS_TIP }, { label: "Replies", value: os?.total_replies, icon: ReplyIcon, dot: "bg-amber-500" }, { label: "Bounces", value: os?.total_bounces, icon: TriangleAlertIcon, dot: "bg-rose-500" }, ]; @@ -169,7 +172,7 @@ export default function AnalyticsPage() { {"note" in q && q.note && ( {q.note} diff --git a/web/src/app/app/api-keys/_components/KeyDetailDrawer.tsx b/web/src/app/app/api-keys/_components/KeyDetailDrawer.tsx index 77e1b6df..e1564fc0 100644 --- a/web/src/app/app/api-keys/_components/KeyDetailDrawer.tsx +++ b/web/src/app/app/api-keys/_components/KeyDetailDrawer.tsx @@ -213,7 +213,7 @@ function Inner({ apiKey, onClose }: { apiKey: APIKey; onClose: () => void }) { {apiKey.description && (

{apiKey.description}

)} -
+
diff --git a/web/src/app/app/campaigns/[id]/layout.tsx b/web/src/app/app/campaigns/[id]/layout.tsx index 3dbd550d..0cfa7ac4 100644 --- a/web/src/app/app/campaigns/[id]/layout.tsx +++ b/web/src/app/app/campaigns/[id]/layout.tsx @@ -10,9 +10,11 @@ import { Loader2Icon, PauseIcon, PlayIcon, + SendIcon, Settings2Icon, UsersIcon, } from "lucide-react"; +import { campaignDisplayLabel, isIdleCampaign, isOneTimeCampaign } from "@/components/app/campaigns/status"; import useCampaign from "@/lib/api/hooks/app/campaigns/useCampaign"; import useStartCampaign from "@/lib/api/hooks/app/campaigns/useStartCampaign"; import useStopCampaign from "@/lib/api/hooks/app/campaigns/useStopCampaign"; @@ -41,6 +43,7 @@ const STATUS_PILL: Record = { paused_undeliverable: "bg-amber-50 text-amber-700 border-amber-200", draft: "bg-slate-100 text-slate-600 border-slate-200", completed: "bg-slate-100 text-slate-600 border-slate-200", + idle: "bg-sky-50 text-sky-700 border-sky-200", }; export default function CampaignLayout() { @@ -73,7 +76,7 @@ export default function CampaignLayout() { if (campaignData.isLoading) { return ( -
+
@@ -106,7 +109,7 @@ export default function CampaignLayout() { const campaign = campaignData.data; const status = campaign.status; - const pill = STATUS_PILL[status] ?? STATUS_PILL.draft; + const pill = isIdleCampaign(campaign) ? STATUS_PILL.idle : (STATUS_PILL[status] ?? STATUS_PILL.draft); const isActive = status === "active"; const canStart = canStartCampaign(status); @@ -126,7 +129,7 @@ export default function CampaignLayout() { return (
-
+
Campaigns -
-

{campaign.name}

+ {/* min-w-0 lets the name truncate instead of pushing the + pills off a narrow screen; the pills wrap under it. */} +
+

{campaign.name}

- {status === "paused_undeliverable" ? "needs verification" : status} + {campaignDisplayLabel(campaign)} + {isOneTimeCampaign(campaign) && ( + + + One-time + + )}

{campaign.id}

@@ -205,7 +219,10 @@ export default function CampaignLayout() { })}
-
+ {/* Leads renders a full-bleed Page (its own topbar, stat strip + and table gutters), so padding it again wastes a fifth of a + phone screen and stops its hairlines short of the edge. */} +
diff --git a/web/src/app/app/campaigns/[id]/page.tsx b/web/src/app/app/campaigns/[id]/page.tsx index 19fa4eb2..546d77f0 100644 --- a/web/src/app/app/campaigns/[id]/page.tsx +++ b/web/src/app/app/campaigns/[id]/page.tsx @@ -8,6 +8,7 @@ import { } from "lucide-react"; import { useCampaign } from "@/hooks/context/campaign"; import useCampaignAnalytics from "@/lib/api/hooks/app/analytics/useCampaignAnalytics"; +import type { CampaignEngagementBreakdown, EngagementBucket } from "@/lib/api/models/app/analytics/CampaignAnalytics"; import useCampaignDailyStats from "@/lib/api/hooks/app/analytics/useCampaignDailyStats"; import { SectionBar, Stat, StatStrip } from "@/components/layout/Page"; import { MultiTrend, type TrendSeries } from "@/components/ui/charts"; @@ -19,6 +20,9 @@ import CampaignFormsPanel from "@/components/app/campaigns/CampaignFormsPanel"; import AnimatedNumber from "@/components/ui/AnimatedNumber"; import AdvisorStrip from "@/components/app/advisor/AdvisorStrip"; +const AUTO_OPENS_TIP = "Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail) or within seconds of sending, not a person reading"; +const AUTO_CLICKS_TIP = "Auto-clicks: links followed by a security gateway scanning the email, not a person; not counted as clicks"; + const pctFmt = (v: number) => `${v.toFixed(1)}%`; type Metric = "sent" | "opens" | "clicks" | "replies"; @@ -99,8 +103,8 @@ export default function CampaignOverview() { const breakdown = [ { label: "Sent", value: summary?.emails_sent, icon: SendIcon, dot: "bg-slate-400" }, - { label: "Opens", value: summary?.unique_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: summary?.machine_opens ? `${summary.machine_opens} auto` : undefined }, - { label: "Clicks", value: summary?.unique_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500" }, + { label: "Opens", value: summary?.unique_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: summary?.machine_opens ? `${summary.machine_opens} auto` : undefined, noteTitle: AUTO_OPENS_TIP }, + { label: "Clicks", value: summary?.unique_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500", note: summary?.machine_clicks ? `${summary.machine_clicks} auto` : undefined, noteTitle: AUTO_CLICKS_TIP }, { label: "Replies", value: summary?.replies, icon: ReplyIcon, dot: "bg-amber-500" }, { label: "Bounces", value: summary?.bounces, icon: TriangleAlertIcon, dot: "bg-rose-500" }, ]; @@ -268,6 +272,8 @@ export default function CampaignOverview() { )}
+ + {/* quick breakdown strip below sequence table, mobile-friendly summary */} @@ -281,7 +287,7 @@ export default function CampaignOverview() { {q.note && ( {q.note} @@ -297,7 +303,7 @@ export default function CampaignOverview() { {/* Live panel */}