From d68bbcd2ab7c848935181195f686fe4e56220c78 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Fri, 4 Sep 2026 05:49:54 -0700 Subject: [PATCH] feat: add a one-command self-host installer at warmbly.com/install.sh with an interactive data-control wizard, give docker-compose.yml image keys and per-store volume variables, add an image-mode updater, move engagement/form/audit retention into instance settings, and add warmblyctl backup/restore --- .github/workflows/build-push.yml | 12 +- .github/workflows/ci.yml | 22 +- .github/workflows/release.yml | 54 +- AGENTS.md | 67 +- Makefile | 21 +- README.md | 26 +- .../app/dashboard/InstanceSettingsPage.tsx | 161 + admin/src/components/layout/UpdateDialog.tsx | 45 +- admin/src/lib/api/client/admin/instance.ts | 10 + admin/src/lib/api/client/admin/updates.ts | 15 +- cmd/backend/main.go | 41 +- cmd/consumer/main.go | 4 + cmd/updater/main.go | 16 +- cmd/warmblyctl/backup.go | 783 +++++ cmd/warmblyctl/main.go | 10 + deploy/config/env.example | 26 + deploy/docker/backend.Dockerfile | 5 +- docker-compose.yml | 46 +- .../docs/development/configuration.mdx | 15 + .../content/docs/development/data-control.mdx | 211 ++ .../docs/development/deployment-guide.mdx | 24 +- docs/content/docs/development/first-run.mdx | 8 +- docs/content/docs/development/install.mdx | 318 ++ docs/content/docs/development/meta.json | 2 + docs/content/docs/development/updates.mdx | 40 +- docs/content/docs/development/warmblyctl.mdx | 58 +- .../docs/guides/workspace-export-import.mdx | 4 + internal/api/handler/admin_instance.go | 3 + internal/app/consumer/event_tracking.go | 32 +- internal/app/instancecheck/checks_updates.go | 4 +- internal/app/instanceconfig/entries.go | 9 + internal/app/instancesettings/document.go | 71 + internal/app/instancesettings/service.go | 35 + internal/app/instancesettings/store.go | 16 + internal/app/updates/service.go | 21 +- internal/config/constants.go | 28 +- internal/jobs/audit_retention.go | 31 +- internal/jobs/form_events_retention.go | 22 +- internal/jobs/retention.go | 15 + internal/updater/api.go | 30 +- internal/updater/compose.go | 34 +- internal/updater/image.go | 201 ++ internal/updater/runner.go | 61 +- scripts/check-installer.sh | 192 ++ site/README.md | 26 + site/public/install.sh | 3071 +++++++++++++++++ site/public/install.sh.sha256 | 1 + skills/warmbly-install/SKILL.md | 162 + skills/warmbly-ops/SKILL.md | 17 + 49 files changed, 6023 insertions(+), 103 deletions(-) create mode 100644 cmd/warmblyctl/backup.go create mode 100644 docs/content/docs/development/data-control.mdx create mode 100644 docs/content/docs/development/install.mdx create mode 100644 internal/jobs/retention.go create mode 100644 internal/updater/image.go create mode 100755 scripts/check-installer.sh create mode 100644 site/public/install.sh create mode 100644 site/public/install.sh.sha256 create mode 100644 skills/warmbly-install/SKILL.md diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 8105b6cf..2e46494e 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: service: - description: "Service to build (all, backend, consumer, worker, forms, tracking, realtime)" + description: "Service to build (all, backend, consumer, worker, forms, updater, tracking, realtime)" required: false default: "all" @@ -61,6 +61,12 @@ jobs: - 'cmd/forms/**' - 'forms/**' - 'deploy/docker/forms.Dockerfile' + updater: + - 'go.mod' + - 'go.sum' + - 'internal/updater/**' + - 'cmd/updater/**' + - 'deploy/docker/updater.Dockerfile' tracking: - 'tracking/**' realtime: @@ -75,6 +81,7 @@ jobs: CONSUMER: ${{ steps.filter.outputs.consumer }} WORKER: ${{ steps.filter.outputs.worker }} FORMS: ${{ steps.filter.outputs.forms }} + UPDATER: ${{ steps.filter.outputs.updater }} TRACKING: ${{ steps.filter.outputs.tracking }} REALTIME: ${{ steps.filter.outputs.realtime }} run: | @@ -82,7 +89,7 @@ jobs: native="" if [ "$EVENT" = "workflow_dispatch" ]; then sel="${SELECTED:-all}" - for s in backend consumer worker forms; do + for s in backend consumer worker forms updater; do if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then go="$go\"$s\","; fi done for s in tracking realtime; do @@ -93,6 +100,7 @@ jobs: if [ "$CONSUMER" = "true" ]; then go="$go\"consumer\","; fi if [ "$WORKER" = "true" ]; then go="$go\"worker\","; fi if [ "$FORMS" = "true" ]; then go="$go\"forms\","; fi + if [ "$UPDATER" = "true" ]; then go="$go\"updater\","; fi if [ "$TRACKING" = "true" ]; then native="$native\"tracking\","; fi if [ "$REALTIME" = "true" ]; then native="$native\"realtime\","; fi fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d119212..6c00c8f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: forms: ${{ steps.filter.outputs.forms }} make: ${{ steps.filter.outputs.make }} ios: ${{ steps.filter.outputs.ios }} + installer: ${{ steps.filter.outputs.installer }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 @@ -70,6 +71,10 @@ jobs: - 'integrations/make/**' ios: - 'ios/**' + installer: + - 'site/public/install.sh' + - 'site/public/install.sh.sha256' + - 'scripts/check-installer.sh' migrations-ci: name: Migrations @@ -229,6 +234,21 @@ jobs: - name: Build run: pnpm build + # The installer is served verbatim from site/public, so what CI checks is + # exactly what a `curl | sh` executes. The checksum is published next to it + # for anyone who would rather download, verify and read before running, and + # it is only useful if it cannot drift. + installer-ci: + name: Installer CI + needs: changes + if: needs.changes.outputs.installer == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check the installer + run: ./scripts/check-installer.sh + make-ci: name: Make App CI needs: changes @@ -398,7 +418,7 @@ jobs: ci-status: name: CI Status runs-on: ubuntu-latest - needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] + needs: [changes, migrations-ci, go-ci, web-ci, admin-ci, site-ci, forms-ci, installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images] if: always() steps: - name: Check CI status diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8726dc6..23e67aee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - service: [backend, consumer, worker] + service: [backend, consumer, worker, forms, updater] runs-on: ubuntu-latest permissions: contents: read @@ -225,6 +225,38 @@ jobs: with: fetch-depth: 0 + # The installer verifies what it pulled against this file, so it is what + # makes "curl | sh" checkable after the fact rather than only before it. + # One line per service, because the thing that reads it is a POSIX shell. + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish the image manifest + env: + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + { + printf '{\n' + printf ' "tag": "%s",\n' "$TAG" + printf ' "registry": "%s",\n' "$IMAGE_PREFIX" + printf ' "images": {\n' + first=1 + for s in backend consumer worker forms updater web admin tracking realtime; do + digest=$(docker buildx imagetools inspect "$IMAGE_PREFIX/$s:$TAG" \ + --format '{{json .Manifest.Digest}}' | tr -d '"') + [ "$first" = 1 ] || printf ',\n' + first=0 + printf ' "%s": "%s"' "$s" "$digest" + done + printf '\n }\n}\n' + } > /tmp/images.json + cat /tmp/images.json + - name: Generate release notes env: GITHUB_SERVER_URL: ${{ github.server_url }} @@ -240,6 +272,23 @@ jobs: cat /tmp/changelog.md cat <<'EOF' + ## Install + + ``` + curl -fsSL https://warmbly.com/install.sh | sh -s -- --version ${{ github.ref_name }} + ``` + + Add `--wizard` to be asked where each store lives, what is kept and + for how long, and how it is backed up. + + `images.json` below lists the manifest digest of every image in this + release. The installer checks what it pulled against it, and you can + too: + + ``` + docker image inspect ${{ env.IMAGE_PREFIX }}/backend:${{ github.ref_name }} --format '{{index .RepoDigests 0}}' + ``` + ## Docker Images All images are available at `ghcr.io/${{ github.repository_owner }}/warmbly`: @@ -253,6 +302,8 @@ jobs: | Realtime | `${{ env.IMAGE_PREFIX }}/realtime:${{ github.ref_name }}` | | Dashboard (web) | `${{ env.IMAGE_PREFIX }}/web:${{ github.ref_name }}` | | Admin | `${{ env.IMAGE_PREFIX }}/admin:${{ github.ref_name }}` | + | Forms | `${{ env.IMAGE_PREFIX }}/forms:${{ github.ref_name }}` | + | Updater | `${{ env.IMAGE_PREFIX }}/updater:${{ github.ref_name }}` | ## Deployment @@ -266,5 +317,6 @@ jobs: tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} body_path: /tmp/release-body.md + files: /tmp/images.json draft: false prerelease: ${{ contains(github.ref_name, '-') }} diff --git a/AGENTS.md b/AGENTS.md index e5de9870..23b1f375 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,15 @@ At a product level, the app does four main things: The backend API is the control plane. Workers are the execution plane. +It ships as a hosted service and as a self-host, and the two are the same code. +The front door for the self-host is one command, +`curl -fsSL https://warmbly.com/install.sh | sh`, which pulls the published +release images and needs no clone and no compiler. `--wizard` turns it into an +interactive install that asks the data-control questions up front: where each +store lives, what is kept and for how long, how it is backed up. The script is +`site/public/install.sh` and it has its own rules below; the docs are +`docs/content/docs/development/install.mdx` and `data-control.mdx`. + ## Working In This Repo CI is strict. `go build ./...` succeeding is not enough — `golangci-lint` runs `gofmt` as part of its checks, and a single unformatted import block or mis-indented doc comment will fail the PR even when the code compiles cleanly. Before declaring any Go change done: @@ -39,6 +48,7 @@ Docs stay in sync: - the customer docs site lives in `docs/` (Fumadocs, served at docs.warmbly.com); content is MDX under `docs/content/docs/` in three sections: `guides/` (product behavior), `learn/` (fundamentals), `api/` (API reference) - any change that alters user-visible behavior must update the matching docs page in the same change: a new or changed endpoint updates `api/endpoints.mdx` (scope map) and, where relevant, `api/authentication.mdx`; a new or changed API permission updates `api/permissions.mdx` including the permission table, presets, and all three language tabs in the constants section; a new or changed error code updates `api/error-codes.mdx`; a new or changed product feature, default, limit, or setting updates the relevant `guides/` page (or adds one, registered in `guides/meta.json` under the right section group) - removing or renaming a feature, endpoint, or permission means removing or updating its docs too; do not leave stale docs behind +- self-hosting behavior has its own pages under `docs/content/docs/development/`: a change to the installer or to what it asks updates `install.mdx`; a change to where a store lives, how long something is kept, or how an instance is backed up or moved updates `data-control.mdx`; a new environment variable updates `configuration.mdx`, and a new database-backed setting updates its table there as well as the admin panel - follow the docs conventions: frontmatter `title` is the H1 (no `#` heading in the body), no decorative sidebar icons (pages and `meta.json` sections carry no `icon`; the source loader has the lucide icon plugin disabled, and code-sample tabs use the real language logo instead), sentence-case headings, no em dashes in prose, internal links use trailing slashes (`/guides/mailboxes/`) - verify with `pnpm types:check` and `pnpm lint` in `docs/` (the site is a fully static export; `pnpm build` writes `out/`) @@ -81,6 +91,50 @@ Rows move as `jsonb` in both directions, so adding a *column* to an existing tab The same applies to the customer-facing side of a feature: if it stores org data, its docs page and `docs/content/docs/guides/workspace-export-import.mdx` should agree about whether that data moves. +### The installer is a published artifact + +`site/public/install.sh` is the one-command self-host installer, served +verbatim from the static site at `https://warmbly.com/install.sh`. What is in +the repo is byte for byte what a stranger pipes into their shell, which makes +it the highest-consequence file here that is not Go. + +It is a wizard: an animated stepper, arrow-key and vim menus, live pull and +health screens, a review pass, and a `--demo` mode that plays the whole thing +while installing nothing. `docs/content/docs/development/install.mdx` documents +it and `data-control.mdx` documents what its questions decide; the +`warmbly-install` skill is the agent-facing version. + +Rules, all of them learned from breaking them: + +- **POSIX sh, not bash.** It runs under whatever `/bin/sh` the host has, which + on Debian and Ubuntu is dash. A `sh -n` that passes under your own shell + proves nothing about that; `make installer-check` runs `dash -n` and + `shellcheck -s sh` +- **`set -eu`, everything in a function, `main "$@"` on the last line**, so a + truncated download executes nothing. Watch for `[ x ] && y` as a function's + LAST command: it returns non-zero when the test fails, and under `set -e` + that ends the run. Use an `if`, or end with `return 0` +- **Nothing drawn inside a redraw loop may be wider than the terminal.** A + wrapped line is two physical rows while every `ESC[nA` counts logical ones, + so one long option hint makes the menu draw over itself and over whatever was + on screen before it. Everything in a loop goes through `fit` +- **The screen is not ours.** It appends by default, `--clear` is opt-in, and + `ESC[3J` (erase scrollback) is never sent +- **Regenerate the checksum.** `site/public/install.sh.sha256` is what makes + "download, verify, read, run" a real alternative to piping into a shell. + `make installer-sha`, and CI fails when the two disagree +- **Every answer is a flag and a `WARMBLY_*` variable.** An install that can + only be driven by keyboard cannot be driven by Ansible, cloud-init or an + agent, and the wizard exists to be optional +- **Idempotent.** A second run adopts the existing `.env`, never regenerates a + secret (a new `CREDENTIALS_ENCRYPTION_KEY` is permanent data loss) and never + moves an existing data root + +Run `make installer-check` before pushing a change to it (POSIX parse, +shellcheck, `--help`, `--demo`, `--print-env`, a compose file per answer shape, +a pty width regression test, and the checksum). `make installer-demo` is how +you see a UI change without installing anything. + ### Verification: what to run, what to skip Keep the loop fast. The signals that matter are formatting, lint, and typecheck — not local builds or browser automation. @@ -92,6 +146,10 @@ Always, before calling a Go change done: For frontend changes, run `pnpm typecheck` and `pnpm lint` in any tree you touched. +For a change to `site/public/install.sh`, run `make installer-check`; it is the +same script CI runs and it regenerates nothing, so a stale checksum fails there +exactly as it will in CI. + Do not: - do not run `go build ./...`, `pnpm build`, or docker image builds as a "did it work" check. They are slow and are not what CI gates on. `go run` (via the make dev targets) already compiles; `make fmt` + `make lint` + `pnpm typecheck` are the real signals. @@ -116,6 +174,8 @@ Infra runs in docker; the Go services and frontends run natively on the host for - `make web` / `make admin` / `make site` — frontend dev servers (5173 / 5174 / 4321), pointed at the native backend. - `make seed` — load fixtures (after the backend has applied migrations). - `make fmt` / `make lint` — format and lint Go. +- `make installer-demo` — walk the self-host installer's wizard with nothing installed: the real questions and review, a played pull and start. No Docker, no network, no file written. `WARMBLY_DEMO_FAST=1` collapses the animations while iterating on them. +- `make installer-check` / `make installer-sha` — everything CI runs against `site/public/install.sh`, and the checksum regeneration that has to follow any edit to it. Prefer native `make backend` over rebuilding the docker backend image: docker rebuilds are slow because the image bakes in the migrations and the compiled binary, so a one-line change means a full image build + container recreate. The native targets skip all of that. The dockerized hot-reload flow (`make app`) and prod-image smoke test (`make up`) remain available when you specifically need containers. @@ -203,10 +263,11 @@ API keys with the `REALTIME_SUBSCRIBE` permission (bit 11) can connect to the sa - `realtime/`: websocket fanout service - `web/`: in-product frontend (dashboard). Customer-facing only: it holds no platform-admin screens, and operator tooling must not be added back here - `admin/`: platform admin panel (:5174), the single operator surface. Workers, users, orgs, warmup, campaigns, analytics, audit. Every route sits behind `RequireAdmin` and the backend's `RequireAdminPermission` gates -- `site/`: public marketing site (Astro 5 + Tailwind v4) +- `site/`: public marketing site (Astro 5 + Tailwind v4). `site/public/install.sh` is the self-host installer served at warmbly.com/install.sh, with its checksum next to it; see the rules above before touching it - `deploy/`: production deploy manifests, infrastructure, and runtime config - `docs/`: documentation site (docs.warmbly.com); product guides, API reference, and self-hosting/engineering docs under `content/docs/development/` -- `scripts/`: one-off tooling (codegen, migrations, local dev utilities) +- `scripts/`: one-off tooling (codegen, migrations, installer checks, local dev utilities) +- `skills/`: agent playbooks shipped with the repo (`warmbly-api` for the product, `warmbly-ops` for instance administration, `warmbly-install` for standing an instance up and moving it). A command an operator can run is not usable by an agent until it is in one of these ## Worker Topology @@ -787,6 +848,8 @@ These files are the fastest way to rebuild context: - `README.md` - `docs/content/docs/development/architecture.mdx` +- `docs/content/docs/development/install.mdx` and `data-control.mdx` (what a self-hoster is asked, and what each answer decides) +- `site/public/install.sh` (the installer itself) - `cmd/worker/main.go` - `internal/app/worker/assignment.go` - `internal/tasks/email_task.go` diff --git a/Makefile b/Makefile index acad3d11..7591ea53 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,8 @@ PROTO_GEN_FILES := $(PROTO_DIR)/tasks.pb.go up upgrade claim doctor cli seed-demo seed seed-plan sandbox sandbox-seed sandbox-simulate reset logs status stop down test-seed \ restart restart-go restart-all infra infra-down app app-down app-logs \ backend forms forms-web consumer worker run dev tracking realtime web \ - admin site docs grant-admin revoke-admin gen-key db-reset db-wipe migrate + admin site docs grant-admin revoke-admin gen-key installer-sha installer-check installer-demo \ + db-reset db-wipe migrate setup-tools: @echo "Installing required Go tools into $(GO_BIN)" @@ -664,6 +665,24 @@ run: gen-key: @openssl rand -base64 32 +# The one-command installer, served verbatim at https://warmbly.com/install.sh. +# The published checksum is what makes "download, verify, read, run" a real +# alternative to piping into a shell, so it is regenerated with the script and +# CI fails when the two disagree. +installer-sha: + @cd site/public && sha256sum install.sh > install.sh.sha256 && cat install.sh.sha256 + +# Everything CI runs against the installer: POSIX parse, shellcheck, --help, +# --print-env, a compose file per answer shape, and the checksum. +installer-check: + @./scripts/check-installer.sh + +# Walk the installer's wizard without installing anything: the real questions, +# the real review, and a played pull and start. Writes no file, pulls no image, +# needs no Docker. This is the "what does it look like" target. +installer-demo: + @sh site/public/install.sh --demo + # ─── one-command dev stack ─────────────────────────────────────────────── # # `make dev` is the "just make it work" target for a fresh clone or a fresh diff --git a/README.md b/README.md index a0d73998..5533235d 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,29 @@ the native services, and how seeding works are in the Warmbly runs with **no cloud account of any kind**: no AWS, no GCP, no Stripe, no Kafka. One command brings up the whole platform on local, open-source pieces: +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +That pulls the published release images, writes a real `.env`, starts the stack +and prints the one-time link that claims the instance and makes you its admin. +No clone, no compiler, under two minutes. Add `--wizard` and it asks where each +store lives, what is kept and for how long, and how it is backed up. The script +is checksummed at `install.sh.sha256` if you would rather download, verify and +read it first, `--dry-run` prints every file it would write without touching +anything, and `--demo` walks the whole wizard while installing nothing. Full +flag list: [install](https://docs.warmbly.com/development/install/). + +To build it from source instead: + ```bash git clone https://github.com/warmbly/warmbly && cd warmbly make up ``` -That is the whole install: `make up` waits for the backend and prints a one-time -link that claims the instance and makes you its admin. You need Docker with -Compose v2 and about 10 GB of free disk. If anything looks wrong, `make doctor` -prints the instance state and every failing check. +Either way you need Docker with Compose v2; the build path also wants about +10 GB of free disk. If anything looks wrong, `make doctor` prints the instance +state and every failing check. From here the [self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) @@ -134,7 +148,9 @@ The full docs live at **[docs.warmbly.com](https://docs.warmbly.com)**. | Read this | To learn | |-----------|----------| -| [Self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) | Step-by-step install, then production, backups, and scaling the worker fleet | +| [Install](https://docs.warmbly.com/development/install/) | The one-command install and its wizard: where the data lives, what is kept, how it is backed up | +| [Self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) | Building from source, then production, backups, and scaling the worker fleet | +| [Data control](https://docs.warmbly.com/development/data-control/) | Every store's location, every retention window, instance backups, and moving to another host | | [First run](https://docs.warmbly.com/development/first-run/) | Claiming the instance, reissuing the setup link, and what to do when accounts already exist | | [Accounts and access](https://docs.warmbly.com/development/accounts-and-access/) | Registration modes, inviting people with or without a mail relay, SSO, and recovering access | | [`warmblyctl`](https://docs.warmbly.com/development/warmblyctl/) | The operator CLI: creating accounts, setting passwords, granting admin, and instance status | diff --git a/admin/src/app/dashboard/InstanceSettingsPage.tsx b/admin/src/app/dashboard/InstanceSettingsPage.tsx index edaf7a2f..71004f2c 100644 --- a/admin/src/app/dashboard/InstanceSettingsPage.tsx +++ b/admin/src/app/dashboard/InstanceSettingsPage.tsx @@ -76,11 +76,57 @@ const SYNC_FIELDS = [ type SyncFieldKey = (typeof SYNC_FIELDS)[number]["key"]; +// Retention windows, mirroring internal/config/constants.go. Each one is also +// how long the personal data in that log is held, so the help text says what +// the data is rather than only what the number does. +const RETENTION_MIN_DAYS = 1; +const RETENTION_MAX_DAYS = 3650; + +const RETENTION_FIELDS = [ + { + key: "engagementDays", + setting: "engagement_event_days", + label: "Opens and clicks (days)", + help: "Per-event open and click logs, with the client, device and approximate location of each. Campaign counts and routing read a separate summary that is never pruned, so shortening this changes what a contact's timeline can show, not what a campaign does.", + }, + { + key: "formDays", + setting: "form_event_days", + label: "Form funnel events (days)", + help: "Views, starts, field-level drop-off and submissions for hosted forms. Funnel reports range up to 90 days, so anything below that shortens the report too. Submitted contacts are unaffected.", + }, + { + key: "auditDays", + setting: "audit_log_days", + label: "Audit log (days)", + help: "Who did what, from which IP address and user agent, with the change payload. This window is how long that record is held, and it is the one most likely to be set by a retention policy.", + }, +] as const; + +type RetentionFieldKey = (typeof RETENTION_FIELDS)[number]["key"]; + +// The two presets are the ends of the band people actually choose between. +const RETENTION_PRESETS = [ + { + id: "default", + label: "Defaults", + description: "365 / 180 / 90 days", + values: { engagementDays: "365", formDays: "180", auditDays: "90" }, + }, + { + id: "minimal", + label: "Minimal retention", + description: "30 / 30 / 30 days", + values: { engagementDays: "30", formDays: "30", auditDays: "30" }, + }, +] as const; + interface FormState { linksEnabled: boolean; ttlHours: string; allowInvitedSignup: boolean; sync: Record; + retention: Record; enforceDomainAuth: boolean; authGraceHours: string; } @@ -96,6 +142,11 @@ function toForm(s: InstanceSettings): FormState { dailyPerMailbox: String(s.sync.daily_messages_per_mailbox), dailyPerOrg: String(s.sync.daily_messages_per_org), }, + retention: { + engagementDays: String(s.retention.engagement_event_days), + formDays: String(s.retention.form_event_days), + auditDays: String(s.retention.audit_log_days), + }, enforceDomainAuth: s.deliverability.enforce_domain_auth, authGraceHours: String(s.deliverability.auth_grace_hours), }; @@ -137,6 +188,12 @@ export default function InstanceSettingsPage() { !!server && !!form && SYNC_FIELDS.some((f) => form.sync[f.key] !== String(server.sync[f.setting])); + const retentionDirty = + !!server && + !!form && + RETENTION_FIELDS.some( + (f) => form.retention[f.key] !== String(server.retention[f.setting]), + ); const dirty = !!server && !!form && @@ -145,9 +202,15 @@ export default function InstanceSettingsPage() { form.allowInvitedSignup !== server.access.allow_invited_signup || form.enforceDomainAuth !== server.deliverability.enforce_domain_auth || form.authGraceHours !== String(server.deliverability.auth_grace_hours) || + retentionDirty || syncDirty); const syncValid = form !== null && SYNC_FIELDS.every((f) => syncFieldValid(form.sync[f.key], f.min, f.max)); + const retentionValid = + form !== null && + RETENTION_FIELDS.every((f) => + syncFieldValid(form.retention[f.key], RETENTION_MIN_DAYS, RETENTION_MAX_DAYS), + ); const authGrace = form ? Number(form.authGraceHours) : NaN; const authGraceValid = @@ -177,6 +240,12 @@ export default function InstanceSettingsPage() { toast.error("Every sync budget must be a whole number inside its range"); return; } + if (!retentionValid) { + toast.error( + `Every retention window must be a whole number of days between ${RETENTION_MIN_DAYS} and ${RETENTION_MAX_DAYS.toLocaleString()}`, + ); + return; + } if (!authGraceValid) { toast.error( `The authentication grace period must be a whole number of hours between ${AUTH_GRACE_MIN_HOURS} and ${AUTH_GRACE_MAX_HOURS}`, @@ -192,6 +261,11 @@ export default function InstanceSettingsPage() { daily_messages_per_mailbox: Number(form.sync.dailyPerMailbox), daily_messages_per_org: Number(form.sync.dailyPerOrg), }, + retention: { + engagement_event_days: Number(form.retention.engagementDays), + form_event_days: Number(form.retention.formDays), + audit_log_days: Number(form.retention.auditDays), + }, deliverability: { enforce_domain_auth: form.enforceDomainAuth, auth_grace_hours: authGrace, @@ -370,6 +444,93 @@ export default function InstanceSettingsPage() { + + + Data retention + + How long event-level history is kept on this instance. Every window + below is also how long the personal data in that log is held, so + these are the settings a retention or privacy policy applies to. A + sweep runs a few times a day and reads these values each pass, so a + change takes effect without a restart. Deletion is permanent: + shortening a window removes what already sits outside it on the + next sweep. + + + +
+ Presets + {RETENTION_PRESETS.map((preset) => { + const active = RETENTION_FIELDS.every( + (f) => form.retention[f.key] === preset.values[f.key], + ); + return ( + + ); + })} +
+
+ {RETENTION_FIELDS.map((f) => { + const valid = syncFieldValid( + form.retention[f.key], + RETENTION_MIN_DAYS, + RETENTION_MAX_DAYS, + ); + return ( +
+ + + setForm({ + ...form, + retention: { + ...form.retention, + [f.key]: e.target.value, + }, + }) + } + aria-invalid={!valid} + className="mt-1" + /> +

+ {f.help} Between {RETENTION_MIN_DAYS} and{" "} + {RETENTION_MAX_DAYS.toLocaleString()} days. +

+ {!valid && ( +

+ Enter a whole number of days between{" "} + {RETENTION_MIN_DAYS} and{" "} + {RETENTION_MAX_DAYS.toLocaleString()}. +

+ )} +
+ ); + })} +
+ + + Sending-domain authentication diff --git a/admin/src/components/layout/UpdateDialog.tsx b/admin/src/components/layout/UpdateDialog.tsx index 9091be59..ca45c47a 100644 --- a/admin/src/components/layout/UpdateDialog.tsx +++ b/admin/src/components/layout/UpdateDialog.tsx @@ -13,6 +13,7 @@ import { ExternalLink, GitBranch, Loader2, + Package, RefreshCw, RotateCw, XCircle, @@ -51,6 +52,7 @@ const DOCS_UPDATES = "/development/updates/"; const STEPS: Record = { compose: ["fetch", "checkout", "build", "restart", "prune", "wait"], + image: ["resolve", "pull", "restart", "prune", "wait"], command: ["fetch", "checkout", "command", "wait"], }; @@ -58,6 +60,8 @@ const STEP_LABELS: Record = { fetch: "Fetch", checkout: "Pull", build: "Build", + resolve: "Pin release", + pull: "Pull images", restart: "Restart", prune: "Clean up", command: "Run script", @@ -137,6 +141,9 @@ export function UpdateDialog({ open, onOpenChange }: Props) { const updater = state?.updater; const checkout = updater?.checkout; const job = updater?.job ?? updater?.last_job; + // An image install never builds, so the confirmation must not promise a + // rebuild it will not do. + const imageMode = updater?.mode === "image"; return ( @@ -170,8 +177,9 @@ export function UpdateDialog({ open, onOpenChange }: Props) {
- This pulls the checkout, rebuilds the images and restarts every - service. + {imageMode + ? "This pulls the release images and restarts every service." + : "This pulls the checkout, rebuilds the images and restarts every service."}
Sending and syncing pause for a few minutes and resume on their own; @@ -280,6 +288,7 @@ export function UpdateDialog({ open, onOpenChange }: Props) { function Overview({ state }: { state: UpdateState }) { const { latest, updater } = state; const checkout = updater.checkout; + const release = updater.release; return (
Latest release
@@ -332,6 +341,23 @@ function Overview({ state }: { state: UpdateState }) { )} + {release && ( + <> +
Installed
+
+ + + {release.prefix}/*:{release.tag} + + + {release.pinned + ? "pinned to this release" + : "following the channel tag"} + +
+ + )} + {checkout && ( <>
Checkout
@@ -371,12 +397,17 @@ function Overview({ state }: { state: UpdateState }) { function UpdaterNotice({ state }: { state: UpdateState }) { const u = state.updater; + // A clone-free install has no checkout to pull, so the by-hand command is + // the compose one. The release block is the only signal for which it is, + // and an unreachable updater does not report one, so fall back to naming + // both rather than printing a command that cannot work. + const byHand = u.release ? "docker compose pull && docker compose up -d" : "git pull && make up"; if (u.status === "unreachable") { return (
The updater is not answering
- {u.error} Until it does, update by hand: - git pull && make up + {u.error} Until it does, update by hand from the install directory: + {byHand} ); } @@ -384,9 +415,9 @@ function UpdaterNotice({ state }: { state: UpdateState }) {
This panel can only report
No updater is configured, so apply updates from a shell on the host: - git pull && make up - To get the button, enable the updater compose profile (`make up` does) or run the - updater unit on a bare-metal host. + {byHand} + To get the button, enable the updater compose profile (`make up` and the installer + both do) or run the updater unit on a bare-metal host. ); } diff --git a/admin/src/lib/api/client/admin/instance.ts b/admin/src/lib/api/client/admin/instance.ts index 432d4593..336c126a 100644 --- a/admin/src/lib/api/client/admin/instance.ts +++ b/admin/src/lib/api/client/admin/instance.ts @@ -111,6 +111,16 @@ export interface InstanceSettings { daily_messages_per_mailbox: number; daily_messages_per_org: number; }; + // How long event-level history is kept. Every window bounds personal data: + // opens and clicks carry a client, a device and a location, funnel events + // carry a visitor's path, and the audit trail carries IP addresses, user + // agents and change payloads. None of them affect a count or a routing + // decision; campaign progress keeps its own summary. + retention: { + engagement_event_days: number; + form_event_days: number; + audit_log_days: number; + }; // The sending-domain authentication gate. A mailbox whose domain has been // failing SPF or DMARC for longer than the grace window stops sending cold // mail and warmup mail until the records are fixed. diff --git a/admin/src/lib/api/client/admin/updates.ts b/admin/src/lib/api/client/admin/updates.ts index 9593ab66..84ddbd30 100644 --- a/admin/src/lib/api/client/admin/updates.ts +++ b/admin/src/lib/api/client/admin/updates.ts @@ -29,6 +29,16 @@ export interface UpdaterCheckout { fetch_error?: string; } +// An image install has no checkout: what it runs is the tag pinned in its +// .env, which the updater reports instead. +export interface UpdaterRelease { + tag: string; + prefix: string; + // False for a moving channel tag (prod, dev), where re-pulling the same + // name is itself an update. + pinned: boolean; +} + export type UpdateJobStatus = "running" | "succeeded" | "failed"; export interface UpdateJob { @@ -51,9 +61,12 @@ export interface UpdaterView { configured: boolean; status: UpdaterStatus; error?: string; - mode?: "compose" | "command"; + mode?: "compose" | "image" | "command"; repo_dir?: string; + // Exactly one of these: a checkout in compose and command mode, a release + // in image mode. checkout?: UpdaterCheckout; + release?: UpdaterRelease; job?: UpdateJob; last_job?: UpdateJob; } diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 617c5a9f..3b3d7ec4 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "log" @@ -589,6 +590,7 @@ func main() { trackedLinkRepository = repository.NewTrackedLinkRepository(primaryDB.Pool) instanceChecksDB = primaryDB.Pool instanceSettings = instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool)) + bootstrapInstanceSettings(ctx, instanceSettings) if err != nil { sentry.CaptureException(err) log.Fatal(err) @@ -1239,7 +1241,7 @@ func main() { formService.SetLinks(repository.NewFormLinkRepository(primaryDB)) formService.SetEvents(formEventRepository) formService.SetDomains(organizationRepoForHandler) - go jobs.NewFormEventsRetentionJob(formEventRepository).Start(ctx, 12*time.Hour) + go jobs.NewFormEventsRetentionJob(formEventRepository).WireRetention(instanceSettings).Start(ctx, 12*time.Hour) go jobs.NewFormsDomainSweep(organizationRepoForHandler).Start(ctx, time.Hour) // A visibly bad import is filed on the workspace's posture. On its own // it can only reach `watch`, which changes nothing. @@ -1689,10 +1691,11 @@ func main() { orgTransferScheduler := jobs.NewOrgTransferScheduler(orgTransferJob, 1*time.Hour) go orgTransferScheduler.Start(ctx) - // Prune audit entries past the retention window (90 days). Bounding the - // trail's age also bounds how long PII is retained. auditRepository is + // Prune audit entries past the retention window. Bounding the trail's + // age also bounds how long PII is retained, so the window is an + // instance setting and is read on every pass. auditRepository is // constructed earlier (before authService). - auditRetentionJob := jobs.NewAuditRetentionJob(auditRepository, 90*24*time.Hour) + auditRetentionJob := jobs.NewAuditRetentionJob(auditRepository).WireRetention(instanceSettings) auditRetentionScheduler := jobs.NewAuditRetentionScheduler(auditRetentionJob, 6*time.Hour) go auditRetentionScheduler.Start(ctx) @@ -2171,3 +2174,33 @@ func (m advisorMembers) MemberPermissions(ctx context.Context, orgID, userID uui } return member.Permissions, nil } + +// bootstrapInstanceSettings applies WARMBLY_SETTINGS_BOOTSTRAP once, on an +// instance whose settings document has never been written. It is how an +// unattended install ships its data-control answers (what is imported, what is +// kept and for how long) with the rest of the environment, so the wizard's +// choices are in place before the first mailbox is connected instead of being +// something the operator has to redo in the panel. +// +// The body is the same partial document PUT /admin/instance/settings takes. +// From the first write onwards the panel is authoritative and this is a no-op, +// so the variable can stay in .env without ever undoing a later edit. +func bootstrapInstanceSettings(ctx context.Context, svc instancesettings.Service) { + raw := strings.TrimSpace(os.Getenv("WARMBLY_SETTINGS_BOOTSTRAP")) + if raw == "" || svc == nil { + return + } + var patch instancesettings.Patch + if err := json.Unmarshal([]byte(raw), &patch); err != nil { + log.Printf("WARMBLY_SETTINGS_BOOTSTRAP is not valid JSON and was ignored: %v", err) + return + } + applied, err := svc.Bootstrap(ctx, patch) + if err != nil { + log.Printf("WARMBLY_SETTINGS_BOOTSTRAP could not be applied: %v", err) + return + } + if applied { + log.Printf("instance settings seeded from WARMBLY_SETTINGS_BOOTSTRAP") + } +} diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index 840b4899..bda09019 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -23,6 +23,7 @@ import ( "github.com/warmbly/warmbly/internal/app/creditwatch" "github.com/warmbly/warmbly/internal/app/feature" "github.com/warmbly/warmbly/internal/app/inboxagent" + "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/app/integration" "github.com/warmbly/warmbly/internal/app/nativeactions" "github.com/warmbly/warmbly/internal/app/notification" @@ -499,6 +500,9 @@ func main() { ); terr != nil { log.Println("tracking consumer unavailable; opens/clicks not consumed:", terr) } else { + // The engagement prune reads its window from the instance settings on + // every pass, so shortening it in the admin panel needs no restart. + trackingConsumer.WireRetention(instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool))) defer trackingConsumer.Close() go func() { if err := trackingConsumer.Start(ctx); err != nil { diff --git a/cmd/updater/main.go b/cmd/updater/main.go index dc5c5667..48673763 100644 --- a/cmd/updater/main.go +++ b/cmd/updater/main.go @@ -1,6 +1,10 @@ // The updater is the host-side agent behind "Update and restart" in the admin -// panel. It moves the checkout forward, rebuilds and restarts the stack, and -// reports progress to the backend. See docs/content/docs/development/updates.mdx. +// panel. It moves the install forward and restarts the stack, then reports +// progress to the backend. UPDATER_MODE picks how: compose pulls a git +// checkout and rebuilds it, image pins a release tag in the install's .env and +// pulls the published images (the clone-free install.sh install), command +// hands the rebuild to a script of your own. +// See docs/content/docs/development/updates.mdx. package main import ( @@ -42,8 +46,10 @@ func main() { AllowDirty: boolean("UPDATER_ALLOW_DIRTY", false), Version: version.String(), } - if cfg.Mode != updater.ModeCompose && cfg.Mode != updater.ModeCommand { - log.Fatalf("updater: UPDATER_MODE must be compose or command, got %q", cfg.Mode) + switch cfg.Mode { + case updater.ModeCompose, updater.ModeCommand, updater.ModeImage: + default: + log.Fatalf("updater: UPDATER_MODE must be compose, image or command, got %q", cfg.Mode) } runner, err := updater.NewRunner(cfg) @@ -70,7 +76,7 @@ func main() { defer cancel() _ = srv.Shutdown(shutdownCtx) }() - log.Printf("updater: %s mode, checkout %s, listening on %s (version %s)", cfg.Mode, cfg.RepoDir, addr, cfg.Version) + log.Printf("updater: %s mode, install %s, listening on %s (version %s)", cfg.Mode, cfg.RepoDir, addr, cfg.Version) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("updater: %v", err) } diff --git a/cmd/warmblyctl/backup.go b/cmd/warmblyctl/backup.go new file mode 100644 index 00000000..4e632827 --- /dev/null +++ b/cmd/warmblyctl/backup.go @@ -0,0 +1,783 @@ +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/warmbly/warmbly/internal/version" +) + +// Instance-level backup and restore: one bundle holding everything a Warmbly +// install is, restorable onto a fresh host. +// +// This is deliberately not `warmblyctl org export`. That one moves ONE +// workspace between two running instances and re-seals its secrets for the +// destination's keys. This one moves THE INSTANCE: every workspace, every +// user, the platform admins, the API keys, and the keys those secrets are +// sealed with, onto a host that has nothing yet. +// +// The reason it is one command rather than a documented list of steps is that +// the parts are only a backup together. A pg_dump alone restores an instance +// whose every mailbox credential decrypts to nothing, because the ciphertext +// is in the database and the key that opens it is in .env; the blob root alone +// restores bodies nothing points at. The bundle carries all three and the +// restore refuses to run when the keys do not match, which is the failure this +// command exists to make impossible. + +// bundleVersion is the archive layout. The restore refuses a version it does +// not know rather than half-applying an archive it cannot read. +const bundleVersion = 1 + +// Paths inside the bundle. +const ( + manifestPath = "manifest.json" + databasePath = "database.sql" + keysPath = "keys.env" + blobsPrefix = "blobs/" +) + +// backupKeys are the environment values that belong in the bundle. The first +// two are unrecoverable: every sealed mailbox credential and every per-org DEK +// is opened with them, so a database without them is not a backup. The rest +// are here because restoring without them signs out every session and breaks +// every worker and websocket connection, which reads as a broken restore. +var backupKeys = []string{ + "CREDENTIALS_ENCRYPTION_KEY", + "KMS_LOCAL_MASTER_KEY", + "AUTH_SECRET", + "INTERNAL_API_TOKEN", + "SECRET_KEY_BASE", +} + +// unrecoverableKeys are the two the restore checks. A mismatch on either is +// refused, because it produces an instance that looks restored and whose +// mailboxes authenticate against nothing. +var unrecoverableKeys = []string{"CREDENTIALS_ENCRYPTION_KEY", "KMS_LOCAL_MASTER_KEY"} + +// manifest describes what a bundle holds, so the restore can say what it is +// about to do before it does it. +type manifest struct { + Version int `json:"version"` + CreatedAt time.Time `json:"created_at"` + AppVersion string `json:"app_version"` + AppURL string `json:"app_url,omitempty"` + Database string `json:"database"` + DatabaseSHA string `json:"database_sha256"` + // Counts are for the confirmation line, not for logic. + Organizations int `json:"organizations"` + Users int `json:"users"` + Mailboxes int `json:"mailboxes"` + // BlobProvider records why blobs are or are not in the bundle. An s3 + // install keeps its bodies in the bucket, and the bucket is not ours to + // copy, so the archive says so instead of pretending to be complete. + BlobProvider string `json:"blob_provider"` + BlobRoot string `json:"blob_root,omitempty"` + BlobFiles int `json:"blob_files"` + BlobBytes int64 `json:"blob_bytes"` + // Keys is the list of key names in keys.env, never their values. + Keys []string `json:"keys"` +} + +// ---------- backup ---------- + +func runBackup(ctx context.Context, args []string) error { + fs := newFlagSet("backup") + out := fs.String("out", "", "Path to write the bundle to. Defaults to ./warmbly-backup-.tar.gz") + noKeys := fs.Bool("no-keys", false, "Leave the encryption keys out. The bundle is then not restorable on its own.") + noBlobs := fs.Bool("no-blobs", false, "Leave message bodies, attachments and avatars out. Database only.") + force := fs.Bool("force", false, "Overwrite the output file if it already exists") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + + target := strings.TrimSpace(*out) + if target == "" { + target = fmt.Sprintf("warmbly-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405")) + } + if _, err := os.Stat(target); err == nil && !*force { + return fmt.Errorf("%s already exists. Pass --force to overwrite it, or --out with another path.", target) + } + + dsn, err := dbEndpoint(ctx) + if err != nil { + return err + } + if _, err := exec.LookPath("pg_dump"); err != nil { + return errors.New("pg_dump is not on PATH, so the database cannot be dumped.\nRun this inside the backend container, where it is installed:\n " + composeExec + "backup --out /data/blobs/warmbly-backup.tar.gz") + } + + m := manifest{ + Version: bundleVersion, + CreatedAt: time.Now().UTC(), + AppVersion: version.String(), + AppURL: strings.TrimSpace(os.Getenv("APP_URL")), + Database: databaseName(dsn), + } + + // Counts first: they are the line that tells an operator this bundle is of + // the instance they meant, and a database that cannot be counted cannot be + // dumped either, so failing here fails before anything is written. + c, err := connect(ctx) + if err != nil { + return err + } + m.Organizations, m.Users, m.Mailboxes = instanceCounts(ctx, c) + c.close() + + fmt.Printf("Backing up %s\n", redact(dsn)) + + dumpFile, dumpSize, dumpSHA, err := dumpDatabase(ctx, dsn) + if err != nil { + return err + } + defer func() { + _ = os.Remove(dumpFile) + }() + m.DatabaseSHA = dumpSHA + fmt.Printf(" database %s\n", humanBytes(dumpSize)) + + var blobFiles []blobEntry + m.BlobProvider = strings.ToLower(strings.TrimSpace(os.Getenv("BLOB_PROVIDER"))) + if m.BlobProvider == "" { + m.BlobProvider = "filesystem" + } + switch { + case *noBlobs: + fmt.Println(" blobs skipped (--no-blobs)") + case m.BlobProvider != "filesystem": + fmt.Printf(" blobs not included: this instance stores them in %s. Back that store up separately.\n", m.BlobProvider) + default: + root := blobRoot() + m.BlobRoot = root + blobFiles, err = collectBlobs(root) + if err != nil { + return err + } + for _, b := range blobFiles { + m.BlobBytes += b.size + } + m.BlobFiles = len(blobFiles) + fmt.Printf(" blobs %d files, %s from %s\n", m.BlobFiles, humanBytes(m.BlobBytes), root) + } + + keys := map[string]string{} + if !*noKeys { + for _, k := range backupKeys { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + keys[k] = v + m.Keys = append(m.Keys, k) + } + } + sort.Strings(m.Keys) + missing := missingUnrecoverable(keys) + if len(missing) > 0 { + warn("%s not set in this environment, so %s not in the bundle. Restoring it elsewhere will not open sealed mailbox credentials.", + strings.Join(missing, " and "), plural(len(missing), "it is", "they are")) + } + fmt.Printf(" keys %d (%s)\n", len(m.Keys), strings.Join(m.Keys, ", ")) + } else { + fmt.Println(" keys skipped (--no-keys); this bundle cannot restore a working instance on its own") + } + + if err := writeBundle(target, m, dumpFile, blobFiles, keys); err != nil { + return err + } + + st, _ := os.Stat(target) + size := int64(0) + if st != nil { + size = st.Size() + } + fmt.Printf("\nWrote %s (%s)\n", target, humanBytes(size)) + steps := []string{ + "Copy it off this host. It holds every mailbox credential and the keys that open them,", + "so treat the file as you would the instance itself: 0600, encrypted at rest, off-site.", + "", + "Restore it on another host with:", + " " + composeExec + "restore --file /path/to/" + filepath.Base(target), + } + if len(m.Keys) == 0 { + steps = append(steps, "", "This bundle carries no keys. Copy CREDENTIALS_ENCRYPTION_KEY and KMS_LOCAL_MASTER_KEY", "from this instance's .env by hand, or the restored mailboxes will not connect.") + } + printSteps("Next:", steps) + return nil +} + +// ---------- restore ---------- + +func runRestore(ctx context.Context, args []string) error { + fs := newFlagSet("restore") + file := fs.String("file", "", "Path to a bundle written by warmblyctl backup (required)") + yes := fs.Bool("yes", false, "Skip the typed confirmation. For scripts.") + noBlobs := fs.Bool("no-blobs", false, "Restore the database only, leaving the blob root alone") + force := fs.Bool("force", false, "Restore even though this host's encryption keys differ from the bundle's") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + if strings.TrimSpace(*file) == "" { + return errors.New("--file is required. Point it at a bundle written by `warmblyctl backup`.") + } + if _, err := exec.LookPath("psql"); err != nil { + return errors.New("psql is not on PATH, so the database cannot be restored.\nRun this inside the backend container, where it is installed.") + } + + m, err := readManifest(*file) + if err != nil { + return err + } + if m.Version != bundleVersion { + return fmt.Errorf("this bundle is layout version %d and this warmblyctl reads version %d. Restore it with the Warmbly it was written by (%s).", m.Version, bundleVersion, m.AppVersion) + } + + dsn, err := dbEndpoint(ctx) + if err != nil { + return err + } + + fmt.Printf("Bundle %s\n", *file) + fmt.Printf(" written %s by Warmbly %s\n", m.CreatedAt.Local().Format(time.RFC1123), m.AppVersion) + if m.AppURL != "" { + fmt.Printf(" from %s\n", m.AppURL) + } + fmt.Printf(" holds %d organizations, %d users, %d mailboxes\n", m.Organizations, m.Users, m.Mailboxes) + if m.BlobFiles > 0 { + fmt.Printf(" %d blob files, %s\n", m.BlobFiles, humanBytes(m.BlobBytes)) + } + fmt.Printf("Target %s\n\n", redact(dsn)) + + // The key check is the point of this command. A restore onto a host whose + // keys differ produces an instance that looks fine and whose every mailbox + // fails to authenticate, days later, with no error that names the cause. + if err := checkRestoreKeys(*file, m, *force); err != nil { + return err + } + + if !*yes { + fmt.Printf("This REPLACES everything in %s. Every organization, user, campaign and\nmailbox currently on this instance is dropped and replaced by the bundle's.\n\n", databaseName(dsn)) + ok, cerr := confirmPhrase("Type 'restore' to continue: ", "restore") + if cerr != nil { + return cerr + } + if !ok { + return errors.New("nothing was changed") + } + fmt.Println() + } + + fmt.Println("Restoring the database...") + if err := restoreDatabase(ctx, dsn, *file); err != nil { + return err + } + fmt.Println(" database restored") + + if !*noBlobs && m.BlobFiles > 0 { + root := blobRoot() + n, bytes, rerr := restoreBlobs(*file, root) + if rerr != nil { + return rerr + } + fmt.Printf(" blobs %d files, %s into %s\n", n, humanBytes(bytes), root) + } + + steps := []string{} + if len(m.Keys) > 0 { + steps = append(steps, + "The bundle's keys match this host, so sealed mailbox credentials open as they did.", + ) + } + steps = append(steps, + "Restart the stack so every service picks the restored database up:", + " docker compose -p warmbly restart", + "", + "Then check it:", + " "+composeExec+"status", + ) + printSteps("Next:", steps) + return nil +} + +// checkRestoreKeys refuses a restore whose ciphertext this host cannot open. +// +// The bundle's keys are compared against the environment rather than written +// anywhere: warmblyctl runs inside a container and the .env that would have to +// change is on the host, so the honest thing is to print the two lines and +// stop, not to half-apply and report success. +func checkRestoreKeys(file string, m manifest, force bool) error { + if len(m.Keys) == 0 { + warn("This bundle carries no encryption keys. If this host's CREDENTIALS_ENCRYPTION_KEY and\nKMS_LOCAL_MASTER_KEY are not the ones the bundle was written with, every restored mailbox\ncredential will fail to decrypt.") + return nil + } + bundled, err := readKeys(file) + if err != nil { + return err + } + var mismatched, missing []string + for _, k := range unrecoverableKeys { + want, inBundle := bundled[k] + if !inBundle { + continue + } + got := strings.TrimSpace(os.Getenv(k)) + switch { + case got == "": + missing = append(missing, k+"="+want) + case got != want: + mismatched = append(mismatched, k+"="+want) + } + } + if len(mismatched) == 0 && len(missing) == 0 { + return nil + } + + lines := append(append([]string{}, missing...), mismatched...) + msg := strings.Builder{} + msg.WriteString("This host's encryption keys are not the ones the bundle was sealed with.\n") + msg.WriteString("Restoring anyway gives you an instance whose mailbox credentials cannot be\ndecrypted, and there is no way to recover them afterwards.\n\n") + msg.WriteString("Put these in the install's .env on the host, recreate the containers, then run\nthe restore again:\n\n") + for _, l := range lines { + msg.WriteString(" " + l + "\n") + } + if force { + warn("%s", msg.String()) + warn("--force was given, so the restore continues. Mailboxes will need reconnecting.") + return nil + } + msg.WriteString("\nPass --force only if you accept losing every stored mailbox credential.") + return errors.New(msg.String()) +} + +// ---------- database ---------- + +// dumpDatabase writes a plain-SQL dump to a temp file and returns its path, +// size and digest. Plain rather than custom format so the bundle can be +// inspected, and so a restore needs psql alone. +func dumpDatabase(ctx context.Context, dsn string) (string, int64, string, error) { + tmp, err := os.CreateTemp("", "warmbly-dump-*.sql") + if err != nil { + return "", 0, "", err + } + path := tmp.Name() + + sum := sha256.New() + w := bufio.NewWriterSize(io.MultiWriter(tmp, sum), 1<<20) + + cmd := exec.CommandContext(ctx, "pg_dump", + "--dbname="+dsn, + // No owner or ACL statements: the destination's database user is + // whatever its own compose file created, and it is never guaranteed to + // carry the same name as this one's. + "--no-owner", "--no-privileges", + "--format=plain", + ) + cmd.Stdout = w + var stderr strings.Builder + cmd.Stderr = &stderr + if rerr := cmd.Run(); rerr != nil { + _ = tmp.Close() + _ = os.Remove(path) + return "", 0, "", fmt.Errorf("pg_dump failed: %s", strings.TrimSpace(stderr.String())) + } + if ferr := w.Flush(); ferr != nil { + _ = tmp.Close() + _ = os.Remove(path) + return "", 0, "", ferr + } + size, _ := tmp.Seek(0, io.SeekCurrent) + if cerr := tmp.Close(); cerr != nil { + _ = os.Remove(path) + return "", 0, "", cerr + } + return path, size, hex.EncodeToString(sum.Sum(nil)), nil +} + +// restoreDatabase empties the schema and replays the dump into it. The schema +// is dropped rather than the dump carrying DROP statements, because the +// destination has already had migrations applied at boot and objects the +// bundle does not know about would otherwise survive into the restored +// instance. +func restoreDatabase(ctx context.Context, dsn, bundle string) error { + reset := exec.CommandContext(ctx, "psql", "--dbname="+dsn, "-v", "ON_ERROR_STOP=1", "-q", + "-c", "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;") + var resetErr strings.Builder + reset.Stderr = &resetErr + if err := reset.Run(); err != nil { + return fmt.Errorf("could not empty the target schema: %s", strings.TrimSpace(resetErr.String())) + } + + sql, closeFn, err := openBundleEntry(bundle, databasePath) + if err != nil { + return err + } + defer closeFn() + + load := exec.CommandContext(ctx, "psql", "--dbname="+dsn, "-v", "ON_ERROR_STOP=1", "-q") + load.Stdin = sql + var loadErr strings.Builder + load.Stderr = &loadErr + if err := load.Run(); err != nil { + return fmt.Errorf("the dump did not load cleanly, so the database is now empty and unusable.\nFix the cause and run the restore again:\n%s", strings.TrimSpace(loadErr.String())) + } + return nil +} + +func instanceCounts(ctx context.Context, c *conn) (orgs, users, mailboxes int) { + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM organizations`).Scan(&orgs) + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&users) + _ = c.db.QueryRow(ctx, `SELECT count(*) FROM email_accounts`).Scan(&mailboxes) + return +} + +// databaseName is the database a connection string names, for the line that +// tells an operator what is about to be replaced. +func databaseName(dsn string) string { + if i := strings.LastIndex(dsn, "/"); i >= 0 { + name := dsn[i+1:] + if j := strings.IndexAny(name, "?"); j >= 0 { + name = name[:j] + } + if name != "" { + return name + } + } + return "the database" +} + +// ---------- blobs ---------- + +type blobEntry struct { + abs string + rel string + size int64 + mode os.FileMode +} + +func blobRoot() string { + if v := strings.TrimSpace(os.Getenv("BLOB_FS_ROOT")); v != "" { + return v + } + return "/data/blobs" +} + +func collectBlobs(root string) ([]blobEntry, error) { + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + warn("the blob root %s does not exist on this host, so no bodies or attachments are in the bundle.", root) + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + var out []blobEntry + err = filepath.WalkDir(root, func(path string, d os.DirEntry, werr error) error { + if werr != nil { + return werr + } + // Symlinks are skipped rather than followed: the blob root is written + // by the app, and a link out of it would put arbitrary host files in a + // bundle the operator believes holds message bodies. + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + st, serr := d.Info() + if serr != nil { + return serr + } + rel, rerr := filepath.Rel(root, path) + if rerr != nil { + return rerr + } + out = append(out, blobEntry{abs: path, rel: filepath.ToSlash(rel), size: st.Size(), mode: st.Mode().Perm()}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].rel < out[j].rel }) + return out, nil +} + +// restoreBlobs unpacks the bundle's blob tree under root. Existing files are +// overwritten; files the bundle does not know about are left alone, so a +// restore onto a host that already holds bodies is additive rather than a +// silent deletion. +func restoreBlobs(bundle, root string) (int, int64, error) { + f, gz, tr, err := openBundle(bundle) + if err != nil { + return 0, 0, err + } + defer f.Close() + defer gz.Close() + + var n int + var total int64 + for { + hdr, nerr := tr.Next() + if nerr == io.EOF { + break + } + if nerr != nil { + return n, total, nerr + } + if hdr.Typeflag != tar.TypeReg || !strings.HasPrefix(hdr.Name, blobsPrefix) { + continue + } + rel := strings.TrimPrefix(hdr.Name, blobsPrefix) + dest, derr := safeJoin(root, rel) + if derr != nil { + return n, total, derr + } + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return n, total, err + } + out, oerr := os.OpenFile(dest, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode).Perm()) //nolint:gosec // path is checked by safeJoin + if oerr != nil { + return n, total, oerr + } + written, cerr := io.Copy(out, tr) //nolint:gosec // sizes come from a bundle the operator wrote + _ = out.Close() + if cerr != nil { + return n, total, cerr + } + n++ + total += written + } + return n, total, nil +} + +// safeJoin refuses a bundle entry that would land outside the destination. +// A backup is an operator's own file, but it is also the kind of file that +// gets emailed around, and a path traversal in one is a host compromise. +func safeJoin(root, rel string) (string, error) { + clean := filepath.Clean("/" + filepath.FromSlash(rel)) + joined := filepath.Join(root, clean) + if !strings.HasPrefix(joined, filepath.Clean(root)+string(os.PathSeparator)) { + return "", fmt.Errorf("the bundle holds an entry that would be written outside %s: %s", root, rel) + } + return joined, nil +} + +// ---------- bundle io ---------- + +func writeBundle(target string, m manifest, dumpFile string, blobs []blobEntry, keys map[string]string) error { + // 0600 from creation, not chmod after: the file holds every secret the + // instance has, and a window in which it is world-readable is a window. + f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) //nolint:gosec // operator-supplied output path + if err != nil { + return err + } + defer f.Close() + + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + + if err := tarBytes(tw, manifestPath, mustJSON(m), 0o600, m.CreatedAt); err != nil { + return err + } + if len(keys) > 0 { + if err := tarBytes(tw, keysPath, []byte(renderKeys(keys)), 0o600, m.CreatedAt); err != nil { + return err + } + } + if err := tarFile(tw, databasePath, dumpFile, 0o600, m.CreatedAt); err != nil { + return err + } + for _, b := range blobs { + if err := tarFile(tw, blobsPrefix+b.rel, b.abs, b.mode, m.CreatedAt); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + return f.Close() +} + +func tarBytes(tw *tar.Writer, name string, body []byte, mode os.FileMode, when time.Time) error { + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: int64(mode), Size: int64(len(body)), ModTime: when, Typeflag: tar.TypeReg, + }); err != nil { + return err + } + _, err := tw.Write(body) + return err +} + +func tarFile(tw *tar.Writer, name, path string, mode os.FileMode, when time.Time) error { + st, err := os.Stat(path) + if err != nil { + return err + } + f, err := os.Open(path) //nolint:gosec // paths come from the instance's own blob root + if err != nil { + return err + } + defer f.Close() + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: int64(mode), Size: st.Size(), ModTime: when, Typeflag: tar.TypeReg, + }); err != nil { + return err + } + _, err = io.Copy(tw, f) + return err +} + +func openBundle(path string) (*os.File, *gzip.Reader, *tar.Reader, error) { + f, err := os.Open(path) //nolint:gosec // operator-supplied bundle path + if err != nil { + return nil, nil, nil, fmt.Errorf("could not read %s: %w", path, err) + } + gz, gerr := gzip.NewReader(bufio.NewReaderSize(f, 1<<20)) + if gerr != nil { + _ = f.Close() + return nil, nil, nil, fmt.Errorf("%s is not a Warmbly bundle (it is not gzip): %w", path, gerr) + } + return f, gz, tar.NewReader(gz), nil +} + +// openBundleEntry streams one entry out of the bundle. The caller closes. +func openBundleEntry(path, want string) (io.Reader, func(), error) { + f, gz, tr, err := openBundle(path) + if err != nil { + return nil, func() {}, err + } + closeFn := func() { + _ = gz.Close() + _ = f.Close() + } + for { + hdr, nerr := tr.Next() + if nerr == io.EOF { + closeFn() + return nil, func() {}, fmt.Errorf("%s holds no %s; it is not a Warmbly bundle", path, want) + } + if nerr != nil { + closeFn() + return nil, func() {}, nerr + } + if hdr.Name == want { + return tr, closeFn, nil + } + } +} + +func readBundleEntry(path, want string, limit int64) ([]byte, error) { + r, closeFn, err := openBundleEntry(path, want) + if err != nil { + return nil, err + } + defer closeFn() + return io.ReadAll(io.LimitReader(r, limit)) +} + +func readManifest(path string) (manifest, error) { + var m manifest + raw, err := readBundleEntry(path, manifestPath, 1<<20) + if err != nil { + return m, err + } + if err := json.Unmarshal(raw, &m); err != nil { + return m, fmt.Errorf("%s holds an unreadable manifest: %w", path, err) + } + return m, nil +} + +func readKeys(path string) (map[string]string, error) { + raw, err := readBundleEntry(path, keysPath, 1<<20) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + out[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + return out, nil +} + +func renderKeys(keys map[string]string) string { + names := make([]string, 0, len(keys)) + for k := range keys { + names = append(names, k) + } + sort.Strings(names) + b := strings.Builder{} + b.WriteString("# Warmbly instance keys. These belong in the destination's .env BEFORE the\n") + b.WriteString("# restore runs; warmblyctl restore refuses to run without them and says so.\n") + for _, k := range names { + b.WriteString(k + "=" + keys[k] + "\n") + } + return b.String() +} + +// ---------- small helpers ---------- + +func missingUnrecoverable(keys map[string]string) []string { + var out []string + for _, k := range unrecoverableKeys { + if keys[k] == "" { + out = append(out, k) + } + } + return out +} + +func mustJSON(v any) []byte { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return []byte("{}") + } + return append(b, '\n') +} + +// confirmPhrase reads one line and reports whether it is the phrase. It needs +// a TTY; piped into a container without one, --yes is the documented path. +func confirmPhrase(prompt, phrase string) (bool, error) { + fmt.Print(prompt) + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil && line == "" { + return false, errors.New("could not read the confirmation. Pass --yes to skip it, or run this without `exec -T`.") + } + return strings.TrimSpace(line) == phrase, nil +} + +func plural(n int, one, many string) string { + if n == 1 { + return one + } + return many +} diff --git a/cmd/warmblyctl/main.go b/cmd/warmblyctl/main.go index 98e5d580..5a06279f 100644 --- a/cmd/warmblyctl/main.go +++ b/cmd/warmblyctl/main.go @@ -69,6 +69,10 @@ func dispatch(ctx context.Context, args []string) error { return runUser(ctx, args[1:]) case "org": return runOrg(ctx, args[1:]) + case "backup": + return runBackup(ctx, args[1:]) + case "restore": + return runRestore(ctx, args[1:]) case "api": return runAPI(ctx, args[1:]) } @@ -99,6 +103,8 @@ var commands = []command{ {"user revoke-admin", "Take platform admin away from an account", composeExec + "user revoke-admin --email old@example.com"}, {"user disable-2fa", "Clear an account's TOTP enrollment after a lost authenticator", composeExec + "user disable-2fa --email you@example.com"}, {"hash-password", "Print an argon2 hash for WARMBLY_BOOTSTRAP_PASSWORD_HASH", "printf '%s' 'your-password' | docker compose -p warmbly exec -T backend warmblyctl hash-password"}, + {"backup", "Write the whole instance (database, blobs, keys) to one restorable bundle", composeExec + "backup --out /data/blobs/warmbly-backup.tar.gz"}, + {"restore", "Restore a bundle onto this instance, replacing everything on it", composeExec + "restore --file /data/blobs/warmbly-backup.tar.gz"}, {"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"}, {"org export", "Write a whole workspace to a portable archive file", composeExec + "org export --org you@example.com --out /tmp/workspace.warmbly.zip"}, {"org import", "Apply an archive to a workspace on this instance", composeExec + "org import --org you@example.com --file /tmp/workspace.warmbly.zip"}, @@ -146,6 +152,10 @@ Environment: WARMBLY_API_URL API base URL for the API commands. Defaults to this instance's API_PUBLIC_URL, then the hosted service. + backup and restore read the same crypto settings as org export/import, plus + BLOB_FS_ROOT for the blob tree. Run them inside the backend container, which + is also where pg_dump and psql live. + org export and org import additionally read the instance's crypto settings, because a workspace's sealed values have to be opened on the way out and re-sealed on the way in: KMS_PROVIDER (with KMS_LOCAL_MASTER_KEY or the AWS diff --git a/deploy/config/env.example b/deploy/config/env.example index 3625b88f..8093454f 100644 --- a/deploy/config/env.example +++ b/deploy/config/env.example @@ -31,6 +31,32 @@ INTERNAL_API_TOKEN=change-me-internal-token PRIMARY_DB=postgres://warmbly:warmbly@localhost:5432/warmbly_dev?sslmode=disable REDIS=redis://localhost:6379 +# === Release and storage layout (docker-compose.yml only) === +# Both keys exist on every app service, so this one compose file either pulls +# the published images or builds this checkout. Pin a release and every later +# `docker compose up` is reproducible; `install.sh` writes these for you. +# WARMBLY_TAG=v1.4.2 # default: prod (a moving tag) +# WARMBLY_IMAGE_PREFIX=ghcr.io/warmbly/warmbly +# +# One variable per store. Compose reads a source starting with / as a bind +# mount and anything else as a named volume, so a path here is the difference +# between "rsync this directory" and "ask docker volume inspect where it went". +# WARMBLY_PG_DATA=/mnt/data/warmbly/postgres +# WARMBLY_BLOBS=/mnt/data/warmbly/blobs +# WARMBLY_NATS_DATA=/mnt/data/warmbly/nats +# WARMBLY_REDIS_DATA=/mnt/data/warmbly/redis +# WARMBLY_WORKER_STATE=/mnt/data/warmbly/worker +# WARMBLY_UPDATER_STATE=/mnt/data/warmbly/updater + +# === Database-backed settings, seeded on first boot === +# The sync budgets and retention windows live in the database and are edited in +# the admin panel under Instance > Instance settings. This seeds them on an +# instance that has never saved that document, so an unattended install can +# ship its data-control answers with the rest of the environment. It is a no-op +# from the first save onwards, so leaving it here never undoes an edit made +# there. https://docs.warmbly.com/development/data-control/ +# WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"engagement_event_days":30,"form_event_days":30,"audit_log_days":30}} + # === Provider switches (no-cloud defaults) === AWS_CONFIG_ENABLED=false # true => read secrets from AWS SSM/Secrets Manager diff --git a/deploy/docker/backend.Dockerfile b/deploy/docker/backend.Dockerfile index f46036d6..edc4bbc5 100644 --- a/deploy/docker/backend.Dockerfile +++ b/deploy/docker/backend.Dockerfile @@ -38,7 +38,10 @@ RUN --mount=type=cache,target=/go/pkg/mod \ FROM alpine:3.23 ARG GO_TAGS="" -RUN apk add --no-cache ca-certificates tzdata && \ +# postgresql-client is here for `warmblyctl backup` and `warmblyctl restore`: +# the instance bundle is a pg_dump and the restore replays it with psql, and the +# backend container is where the CLI already has PRIMARY_DB and the blob root. +RUN apk add --no-cache ca-certificates tzdata postgresql-client && \ if echo "$GO_TAGS" | grep -qw kafka; then apk add --no-cache librdkafka; fi && \ adduser -D -u 1000 warmbly diff --git a/docker-compose.yml b/docker-compose.yml index a393f301..f678ae1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,19 @@ # # To run on Kafka + S3 instead, set the provider vars above and rebuild the Go # images with --build-arg GO_TAGS=kafka (tracking: --build-arg CARGO_FEATURES=kafka). +# +# EVERY APP SERVICE CARRIES BOTH image: AND build:, so this one file serves both +# installs. `docker compose pull && docker compose up -d` runs the published +# release images and compiles nothing; `docker compose up --build` still builds +# from this checkout. Pin a release with WARMBLY_TAG=v1.4.2 in .env, or point +# WARMBLY_IMAGE_PREFIX at your own registry. +# +# EVERY STORE'S VOLUME SOURCE IS A VARIABLE. Compose reads a source starting +# with / or . as a bind mount and anything else as a named volume, so +# WARMBLY_PG_DATA=/mnt/ssd/warmbly/postgres puts the database on a path you can +# rsync and snapshot, while the default keeps the named volume. The full set is +# WARMBLY_PG_DATA, WARMBLY_REDIS_DATA, WARMBLY_NATS_DATA, WARMBLY_BLOBS, +# WARMBLY_WORKER_STATE and WARMBLY_UPDATER_STATE. # Everything below is overridable from a single .env next to this file. The # defaults are the no-cloud stack, so an empty .env still boots; set only what @@ -216,7 +229,7 @@ services: POSTGRES_DB: warmbly_dev ports: ["15432:5432"] volumes: - - postgres_data:/var/lib/postgresql/data + - ${WARMBLY_PG_DATA:-postgres_data}:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U warmbly"] interval: 5s @@ -228,7 +241,7 @@ services: image: redis:7-alpine ports: ["16379:6379"] volumes: - - redis_data:/data + - ${WARMBLY_REDIS_DATA:-redis_data}:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -242,7 +255,7 @@ services: command: ["-js", "-sd", "/data", "-m", "8222"] ports: ["4222:4222", "8222:8222"] volumes: - - nats_data:/data + - ${WARMBLY_NATS_DATA:-nats_data}:/data healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8222/healthz"] interval: 5s @@ -295,6 +308,7 @@ services: backend: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/backend:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/backend.Dockerfile @@ -358,7 +372,7 @@ services: UPDATER_URL: ${UPDATER_URL:-http://updater:8095} UPDATER_TOKEN: ${UPDATER_TOKEN:-${INTERNAL_API_TOKEN:-local-dev-internal-token}} volumes: - - blobs:/data/blobs + - ${WARMBLY_BLOBS:-blobs}:/data/blobs depends_on: postgres: { condition: service_healthy } redis: { condition: service_healthy } @@ -374,6 +388,7 @@ services: # database; the backend's internal API is its only dependency. forms: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/forms:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/forms.Dockerfile @@ -404,6 +419,7 @@ services: consumer: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/consumer:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/consumer.Dockerfile @@ -418,7 +434,7 @@ services: # country and city on the contact's timeline. GEODB_PATH: ${GEODB_PATH:-/app/data/GeoLite2-City.mmdb} volumes: - - blobs:/data/blobs + - ${WARMBLY_BLOBS:-blobs}:/data/blobs depends_on: backend: { condition: service_healthy } nats: { condition: service_healthy } @@ -428,6 +444,7 @@ services: # belong to the mail provider, so more workers = more parallelism, not more IPs. worker: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/worker:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/worker.Dockerfile @@ -455,14 +472,15 @@ services: BOX_OUTLOOK_CLIENT_ID: ${BOX_OUTLOOK_CLIENT_ID:-} BOX_OUTLOOK_CLIENT_SECRET: ${BOX_OUTLOOK_CLIENT_SECRET:-} volumes: - - blobs:/data/blobs - - worker_state:/data/state + - ${WARMBLY_BLOBS:-blobs}:/data/blobs + - ${WARMBLY_WORKER_STATE:-worker_state}:/data/state depends_on: backend: { condition: service_healthy } nats: { condition: service_healthy } tracking: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/tracking:${WARMBLY_TAG:-prod} build: context: ./tracking dockerfile: Dockerfile @@ -496,6 +514,7 @@ services: realtime: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/realtime:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/realtime.Dockerfile @@ -530,6 +549,7 @@ services: # use `make app` / docker-compose.dev.yml instead. web: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/web:${WARMBLY_TAG:-prod} build: context: ./web dockerfile: Dockerfile @@ -546,6 +566,7 @@ services: admin: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/admin:${WARMBLY_TAG:-prod} build: context: ./admin dockerfile: Dockerfile @@ -567,6 +588,7 @@ services: # to update by hand with `git pull && make up`. updater: restart: unless-stopped + image: ${WARMBLY_IMAGE_PREFIX:-ghcr.io/warmbly/warmbly}/updater:${WARMBLY_TAG:-prod} build: context: . dockerfile: deploy/docker/updater.Dockerfile @@ -588,12 +610,17 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ${WARMBLY_REPO_DIR:-${PWD}}:${WARMBLY_REPO_DIR:-${PWD}} - - updater_state:/var/lib/warmbly-updater + - ${WARMBLY_UPDATER_STATE:-updater_state}:/var/lib/warmbly-updater profiles: ["updater"] # ─── one-shots ──────────────────────────────────────────────────────── # Optional demo seed. Run explicitly: docker compose --profile seed run --rm seed + # Build-only on purpose, unlike every service above. The seed is not a + # published image (it plants a known super-admin credential, so it must never + # be one `docker compose pull` away on a real host), and sharing the backend's + # image tag would mean `make seed-demo` retagged the image the running backend + # was created from. seed: build: context: . @@ -613,6 +640,9 @@ services: backend: { condition: service_healthy } profiles: ["seed"] +# Declared for the defaults. A store pointed at a host path by its +# WARMBLY_*_DATA variable simply stops using the named volume below; an unused +# declaration costs nothing and keeps switching back a one-line .env edit. volumes: postgres_data: redis_data: diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index aa5de654..ff51a109 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -471,11 +471,24 @@ These are the only settings a browser can change, and no environment variable ow | `sync.backfill_messages` | integer, 1 to 100000 | `5000` | The most messages that import stores per mailbox | | `sync.daily_messages_per_mailbox` | integer, 1 to 100000 | `2000` | New (live) messages one mailbox may store per UTC day. Over it, mail waits for the next day; replies to the mailbox's own sends have a separate budget of the same size | | `sync.daily_messages_per_org` | integer, 1 to 2000000 | `25000` | New plus imported messages one organization may store per UTC day | +| `retention.engagement_event_days` | integer, 1 to 3650 | `365` | How long the per-event open and click logs (client, device, approximate location) are kept. Campaign progress keeps its own summary that outlives them, so counts, filters and branching never change | +| `retention.form_event_days` | integer, 1 to 3650 | `180` | How long form funnel events (views, starts, field-level drop-off) are kept. Funnel reports range up to 90 days, so anything shorter shortens the report too | +| `retention.audit_log_days` | integer, 1 to 3650 | `90` | How long the audit trail is kept. It carries IP addresses, user agents and change payloads, so this is also how long that data is held | | `deliverability.enforce_domain_auth` | boolean | `true` | Whether a sending domain that fails SPF or DMARC stops cold campaign sending and warmup sending from every mailbox on it. Off keeps the check running and still shows the state and the Advisor card, it just never blocks | | `deliverability.auth_grace_hours` | integer, 1 to 720 | `72` | How long a domain must stay failing before the gate applies. The clock starts when the background check first sees the failure, so this is also how much warning the owner gets | The four `sync.*` values are read by the backend when a mailbox is loaded onto a worker (on connect, on reassignment, and by the reconciler's periodic republish), so a change reaches every mailbox within a few minutes without a restart. The fixed pacing numbers around them (burst per five minutes, hourly, backfill pace, the flood threshold and the chronic-overage rule) are compiled constants listed under **Instance > Limits**; see [Mailboxes](/guides/mailboxes/#what-gets-synced) for how the budgets behave. +The three `retention.*` values are read by the pruning sweeps on every pass, so shortening one takes effect on the next sweep rather than at the next restart. Deletion is permanent and there is no grace period: what already sits outside a shortened window goes on that sweep. See [data control](/development/data-control/#what-is-kept-and-for-how-long) for what each log holds and what a shorter window costs. + +An unattended install can seed the whole document before anyone signs in, with `WARMBLY_SETTINGS_BOOTSTRAP` holding the same partial JSON the admin API takes: + +```bash +WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"audit_log_days":30}} +``` + +It is applied only while the settings row has never been written, so from the first save in the panel onwards the panel is authoritative and the variable is a no-op. That is what [`install.sh --wizard`](/development/install/) writes when you answer its retention questions. + The two `deliverability.*` values are read on every scheduling pass and every warmup send, so turning the gate off releases blocked mailboxes within the 30 second cache window. Turning it back on does not stop anything retroactively: a domain still has to spend its whole grace window failing first. Only a sustained failure gates, so a domain reading `unknown` (never checked, DNS could not answer, or a special-use domain that cannot resolve) always sends. Manual sends and unibox replies are never gated; see [domain authentication](/guides/deliverability/#domain-authentication). Changing them is audited, and every value is validated and clamped server side on write as well as on read, so a row written by an older version still resolves. @@ -491,6 +504,8 @@ Changing them is audited, and every value is validated and clamped server side o ## See also +- [Install](/development/install/) for the one-command install that writes all of this for you +- [Data control](/development/data-control/) for where each store lives and what each retention window governs - [First run](/development/first-run/) for claiming a fresh instance - [Accounts and access](/development/accounts-and-access/) for who may sign in and how to invite people - [Instance health](/development/instance-health/) for the checks that read these values back diff --git a/docs/content/docs/development/data-control.mdx b/docs/content/docs/development/data-control.mdx new file mode 100644 index 00000000..f5c2c3b2 --- /dev/null +++ b/docs/content/docs/development/data-control.mdx @@ -0,0 +1,211 @@ +--- +title: Data control +description: Where every store lives on disk, what each retention window governs, how backups work, and how to move an instance to another host. +--- + +Self-hosting Warmbly means holding mailbox credentials, message bodies and contact records on your own disk. This page is the whole answer to where that data is, how long it stays, and how you get it off this machine. + +Three questions, and they have different answers: + +| Question | Answer | +|---|---| +| Where is it? | [Six stores](#where-the-data-sits), each pointed at by one variable | +| How long does it stay? | [Retention windows](#what-is-kept-and-for-how-long), all editable in the admin panel | +| How do I move it? | [`warmblyctl backup`](#backups) for the instance, [workspace export](/guides/workspace-export-import/) for one workspace | + +## Where the data sits + +Every store's location is one variable in `.env`. Compose reads a source starting with `/` as a bind mount and anything else as a named volume, so the same variable covers both and there is no second compose file. + +| Variable | Holds | Default | +|---|---|---| +| `WARMBLY_PG_DATA` | Postgres: organizations, users, mailboxes (credentials sealed), contacts, campaigns, the audit trail | `/postgres` | +| `WARMBLY_BLOBS` | Message bodies, attachments, avatars and logos | `/blobs` | +| `WARMBLY_NATS_DATA` | The event bus's JetStream state | `/nats` | +| `WARMBLY_REDIS_DATA` | Cache and rate-limit counters. Disposable | `/redis` | +| `WARMBLY_WORKER_STATE` | A worker's own id and sync cursors. Disposable | `/worker` | +| `WARMBLY_UPDATER_STATE` | The last update job's log. Disposable | `/updater` | + +An [install from `install.sh`](/development/install/) points all six under one data root, so `/opt/warmbly/data` is the whole of it and you can `rsync` that path. A clone-and-build install keeps Docker named volumes unless you set the same variables. + +Only the first two carry anything you cannot rebuild. The other four are state a fresh container reconstructs, which is why a backup does not include them. + +### Encryption + +Two keys, and they are not interchangeable. + +| Key | Opens | Read by | +|---|---|---| +| `CREDENTIALS_ENCRYPTION_KEY` | Mailbox SMTP and IMAP credentials | Backend and workers, without an organization context | +| `KMS_LOCAL_MASTER_KEY` | The per-organization data keys, which in turn open everything else | Backend and consumer | + +Both are unrecoverable. A database backup without them restores an instance whose mailboxes authenticate against nothing, and there is no way back from that. They are in `.env`, and `install.sh` also writes them to `keys-backup.txt` next to the install, which is on the same disk as the database and therefore not a backup either. Copy them somewhere else. + +### External stores + +Postgres, Redis and blob storage each accept an external target, set at install time or by editing `.env`: + +```bash +PRIMARY_DB=postgres://user:pass@db.internal:5432/warmbly?sslmode=require +REDIS=redis://cache.internal:6379 +BLOB_PROVIDER=s3 +BLOB_BUCKET=warmbly +AWS_ENDPOINT_URL_S3=https://.r2.cloudflarestorage.com +``` + + +A remote worker writes bodies to its own disk, so the dashboard on the control-plane host finds nothing. Any deployment with workers on more than one machine needs S3-compatible blob storage. This is the one storage choice that is not just a preference. + + +## What is kept, and for how long + +Every window below lives in the database, not the environment, and is edited under **Instance > Instance settings** in the admin panel. A sweep runs a few times a day and reads the current value on every pass, so a change takes effect without a restart. + +### Mailbox import and sync + +| Setting | Default | Range | What it governs | +|---|---|---|---| +| `sync.backfill_days` | 90 | 1 to 730 | How far back the initial import reaches when a mailbox is connected, newest first | +| `sync.backfill_messages` | 5,000 | 1 to 100,000 | The most messages that import stores per mailbox | +| `sync.daily_messages_per_mailbox` | 2,000 | 1 to 100,000 | New mail one mailbox may store per UTC day | +| `sync.daily_messages_per_org` | 25,000 | 1 to 2,000,000 | New plus imported mail across one workspace per UTC day | + +Mail over a daily budget is deferred, never dropped: the provider cursor is held and the mail is re-offered on the next pass. Replies to the mailbox's own outreach ride a separate budget of the same size and keep landing regardless. + +### Event history + +| Setting | Default | What it holds | +|---|---|---| +| `retention.engagement_event_days` | 365 | Per-event open and click logs: client, device, approximate location | +| `retention.form_event_days` | 180 | Form funnel events: views, starts, field-level drop-off | +| `retention.audit_log_days` | 90 | The audit trail: actor, IP address, user agent, change payload | + +Each is between 1 and 3,650 days. These are the three settings a retention or privacy policy applies to, because each window is also how long the personal data in that log is held. + +None of them change a number anyone reads. Campaign progress keeps its own summary of opens and clicks that outlives the per-event log, so counts, filters and branching are unaffected by shortening any of these. What gets shorter is what a contact's timeline can show, how far a funnel report reaches, and how far back an admin can audit. + + +There is no grace period and no copy. Take a backup first if you are not sure. + + +The **minimal retention** preset in the admin panel and in the installer sets all three to 30 days. + +### Set them at install time + +The installer writes the answers into `.env` as one document, applied on the first boot of a fresh database: + +```bash +WARMBLY_SETTINGS_BOOTSTRAP={"sync":{"backfill_days":30},"retention":{"audit_log_days":30}} +``` + +It is read only while the settings row has never been written. From the first save in the admin panel onwards the panel is authoritative, so leaving the line in `.env` never undoes a later edit. + +## Backups + +`warmblyctl backup` writes one bundle holding the three things that only restore together: + +- the database, as a `pg_dump` +- the blob root, when blobs are on the filesystem +- the encryption keys, unless you pass `--no-keys` + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz +``` + +The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. Treat the file as you would the instance itself. + +An install from `install.sh --wizard` can schedule this for you: `backup.sh` next to the install, a systemd timer, a retention count, and an optional `aws s3 cp` to somewhere off the host. A backup that only exists on the machine it backs up is not one. + +### Restore + +On the destination host, with the same keys in place: + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + +The restore empties the schema and replays the dump, so it replaces everything currently on that instance and asks you to type `restore` first. + +Before it does anything it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` against the destination's and refuses to continue when they differ, printing the two lines to put in `.env`. That check is the point of the command: without it a restore looks like it worked and every mailbox fails to authenticate days later, with no error that names the cause. + +## Moving an instance + +Two ways, and they answer different questions. + +### The whole instance, to a new host + +Every workspace, every user, the platform admins, the API keys. + + + + + +### Install Warmbly on the new host + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --host +``` + + + + + +### Put the old keys in the new `.env` + +Copy `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` from the old install, then recreate the containers so they take: + +```bash +docker compose -p warmbly up -d +``` + + + + + +### Restore the bundle + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + + + + + +### Check it + +```bash +docker compose -p warmbly exec backend warmblyctl status +``` + +Mailboxes should be connected, not needing a reconnect. If they need one, the keys did not match. + + + + + +The `rsync` alternative works too and is sometimes simpler: stop the stack, copy the data root and the `.env` to the new host, start it there. It moves the same bytes; the bundle exists because it is the version that survives a different host layout, a different Postgres, and a partial copy. + +### One workspace, to another instance + +[Workspace export and import](/guides/workspace-export-import/) moves a single organization's data between two running instances, re-sealing its secrets for the destination's keys. That is the per-customer tool; the bundle here is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance. + +## Outbound calls + +A self-hosted Warmbly makes no outbound call of its own except one, and it is off with a single setting. + +| Call | When | Turn it off | +|---|---|---| +| GitHub releases API | Every 30 minutes, to tell the admin panel a newer version exists | `UPDATE_CHECK_ENABLED=false` | + +Everything else is you: mail through the mailboxes you connect, DNS lookups for the domains you check, and whatever integrations you configure. There is no telemetry, no phone-home, and no license check. + +## See also + +- [Install](/development/install/): the wizard that asks all of this up front +- [Configuration](/development/configuration/): every variable and every database-backed setting +- [warmblyctl](/development/warmblyctl/): `backup`, `restore`, and the operator commands +- [Workspace export and import](/guides/workspace-export-import/): the per-workspace story diff --git a/docs/content/docs/development/deployment-guide.mdx b/docs/content/docs/development/deployment-guide.mdx index 7609afbe..7c058cb5 100644 --- a/docs/content/docs/development/deployment-guide.mdx +++ b/docs/content/docs/development/deployment-guide.mdx @@ -5,6 +5,15 @@ description: A step-by-step guide to running the whole Warmbly platform yourself Warmbly self-hosts with **no cloud account of any kind**: no AWS, no GCP, no Stripe, no Kafka. One `docker-compose.yml` runs everything on local, open-source pieces, and every external dependency is an environment variable you can flip later. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` +That pulls the published release images and skips everything on this page: no clone, no compiler, under two minutes. Add `--wizard` and it asks where your data lives, what is kept and for how long, and how it is backed up. See [Install](/development/install/). + +This page is the build-from-source path, and the reference for every setting either path writes. Take it when you want to run modified code, a platform we publish no image for, or a build you produced yourself. + + Three commands give you a working install. Everything after them is optional, and nothing below is needed before you have seen it running. /warmbly/`, which works on any | Trigger | Images | Tags | |---------|--------|------| -| Push to `main` | backend, consumer, worker, tracking, realtime | `:`, `:dev` | +| Push to `main` | backend, consumer, worker, forms, updater, tracking, realtime | `:`, `:dev` | | Tag `vX.Y.Z` | all of the above plus web and admin | `:vX.Y.Z`, `:vX.Y`, `:vX`, `:prod` | +Every app service in `docker-compose.yml` carries both an `image:` and a `build:` key, so the same file serves both paths. `docker compose pull && docker compose up -d` runs the published images and compiles nothing; `docker compose up --build` still builds this checkout. Pin a release with `WARMBLY_TAG=v1.4.2` in `.env`, or point `WARMBLY_IMAGE_PREFIX` at your own registry. + To move a worker fleet onto a new release, update `WORKER_IMAGE` and use "Pull latest and restart" on each worker. Control-plane migrations are forward-only, so prefer rolling forward over rolling back. ## Upgrading and backups @@ -759,7 +770,16 @@ Upgrading is safe with data in place: migrations are forward-only and apply on b ### Backing up -Back up three things. All three, or the backup is not a backup: +One command writes all three of the things that only restore together: + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz +``` + +That bundle holds the database, the blob root and the encryption keys, and `warmblyctl restore` puts it back on another host, refusing to run when that host's keys are not the ones the bundle was sealed with. [Data control](/development/data-control/#backups) covers scheduling it and moving an instance. + +By hand, if you would rather assemble it yourself, it is these three and no fewer: ```bash docker compose -p warmbly exec -T postgres pg_dump -U warmbly warmbly_dev > backup.sql diff --git a/docs/content/docs/development/first-run.mdx b/docs/content/docs/development/first-run.mdx index 23a02060..241ad585 100644 --- a/docs/content/docs/development/first-run.mdx +++ b/docs/content/docs/development/first-run.mdx @@ -7,12 +7,18 @@ A fresh instance has no accounts and is described as **unclaimed**. Claiming it ## Claim the instance +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +The [installer](/development/install/) starts the stack and prints the claim link when it finishes. From a clone it is the same two steps and a build: + ```bash git clone https://github.com/warmbly/warmbly && cd warmbly make up ``` -`make up` builds the images, starts the stack, waits for the backend to answer, then runs `make claim`, which prints a single-use link and ends on the command that reissues it: +Either way you end on `make claim` (or `warmblyctl setup-link`), which prints a single-use link and ends on the command that reissues it: ``` No accounts exist yet. Open this link to claim the instance and diff --git a/docs/content/docs/development/install.mdx b/docs/content/docs/development/install.mdx new file mode 100644 index 00000000..dc1305f4 --- /dev/null +++ b/docs/content/docs/development/install.mdx @@ -0,0 +1,318 @@ +--- +title: Install +description: One command installs Warmbly from published release images, with a wizard that asks where your data lives, what is kept and for how long, and how it is backed up. +--- + +```bash +curl -fsSL https://warmbly.com/install.sh | sh +``` + +No clone, no compiler, no toolchain. The script checks Docker, creates one directory, generates every secret, pulls the pinned release images, starts the stack and prints the link that claims your instance. On a clean box with Docker already installed it finishes in under two minutes. + +Add `--wizard` and it asks the questions a config reference cannot ask for you. + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --wizard +``` + +## What it installs + +| | | +|---|---| +| Where | `/opt/warmbly`, or wherever you point `--dir` | +| What | `docker-compose.yml`, a `.env` at 0600, `keys-backup.txt`, and the data root | +| Version | The newest release, resolved once and pinned in `.env`. Never `latest` | +| Images | `ghcr.io/warmbly/warmbly/*`, multi-arch for amd64 and arm64 | +| Services | Backend, consumer, worker, dashboard, admin, Postgres, Redis, NATS, and (unless you say core only) tracking, realtime and forms | + +Nothing is built on your machine. The clone-and-build path still exists and is still supported; see [self-hosting](/development/deployment-guide/) for when you want it. + +## Before you run it + +Piping a script into a shell is a fair thing to be uneasy about, and the objection is only answerable if the script is versioned, inspectable and checksummed. All three are: + +```bash +curl -fsSLO https://warmbly.com/install.sh +curl -fsSLO https://warmbly.com/install.sh.sha256 +sha256sum -c install.sh.sha256 +less install.sh +sh install.sh +``` + +The script itself is written to deserve that: `set -eu`, every line inside a function, and `main "$@"` on the very last line, so a download that is cut off half way executes nothing at all. It never asks to be piped into `sudo`; it asks for elevation only for the specific steps that need it, and names them. + +You also do not have to run it to see what it would do: + +```bash +sh install.sh --dry-run +``` + +That prints the exact `.env` and compose file it would write, and touches nothing. + +### And after it runs + +Every release publishes an `images.json` asset listing the manifest digest of each image in it. Once the pull finishes the installer compares what actually landed on the machine against that file, and stops before starting anything if they differ. You can check the same thing yourself at any time: + +```bash +docker image inspect ghcr.io/warmbly/warmbly/backend:v1.4.2 --format '{{index .RepoDigests 0}}' +``` + +An install pointed at your own registry with `--registry` has no manifest to check against, and the installer says so rather than pretending it verified something. + +## Requirements + +| You need | Notes | +|---|---| +| Linux or macOS, amd64 or arm64 | On Windows, run it inside WSL | +| Docker 20.10+ with Compose v2 | The script offers to install Docker on Linux, and never does so quietly | +| ~2 GB disk | The images, not a build cache. There is no build | +| 4 GB RAM | The running stack idles near 300 MB | +| Free ports | 8080, 5173, 5174, and 3000, 4000, 8090 with every component | + +Port `3000` is the one that actually collides: it is the default for a lot of other self-hosted software. The installer checks every port before it writes anything and tells you which one is taken. + +## The wizard + +`--wizard` walks nine steps. Every answer has a default that matches the fast path, so pressing enter through the whole thing produces exactly the install the one-liner produces. + +Every menu takes the keys you already use: + +| Keys | Does | +|---|---| +| `↑` `↓`, `j` `k`, `ctrl-n` `ctrl-p` | Move | +| `ctrl-d` `ctrl-u` | Half a screen | +| `gg`, `G` | First, last | +| `1` to `9` | Jump straight to that option | +| `enter`, `space`, `l` | Take it | +| `q`, `esc`, `ctrl-c` | Leave without installing | + +The review screen at the end takes `enter` to install, `e` to go back into a section, and `q` to quit. + +Steps append rather than replacing each other, so the whole run stays scrollable and nothing above the installer is touched. `--clear` redraws the screen at each step instead, if you prefer that. + + + + + +### Where it lives + +One directory holds the compose file, the `.env`, and, unless you move it in step three, every store this instance writes to. Default `/opt/warmbly`. + + + + + +### How it is reached + +The hostname every link is built from: the dashboard URL, the unsubscribe link in campaign mail, the tracking pixel. Then how it is served: + +| Choice | What it does | +|---|---| +| **Plain HTTP** | Ports published as they are. Correct for localhost or a private network | +| **Bundled Caddy** | Adds a Caddy service with automatic HTTPS, one hostname per surface, and binds everything else to loopback | +| **Behind your proxy** | Binds every service to `127.0.0.1` and sets `TRUSTED_PROXIES`, which is silently wrong on every proxied install that was configured by hand | + +`TRUSTED_PROXIES` matters more than it looks. Without it every request appears to come from the proxy, so per-IP rate limits and click deduplication stop distinguishing between people. + +The bundled Caddy option serves six hostnames under the domain you give it, and the installer prints the DNS records at the end: + +``` +app.example.com admin.example.com api.example.com +ws.example.com track.example.com forms.example.com +``` + + + + + +### Where the data sits + +The question the rest of the install cannot answer for you, and the reason the wizard exists. + +| Choice | What you get | +|---|---| +| **Under the install directory** | `/opt/warmbly/data` holds Postgres, blobs, NATS and worker state | +| **A path you choose** | An external disk or a mount you already snapshot | +| **Docker named volumes** | Managed by Docker, portable only through `docker volume` | + +A real path is the default, because it is the choice that makes backup and migration obvious: moving the instance to another host is `rsync` of that path plus the `.env`. Compose reads a volume source starting with `/` as a bind mount and anything else as a named volume, so this is one variable per store and no second compose file. + +Then, behind one prompt: an external Postgres or Redis (the bundled container is left out entirely when you bring your own), and where message bodies, attachments and avatars go. + + +A worker running on another host writes to its own disk. Once you scale workers out, blob storage has to be S3-compatible or bodies go missing. The wizard offers MinIO, R2, B2 and AWS. + + + + + + +### Keys and secrets + +Five secrets are generated: `AUTH_SECRET`, `INTERNAL_API_TOKEN`, `SECRET_KEY_BASE`, `KMS_LOCAL_MASTER_KEY` and `CREDENTIALS_ENCRYPTION_KEY`. They are written 0600, `APP_ENV` is set to `prod`, and no published default is ever reachable through this path. + +The last two are unrecoverable. Every mailbox credential and every per-organization key is sealed with them, so a database backup that does not travel with them restores an instance whose mailboxes authenticate against nothing. The wizard prints both, writes them to `keys-backup.txt` next to the install, and waits until you type that you have a copy somewhere else. + + + + + +### What is kept, and for how long + +Two groups, both editable afterwards in **Instance > Instance settings**: + +| Setting | Default | What it governs | +|---|---|---| +| Import window | 90 days | How far back the initial import reaches when a mailbox is connected | +| Import cap | 5,000 | The most messages that import stores per mailbox | +| Daily per mailbox | 2,000 | New mail one mailbox may store per UTC day | +| Daily per organization | 25,000 | New plus imported mail across one workspace per UTC day | +| Opens and clicks | 365 days | Per-event logs, with the client, device and approximate location of each | +| Form funnel events | 180 days | Views, starts, field-level drop-off | +| Audit log | 90 days | Who did what, from which IP address and user agent | + +The last three are also how long that personal data is held. A "minimal retention" preset sets every event window to 30 days. + + + + + +### Backups + +A scheduled `warmblyctl backup`: a target directory, a frequency, how many to keep, and whether the encryption keys travel in the bundle. It writes `backup.sh` next to the install and, on a systemd host, a service and a timer. + +Including the keys makes the bundle as sensitive as the instance itself. Leaving them out means keeping `keys-backup.txt` somewhere else, or the bundles cannot be restored at all. + + + + + +### Who gets in + +The first owner, either as a printed claim link or unattended from `WARMBLY_BOOTSTRAP_EMAIL` plus a hash from `warmblyctl hash-password`. Then the registration posture, and platform mail: an SMTP relay, or skip it and have password resets and login codes printed to the backend log. + + + + + +### Footprint + +Everything, or core only (no tracking pixel, no websockets, no hosted forms). Ports are checked for collisions before anything is written. Last, the update check, which is one outbound call to the GitHub releases API and the only outbound call this instance makes on its own. There is no telemetry in Warmbly. + + + + + +### Review + +Every answer on one screen. Enter installs, `e` goes back to any section, `q` leaves without writing anything. + + + + + +## See it first + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --demo +``` + +The real wizard and the real review, with the pull, the container creation and the health wait played rather than run. It writes no file, pulls no image, creates no container and needs no Docker, so it answers "what does this actually do" without a server to try it on. + +It takes about half a minute, paced so the phases are readable; a real install is several minutes, most of it the pull and the first boot's migrations. `WARMBLY_DEMO_FAST=1` collapses it to under ten seconds. From a checkout: `make installer-demo`. + +## For agents + +An agent should never drive the wizard: it reads raw keypresses and needs a +terminal. Every answer is also a flag, so `--yes` with flags is silent, +deterministic and idempotent, and `--print-env` and `--dry-run` show what a run +would produce without producing it. + +The repository ships a `warmbly-install` skill under `skills/` covering exactly +that, plus backups and moving an instance. Install it the way your agent +expects, for example `cp -r skills/warmbly-install ~/.claude/skills/`. + +## Unattended + +Every answer is a flag and a `WARMBLY_*` environment variable, so the same script provisions from Ansible or cloud-init: + +```bash +curl -fsSL https://warmbly.com/install.sh | sh -s -- --yes \ + --dir /opt/warmbly \ + --host warmbly.acme.com \ + --tls caddy \ + --data-root /mnt/data/warmbly \ + --blobs s3 \ + --retention-preset minimal \ + --backup-dir /mnt/backups/warmbly \ + --version v1.4.2 +``` + +| Flag | Variable | Default | +|---|---|---| +| `--dir` | `WARMBLY_DIR` | `/opt/warmbly` | +| `--host` | `WARMBLY_HOST` | `localhost` | +| `--tls` | `WARMBLY_TLS` | `none` (`caddy`, `proxy`) | +| `--data-root` | `WARMBLY_DATA_ROOT` | `/data` (or `volumes`) | +| `--blobs` | `WARMBLY_BLOBS` | `filesystem` (`s3`) | +| `--database-url` | `WARMBLY_DATABASE_URL` | bundled Postgres | +| `--redis-url` | `WARMBLY_REDIS_URL` | bundled Redis | +| `--components` | `WARMBLY_COMPONENTS` | `full` (`core`) | +| `--version` | `WARMBLY_VERSION` | newest release | +| `--channel` | `WARMBLY_CHANNEL` | `stable` (`dev`) | +| `--registry` | `WARMBLY_IMAGE_PREFIX` | `ghcr.io/warmbly/warmbly` | +| `--backup-dir` | `WARMBLY_BACKUP_DIR` | none | +| `--backup-keep` | `WARMBLY_BACKUP_KEEP` | `14` | +| `--retention-preset` | | `default` (`minimal`) | +| `--no-update-check` | `WARMBLY_UPDATE_CHECK` | on | + +`--print-env` emits the `.env` and exits. `--dry-run` prints every file it would write and exits. `--demo` walks the whole thing and installs nothing. + +## Re-running it + +The installer is idempotent. A second run: + +- adopts the existing `.env` and never regenerates a secret, because a new `AUTH_SECRET` signs everyone out and a new `CREDENTIALS_ENCRYPTION_KEY` is permanent data loss +- keeps the existing data root, because pointing an install at a new path is not a move, it is an empty instance next to a full one +- keeps any keys you added to `.env` yourself, in a clearly marked block at the end +- regenerates `docker-compose.yml`, keeping the previous one as `docker-compose.yml.bak` +- refuses a directory it did not create, unless you pass `--force` + +Put your own compose changes in a `docker-compose.override.yml`, which compose merges on top and the installer never touches. + +## Updating + +The version pill in the admin panel, then **Update and restart**. On an install created this way the updater runs in image mode: it writes the new tag into `.env`, pulls, and recreates the containers whose image changed. Nothing is built. See [updates](/development/updates/). + +By hand, from the install directory: + +```bash +docker compose -p warmbly pull && docker compose -p warmbly up -d +``` + +## Removing it + +```bash +sh install.sh --uninstall +``` + +Stops and removes the containers and leaves every byte of data where it is. Adding `--purge-data` deletes the data root and the install directory as well, and asks you to type a phrase first, because that also destroys the only keys that could have opened a backup of it. + +## When it goes wrong + +| Symptom | Cause | +|---|---| +| `Docker is installed but not answering` | The daemon is not running, or your user is not in the `docker` group yet. `sudo systemctl start docker`, then log out and back in | +| `Something already listens on 3000` | Another self-hosted tool. Change the published port in `docker-compose.yml` before starting | +| `Could not pull the release images` | The tag does not exist, or this host cannot reach `ghcr.io`. Check [releases](https://github.com/warmbly/warmbly/releases) | +| The API never answers | The first boot applies every migration. `docker compose -p warmbly logs -f backend` | +| No claim link printed | The database already has accounts, so there is nothing to claim. See [first run](/development/first-run/) | + +More in [troubleshooting](/development/troubleshooting/) and [instance health](/development/instance-health/). + +## Next + +- [First run](/development/first-run/): claiming the instance, the first owner, demo data +- [Data control](/development/data-control/): where every store lives, what each retention window governs, backups, and moving an instance +- [Configuration](/development/configuration/): every environment variable and every database-backed setting +- [Self-hosting](/development/deployment-guide/): the full guide, including building from source diff --git a/docs/content/docs/development/meta.json b/docs/content/docs/development/meta.json index 638b7b56..cd8f8085 100644 --- a/docs/content/docs/development/meta.json +++ b/docs/content/docs/development/meta.json @@ -4,11 +4,13 @@ "root": true, "pages": [ "---Self-hosting---", + "install", "deployment-guide", "bare-metal", "first-run", "accounts-and-access", "warmblyctl", + "data-control", "configuration", "instance-health", "updates", diff --git a/docs/content/docs/development/updates.mdx b/docs/content/docs/development/updates.mdx index 97438199..2e4519e1 100644 --- a/docs/content/docs/development/updates.mdx +++ b/docs/content/docs/development/updates.mdx @@ -3,7 +3,15 @@ title: Updates description: How a self-hosted instance learns about a new Warmbly version, what the indicator in the admin panel means, and how Update and restart pulls, rebuilds and restarts the stack for you. --- -A self-hosted Warmbly checks for a newer version on its own and shows it in the admin panel. Applying it is one button when the updater runs next to the stack: the checkout is pulled, every image is rebuilt, the services that changed are recreated, and the panel reconnects once the backend answers again. Nothing in flight is lost: sending and syncing pause for the restart and resume from the database on the other side. +A self-hosted Warmbly checks for a newer version on its own and shows it in the admin panel. Applying it is one button when the updater runs next to the stack: the new version is fetched, the services that changed are recreated, and the panel reconnects once the backend answers again. Nothing in flight is lost: sending and syncing pause for the restart and resume from the database on the other side. + +How "fetched" happens depends on how the instance was installed, and the updater has a mode for each: + +| Mode | For | What an update is | +|---|---|---| +| `image` | An install from [`install.sh`](/development/install/) | Pin the new tag in `.env`, `docker compose pull`, recreate. Nothing is built | +| `compose` | A git clone | `git pull`, `docker compose build`, recreate | +| `command` | Anything else | `git pull`, then a build-and-restart script of your own | ## The indicator @@ -47,7 +55,15 @@ The version comes from the binary itself: the Dockerfiles and `make up` stamp th The button appears when the updater is reachable and something newer exists. It needs the `manage_settings` admin permission, and every press is an audit row (`upgrade` on `instance`). -What happens, in the order the dialog shows it: +On an install from `install.sh`, in image mode: + +1. **Pin release.** The chosen tag is written into `WARMBLY_TAG` in the install's `.env`, so the pull, the recreate, and every later `docker compose up` you run by hand all resolve the same images. A pull that fails puts the old tag back rather than leaving `.env` pointing at images that do not exist. +2. **Pull images.** `docker compose pull`. +3. **Restart.** `docker compose up -d --no-build` for every service, minus the updater itself. +4. **Clean up.** `docker image prune -f`, unless `UPDATER_PRUNE=false`. +5. **Wait for backend.** The updater polls `/health` until it answers, for up to six minutes. + +On a git checkout, in compose mode: 1. **Fetch.** `git fetch --tags --prune` on the checkout. 2. **Pull.** On a branch, a fast-forward merge of the remote branch. On a checkout pinned to a tag, a checkout of the newest release. A checkout with local modifications is refused rather than merged over; commit or stash them, or set `UPDATER_ALLOW_DIRTY=true`. @@ -63,12 +79,24 @@ The dialog streams the log. Close it and the pill keeps following the job; reloa Migrations are forward-only, so an update never needs to be rolled back to be safe with data in place. Read [backing up](/development/deployment-guide/#backing-up) anyway: a backup before an update is the one you are glad to have. -Inside the compose stack the updater mounts `/var/run/docker.sock`, which is root on the host. That is why it lives behind the `updater` compose profile and answers only to a bearer token the backend holds, on the compose network, with no published port. The backend gates the button on an admin permission and audits it. If any of that is more trust than the host should extend, leave the profile off and update by hand. +The updater mounts `/var/run/docker.sock`, which is root on the host. That is why on a clone it lives behind the `updater` compose profile, and why in every install it answers only to a bearer token the backend holds, on the compose network, with no published port. The backend gates the button on an admin permission and audits it. If any of that is more trust than the host should extend, leave the profile off and update by hand. ## Enabling the updater -### Docker Compose +### An install from install.sh + +Already done. The installer writes the updater into the generated compose file in image mode, pointed at the install directory, sharing `INTERNAL_API_TOKEN` with the backend. Nothing to configure. + +To update by hand instead, from the install directory: + +```bash +docker compose -p warmbly pull && docker compose -p warmbly up -d +``` + +To leave the button out entirely, delete the `updater` service from `docker-compose.yml` and set `UPDATER_URL=none` in `.env`. + +### Docker Compose from a clone `make up` starts the stack with the `updater` profile, so a stock install already has the button. For a plain `docker compose up`, put this in `.env`: @@ -132,9 +160,9 @@ Updater: | Variable | What it does | Default | |---|---|---| | `UPDATER_TOKEN` | The token it accepts; falls back to `INTERNAL_API_TOKEN`. Refuses to start without one | unset | -| `UPDATER_MODE` | `compose` rebuilds and recreates the compose project; `command` runs `UPDATER_COMMAND` | `compose` | +| `UPDATER_MODE` | `image` pins a release tag and pulls; `compose` rebuilds a checkout; `command` runs `UPDATER_COMMAND` | `compose` | | `UPDATER_COMMAND` | The build-and-restart command for `command` mode | unset | -| `UPDATER_REPO_DIR` | The checkout | working directory | +| `UPDATER_REPO_DIR` | The checkout, or in image mode the install directory holding the compose file and `.env` | working directory | | `UPDATER_REMOTE` | The git remote to fetch and pull from | `origin` | | `UPDATER_COMPOSE_PROJECT` | The `-p` the stack was started with | `warmbly` | | `UPDATER_COMPOSE_PROFILES` | Extra profiles on every compose call, so its own image is rebuilt | `updater` | diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index c4657bf7..aee9d5e2 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -55,10 +55,12 @@ A command that would set a password refuses on a non-TTY unless you passed `--pa | `APP_URL` | every printed link and sign-in hint | Links are built against `https://app.warmbly.com`, which is the hosted service and not your instance | | `KMS_PROVIDER` and its key | `org export`, `org import` | The command stops: a workspace's sealed values cannot be opened, so an archive would be useless | | `CREDENTIALS_ENCRYPTION_KEY` | `org export`, `org import` | A warning, and mailbox credentials are neither read nor written. Everything else still moves | +| `CREDENTIALS_ENCRYPTION_KEY`, `KMS_LOCAL_MASTER_KEY` | `backup`, `restore` | `backup` warns and leaves them out of the bundle. `restore` refuses when this host's differ from the bundle's | +| `BLOB_FS_ROOT` | `backup`, `restore` | Defaults to `/data/blobs`, which is where the compose stack mounts it | | `WARMBLY_API_KEY` | every API command | The command stops and explains where a key comes from | | `WARMBLY_API_URL` | every API command | Falls back to the instance's own `API_PUBLIC_URL`, then the hosted service | -Inside the backend container all of these are already set, except the API key, which is yours. Outside it, export what the command needs, and match `AUTH_SECRET` to the backend's exactly. +`backup` and `restore` also need `pg_dump` and `psql`, which the backend image ships for exactly this reason. Inside the backend container all of these are already set, except the API key, which is yours. Outside it, export what the command needs, and match `AUTH_SECRET` to the backend's exactly. ## The operator commands @@ -73,6 +75,8 @@ Inside the backend container all of these are already set, except the API key, w | [`user revoke-admin`](#user-revoke-admin) | Takes platform admin away from an account | | [`user disable-2fa`](#user-disable-2fa) | Clears an account's authenticator enrolment | | [`hash-password`](#hash-password) | Prints an argon2 hash for unattended provisioning | +| [`backup`](#backup) | Writes the whole instance to one restorable bundle | +| [`restore`](#restore) | Restores a bundle onto this instance, replacing everything on it | | [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size | | [`org export`](#org-export) | Writes a whole workspace to a portable archive file | | [`org import`](#org-import) | Applies an archive to a workspace on this instance | @@ -282,6 +286,54 @@ It prompts when a terminal is attached and reads the pipe when one is not, so th An argon2 PHC string contains `$` characters. Docker Compose reads those as interpolation, so a bare hash in `.env` silently loses part of itself. Wrap it in single quotes there, and double every `$` if you paste it into `docker-compose.yml` directly. +## backup + +```bash +docker compose -p warmbly exec backend warmblyctl backup --out /data/blobs/warmbly.tar.gz +docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz +``` + +Writes one bundle holding the three things that only restore together: a `pg_dump` of the database, the blob root (message bodies, attachments, avatars), and the encryption keys. + +That combination is the whole reason this is a command rather than a documented list of steps. A dump alone restores an instance whose every mailbox credential decrypts to nothing, because the ciphertext is in the database and the key that opens it is in `.env`. The blob root alone restores bodies nothing points at. + +| Flag | Does | +|---|---| +| `--out` | Where to write it. Defaults to `warmbly-backup-.tar.gz` in the working directory | +| `--no-keys` | Leaves the encryption keys out. The bundle is then not restorable on its own | +| `--no-blobs` | Database only | +| `--force` | Overwrite an existing output file | + +The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. Treat the file as you would the instance itself: encrypted at rest, off the host, and not in a shared drive. + +Blobs travel only on the filesystem provider. An instance storing them in S3 keeps them in the bucket, and the bundle says so rather than pretending to be complete. + + +[`install.sh --wizard`](/development/install/) writes a `backup.sh` and a systemd timer that runs exactly this, keeps the last N bundles and can copy each one off the host. + + +## restore + +```bash +docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/warmbly.tar.gz +docker compose -p warmbly restart +``` + +Empties the schema and replays the bundle into it, so it replaces every organization, user, campaign and mailbox currently on the instance. It prints what the bundle holds and asks you to type `restore` before it does; `--yes` skips that for scripts. + +Before anything is written it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` against this host's, and refuses to continue when they differ, printing the two lines to put in `.env` first. That check is the point of the command: without it a restore looks like it worked, and every mailbox fails to authenticate days later with no error that names the cause. + +| Flag | Does | +|---|---| +| `--file` | The bundle. Required | +| `--yes` | Skip the typed confirmation | +| `--no-blobs` | Restore the database only, leaving the blob root alone | +| `--force` | Restore even though the keys differ. Accepts losing every stored mailbox credential | + +Blobs are unpacked over the blob root additively: files the bundle knows about are overwritten, anything else is left alone. + +[Data control](/development/data-control/#moving-an-instance) walks the whole move to a new host. + ## org list ```bash @@ -344,6 +396,8 @@ Run it with `--dry-run` first. The report is the same preflight the dashboard sh The whole import runs in one transaction: if any part fails, nothing lands and the workspace is untouched. Members are matched to accounts on this instance by email address, and anyone without one has their rows reassigned to the workspace owner, named in the report before you commit. An archive carries no password material, so it can never create an account here. +`org export` and `org import` are the per-workspace tool and [`backup`](#backup) is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance. + Billing history, plan overrides, worker placement, mailbox sync checkpoints and warmup pool membership are exported for the record but never applied: each belongs to the instance rather than to the workspace. [Export and import](/guides/workspace-export-import/) has the full table. ## The API commands @@ -405,7 +459,7 @@ Paths are relative to `/v1`. `--data` takes a JSON literal, `-` for stdin, or `@ ### For AI agents -The repository ships skills under `skills/` (`warmbly-api` for the product, `warmbly-ops` for instance administration) that teach a coding agent these commands, the conventions above, and the sending-safety rules. Install them the way your agent expects, for example `cp -r skills/warmbly-api ~/.claude/skills/` for Claude Code, or point the agent at the `SKILL.md` directly. An agent given a scoped key and those skills can operate a workspace end to end without touching the database. +The repository ships skills under `skills/` (`warmbly-api` for the product, `warmbly-ops` for instance administration, `warmbly-install` for standing an instance up and moving it) that teach a coding agent these commands, the conventions above, and the sending-safety rules. Install them the way your agent expects, for example `cp -r skills/warmbly-api ~/.claude/skills/` for Claude Code, or point the agent at the `SKILL.md` directly. An agent given a scoped key and those skills can operate a workspace end to end without touching the database. ## When Redis is down diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index 1869eb7e..42d4c03c 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -5,6 +5,10 @@ description: "Move a whole workspace between Warmbly instances, including mailbo A workspace archive is a single file holding everything one workspace owns. It exists so you can move between instances: a self-hosted install to the cloud, the cloud back to self-hosted, or one self-host to another. + +This page moves **one workspace** between two running instances, re-sealing its secrets for the destination's keys. To move an entire self-hosted install (every workspace, its users, its platform admins) use `warmblyctl backup` and `warmblyctl restore`, which carry the database, the blob root and the encryption keys as one bundle. See [data control](/development/data-control/#backups). The two are not interchangeable: a bundle cannot be applied to a single workspace, and this archive cannot restore an instance. + + Everything here lives under **Settings > Data**, and is limited to the workspace owner. An export with credentials contains every mailbox password in the workspace, so it sits at the same level as deleting the workspace. ## What an archive contains diff --git a/internal/api/handler/admin_instance.go b/internal/api/handler/admin_instance.go index 6f658619..ef388982 100644 --- a/internal/api/handler/admin_instance.go +++ b/internal/api/handler/admin_instance.go @@ -115,6 +115,9 @@ func instanceSettingsAuditDetails(doc instancesettings.Document) map[string]any "sync_backfill_messages": doc.Sync.BackfillMessages, "sync_daily_messages_mailbox": doc.Sync.DailyMessagesPerMailbox, "sync_daily_messages_org": doc.Sync.DailyMessagesPerOrg, + "retention_engagement_event_days": doc.Retention.EngagementEventDays, + "retention_form_event_days": doc.Retention.FormEventDays, + "retention_audit_log_days": doc.Retention.AuditLogDays, "deliverability_enforce_domain_auth": doc.Deliverability.EnforceDomainAuth, "deliverability_auth_grace_hours": doc.Deliverability.AuthGraceHours, } diff --git a/internal/app/consumer/event_tracking.go b/internal/app/consumer/event_tracking.go index a4c47814..45638f84 100644 --- a/internal/app/consumer/event_tracking.go +++ b/internal/app/consumer/event_tracking.go @@ -12,6 +12,7 @@ import ( "github.com/mileusna/useragent" "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/events" "github.com/warmbly/warmbly/internal/infrastructure/codec" @@ -49,8 +50,30 @@ type TrackingConsumer struct { // location for opens and clicks. Both optional. opens repository.EmailOpenRepository geo *geo.Client - topic string - group string + // retention is the operator-editable window the engagement prune obeys. + // Injected post-construction; nil keeps the compiled default. + retention RetentionSource + topic string + group string +} + +// RetentionSource is the operator-editable retention section, satisfied by +// instancesettings.Service. Read on every prune pass, so an edit in the admin +// panel takes effect on the next sweep rather than at the next restart. +type RetentionSource interface { + RetentionWindows(ctx context.Context) instancesettings.Retention +} + +// WireRetention attaches the instance settings the engagement prune reads its +// window from. +func (tc *TrackingConsumer) WireRetention(src RetentionSource) { tc.retention = src } + +// engagementRetentionDays is the window the next prune pass uses. +func (tc *TrackingConsumer) engagementRetentionDays(ctx context.Context) int { + if tc.retention == nil { + return config.EngagementEventRetentionDaysDefault + } + return tc.retention.RetentionWindows(ctx).EngagementEventDays } // NewTrackingConsumer wires the tracking consumer to the shared event bus. @@ -154,15 +177,16 @@ func (tc *TrackingConsumer) pruneEngagementLogs(ctx context.Context) { defer ticker.Stop() for { pctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + days := tc.engagementRetentionDays(pctx) if tc.opens != nil { - if n, err := tc.opens.Cleanup(pctx, config.EngagementEventRetentionDays); err != nil { + if n, err := tc.opens.Cleanup(pctx, days); err != nil { log.Warn().Err(err).Msg("open log prune failed") } else if n > 0 { log.Info().Int64("deleted", n).Msg("open log pruned") } } if tc.linkClicks != nil { - if n, err := tc.linkClicks.Cleanup(pctx, config.EngagementEventRetentionDays); err != nil { + if n, err := tc.linkClicks.Cleanup(pctx, days); err != nil { log.Warn().Err(err).Msg("click log prune failed") } else if n > 0 { log.Info().Int64("deleted", n).Msg("click log pruned") diff --git a/internal/app/instancecheck/checks_updates.go b/internal/app/instancecheck/checks_updates.go index 0681e4f9..c42efeca 100644 --- a/internal/app/instancecheck/checks_updates.go +++ b/internal/app/instancecheck/checks_updates.go @@ -34,7 +34,9 @@ func checkUpdateAvailable(ctx context.Context, d Deps, _ Input) *Finding { msg = "A newer version is available. " } if st.Updater.Status == "ok" { - msg += "Open the version pill in the top bar and choose Update and restart; the stack rebuilds, restarts and resumes on its own." + msg += "Open the version pill in the top bar and choose Update and restart; the stack restarts and resumes on its own." + } else if st.Updater.Release != nil { + msg += "On the host, run docker compose pull && docker compose up -d in the install directory, or enable the updater to do it from this panel." } else { msg += "Run make upgrade on the host, or enable the updater to do it from this panel." } diff --git a/internal/app/instanceconfig/entries.go b/internal/app/instanceconfig/entries.go index 9d17b8d7..57fb9c42 100644 --- a/internal/app/instanceconfig/entries.go +++ b/internal/app/instanceconfig/entries.go @@ -27,6 +27,9 @@ const ( docsSSO = "/development/accounts-and-access/#single-sign-on" docsFirstOwner = "/development/accounts-and-access/#first-owner" docsUpdates = "/development/updates/" + // The database-backed settings document, which is the one tier the + // environment does not own. + docsSettingsDoc = "/development/configuration/#settings-stored-in-the-database" ) // table is the static inventory. Declaration order is display order. @@ -508,6 +511,12 @@ var table = []Entry{ DocsAnchor: docsFirstOwner, Resolve: envValue("WARMBLY_BOOTSTRAP_ORG"), }, + { + Key: "WARMBLY_SETTINGS_BOOTSTRAP", Group: GroupDeployment, RuntimeChangeable: ChangeBootOnly, + Effect: "Seeds the database-backed settings document (sync budgets, retention windows) on an instance that has never saved one. A no-op from the first save in Instance settings onwards, so leaving it here cannot undo an edit made there.", + DocsAnchor: docsSettingsDoc, + Resolve: envValue("WARMBLY_SETTINGS_BOOTSTRAP"), + }, // Captcha. { diff --git a/internal/app/instancesettings/document.go b/internal/app/instancesettings/document.go index fc6a1a2c..eabd7a7b 100644 --- a/internal/app/instancesettings/document.go +++ b/internal/app/instancesettings/document.go @@ -48,6 +48,27 @@ type Sync struct { DailyMessagesPerOrg int `json:"daily_messages_per_org"` } +// Retention holds how long event-level history is kept. Every window here +// bounds personal data: opens and clicks carry a client, a device and a +// location, funnel events carry a visitor's path, and the audit trail carries +// IP addresses, user agents and change payloads. Shortening one is the only +// way an operator can hold less without patching the binary. +// +// None of them affect a count, a filter or a routing decision: campaign +// progress keeps its own summary of opens and clicks, which outlives the +// per-event log. +type Retention struct { + // EngagementEventDays is how long the per-event open and click logs are + // kept. + EngagementEventDays int `json:"engagement_event_days"` + // FormEventDays is how long form funnel events are kept. Forms analytics + // ranges top out at 90 days, so anything shorter than that shortens what + // the funnel report can show. + FormEventDays int `json:"form_event_days"` + // AuditLogDays is how long the audit trail is kept. + AuditLogDays int `json:"audit_log_days"` +} + // Bounds on the domain-authentication grace window. One hour is the shortest // window that still absorbs a resolver blip; 30 days is the longest a domain // should keep sending cold mail unauthenticated while being warned about it. @@ -76,6 +97,7 @@ type Document struct { Invitations Invitations `json:"invitations"` Access Access `json:"access"` Sync Sync `json:"sync"` + Retention Retention `json:"retention"` Deliverability Deliverability `json:"deliverability"` } @@ -90,6 +112,7 @@ func Defaults() Document { AllowInvitedSignup: true, }, Sync: DefaultSync(), + Retention: DefaultRetention(), Deliverability: DefaultDeliverability(), } } @@ -115,6 +138,37 @@ func DefaultSync() Sync { } } +// DefaultRetention is the compiled retention window for each event log. +func DefaultRetention() Retention { + return Retention{ + EngagementEventDays: config.EngagementEventRetentionDaysDefault, + FormEventDays: config.FormEventsRetentionDaysDefault, + AuditLogDays: config.AuditLogRetentionDaysDefault, + } +} + +// Normalize clamps every window into its accepted range. Zero and negative +// resolve to the compiled default rather than to "keep nothing": a document +// written before this section existed must not silently start deleting +// everything on the next sweep. +func (r *Retention) Normalize() { + clamp := func(v, def int) int { + if v <= 0 { + return def + } + if v < config.RetentionDaysMin { + return config.RetentionDaysMin + } + if v > config.RetentionDaysMax { + return config.RetentionDaysMax + } + return v + } + r.EngagementEventDays = clamp(r.EngagementEventDays, config.EngagementEventRetentionDaysDefault) + r.FormEventDays = clamp(r.FormEventDays, config.FormEventsRetentionDaysDefault) + r.AuditLogDays = clamp(r.AuditLogDays, config.AuditLogRetentionDaysDefault) +} + // Normalize clamps a document into its accepted range. It is applied on read // as well as on write, so a row written by an older version still resolves. func (d *Document) Normalize() { @@ -128,6 +182,7 @@ func (d *Document) Normalize() { d.Invitations.TTLHours = TTLHoursMax } d.Sync.Normalize() + d.Retention.Normalize() d.Deliverability.Normalize() } @@ -192,6 +247,11 @@ type Patch struct { DailyMessagesPerMailbox *int `json:"daily_messages_per_mailbox"` DailyMessagesPerOrg *int `json:"daily_messages_per_org"` } `json:"sync"` + Retention *struct { + EngagementEventDays *int `json:"engagement_event_days"` + FormEventDays *int `json:"form_event_days"` + AuditLogDays *int `json:"audit_log_days"` + } `json:"retention"` Deliverability *struct { EnforceDomainAuth *bool `json:"enforce_domain_auth"` AuthGraceHours *int `json:"auth_grace_hours"` @@ -230,6 +290,17 @@ func (p Patch) Apply(doc Document) Document { doc.Sync.DailyMessagesPerOrg = *p.Sync.DailyMessagesPerOrg } } + if p.Retention != nil { + if p.Retention.EngagementEventDays != nil { + doc.Retention.EngagementEventDays = *p.Retention.EngagementEventDays + } + if p.Retention.FormEventDays != nil { + doc.Retention.FormEventDays = *p.Retention.FormEventDays + } + if p.Retention.AuditLogDays != nil { + doc.Retention.AuditLogDays = *p.Retention.AuditLogDays + } + } if p.Deliverability != nil { if p.Deliverability.EnforceDomainAuth != nil { doc.Deliverability.EnforceDomainAuth = *p.Deliverability.EnforceDomainAuth diff --git a/internal/app/instancesettings/service.go b/internal/app/instancesettings/service.go index 0e74294c..f54b4dd8 100644 --- a/internal/app/instancesettings/service.go +++ b/internal/app/instancesettings/service.go @@ -27,12 +27,23 @@ type Service interface { // settings row is missing. Get(ctx context.Context) Document Put(ctx context.Context, patch Patch, updatedBy *uuid.UUID) (Document, error) + // Bootstrap applies patch only on an instance whose settings row has never + // been written, and reports whether it did. It is how an unattended + // install ships the data-control answers with the rest of the + // environment; from the first write onwards the admin panel is + // authoritative and this is a no-op, so leaving the variable in place does + // not undo an operator's later edit. + Bootstrap(ctx context.Context, patch Patch) (bool, error) InvitationTTL(ctx context.Context) time.Duration InviteLinksEnabled(ctx context.Context) bool AllowInvitedSignup(ctx context.Context) bool // SyncBudget is the mailbox sync fair-use section, already normalized. SyncBudget(ctx context.Context) Sync + // RetentionWindows is the event-log retention section, already normalized. + // The retention sweeps read it on every pass, so an edit takes effect on + // the next one rather than at the next restart. + RetentionWindows(ctx context.Context) Retention // DomainAuth is the sending-domain authentication gate: whether it is // enforced at all, and how long a domain must stay failing first. DomainAuth(ctx context.Context) (enforce bool, grace time.Duration) @@ -90,6 +101,24 @@ func (s *service) Put(ctx context.Context, patch Patch, updatedBy *uuid.UUID) (D return next, nil } +func (s *service) Bootstrap(ctx context.Context, patch Patch) (bool, error) { + if s.store == nil { + return false, nil + } + written, err := s.store.Exists(ctx) + if err != nil { + return false, err + } + if written { + return false, nil + } + // updatedBy is nil: nobody signed in made this change, the environment did. + if _, err := s.Put(ctx, patch, nil); err != nil { + return false, err + } + return true, nil +} + func (s *service) InvitationTTL(ctx context.Context) time.Duration { return s.Get(ctx).TTL() } func (s *service) InviteLinksEnabled(ctx context.Context) bool { @@ -106,6 +135,12 @@ func (s *service) SyncBudget(ctx context.Context) Sync { return sync } +func (s *service) RetentionWindows(ctx context.Context) Retention { + r := s.Get(ctx).Retention + r.Normalize() + return r +} + func (s *service) DomainAuth(ctx context.Context) (bool, time.Duration) { d := s.Get(ctx).Deliverability d.Normalize() diff --git a/internal/app/instancesettings/store.go b/internal/app/instancesettings/store.go index f366acee..ea5443a9 100644 --- a/internal/app/instancesettings/store.go +++ b/internal/app/instancesettings/store.go @@ -13,6 +13,10 @@ import ( type Store interface { Get(ctx context.Context) (Document, error) Put(ctx context.Context, doc Document, updatedBy *uuid.UUID) error + // Exists reports whether the row has ever been written. Get cannot answer + // that: it resolves an absent row to the defaults, deliberately, so the + // invitation path never breaks on a missing row. + Exists(ctx context.Context) (bool, error) } type pgStore struct { @@ -43,6 +47,18 @@ func (s *pgStore) Get(ctx context.Context) (Document, error) { return doc, nil } +func (s *pgStore) Exists(ctx context.Context) (bool, error) { + var one int + err := s.db.QueryRow(ctx, `SELECT 1 FROM instance_settings WHERE id = true`).Scan(&one) + if err == pgx.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + func (s *pgStore) Put(ctx context.Context, doc Document, updatedBy *uuid.UUID) error { raw, err := json.Marshal(doc) if err != nil { diff --git a/internal/app/updates/service.go b/internal/app/updates/service.go index 1a92d790..21f57818 100644 --- a/internal/app/updates/service.go +++ b/internal/app/updates/service.go @@ -56,11 +56,14 @@ type UpdaterView struct { // Configured is whether UPDATER_URL is set at all. Configured bool `json:"configured"` // Status is off (not configured), ok, or unreachable. - Status string `json:"status"` - Error string `json:"error,omitempty"` - Mode updater.Mode `json:"mode,omitempty"` - RepoDir string `json:"repo_dir,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Mode updater.Mode `json:"mode,omitempty"` + RepoDir string `json:"repo_dir,omitempty"` + // Exactly one of these is set: a checkout in compose and command mode, a + // release in image mode (the clone-free install). Checkout *updater.Checkout `json:"checkout,omitempty"` + Release *updater.Release `json:"release,omitempty"` Job *updater.Job `json:"job,omitempty"` LastJob *updater.Job `json:"last_job,omitempty"` } @@ -84,7 +87,7 @@ type State struct { var ( ErrUpdaterNotConfigured = errors.New("no updater is configured on this instance") - ErrNothingToApply = errors.New("the checkout is pinned to a tag and no release is known to move to") + ErrNothingToApply = errors.New("this install is pinned to a version and no release is known to move to; run a check first") ) type Service struct { @@ -209,7 +212,12 @@ func (s *Service) Apply(ctx context.Context, target string) (*updater.Job, error req := updater.UpdateRequest{} switch strings.TrimSpace(target) { case "", "latest", "branch": - if view.Checkout != nil && view.Checkout.Detached { + // A pinned install has nothing to move to on its own: an image install + // reads its tag from .env and a detached checkout is on a tag, so both + // need the release the check found naming the destination. + pinned := (view.Checkout != nil && view.Checkout.Detached) || + (view.Mode == updater.ModeImage && (view.Release == nil || view.Release.Pinned)) + if pinned { s.mu.Lock() latest := s.latest s.mu.Unlock() @@ -357,6 +365,7 @@ func (s *Service) updaterStatus(ctx context.Context, method, path string) Update view.Mode = st.Mode view.RepoDir = st.RepoDir view.Checkout = st.Checkout + view.Release = st.Release view.Job = st.Job view.LastJob = st.LastJob return view diff --git a/internal/config/constants.go b/internal/config/constants.go index 5f8dc051..7781394e 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -68,9 +68,9 @@ const ( SyncFloodPerHour = 5_000 // new live messages observed in one hour that mark a mailbox as flooding SyncThrottleEscalationDays = 3 // throttled UTC days out of the last 7 that deactivate a mailbox - // Forms. Funnel events feed analytics ranges up to 90 days; the fixed - // retention window keeps double coverage without a per-org setting. - FormEventsRetentionDays = 180 + // Forms. Funnel events feed analytics ranges up to 90 days, so the default + // window keeps double coverage. Operator-editable under Instance settings. + FormEventsRetentionDaysDefault = 180 // Sequences. Empty by default so the editor shows a smart, position-based // label (e.g. "Email 1") until the user names the step themselves. @@ -145,10 +145,24 @@ const ( // a scanner walking the message. A person follows one link at a time. TrackingClickBurstSeconds = 5 - // EngagementEventRetentionDays is how long the per-event open and click - // logs (client, device, location) are kept. The summary on the progress - // row outlives them, so counts and routing never change. - EngagementEventRetentionDays = 365 + // EngagementEventRetentionDaysDefault is how long the per-event open and + // click logs (client, device, location) are kept. The summary on the + // progress row outlives them, so counts and routing never change. + // Operator-editable under Instance settings. + EngagementEventRetentionDaysDefault = 365 + + // AuditLogRetentionDaysDefault is how long the audit trail is kept. The + // trail carries IP addresses, user agents and change payloads, so this + // window is also how long that PII is held; a privacy-conscious operator + // shortens it under Instance settings. + AuditLogRetentionDaysDefault = 90 + + // Ten years is the ceiling every retention window shares. It is not a + // recommendation: it is the point past which "keep it" and "keep it + // forever" stop differing, and it bounds a typo. One day is the floor, so + // there is always a window in which an event can be read. + RetentionDaysMin = 1 + RetentionDaysMax = 3650 // CampaignSendStampAttempts is how many times the control plane retries the // sent_at stamp after a send is already on the bus. The reservation is what diff --git a/internal/jobs/audit_retention.go b/internal/jobs/audit_retention.go index a56222d1..9e129d1f 100644 --- a/internal/jobs/audit_retention.go +++ b/internal/jobs/audit_retention.go @@ -5,33 +5,42 @@ import ( "time" "github.com/getsentry/sentry-go" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/repository" ) // AuditRetentionJob deletes audit-log entries older than the retention window. // Bounding the trail's age also bounds how long PII (IP addresses, user agents, -// change payloads) is retained, which is a privacy-positive property. +// change payloads) is retained, which is a privacy-positive property. The +// window is an instance setting, read on every pass. type AuditRetentionJob struct { repo repository.AuditRepository - retention time.Duration + retention RetentionSource } -// NewAuditRetentionJob creates a retention job that prunes entries older than -// the given retention window. -func NewAuditRetentionJob(repo repository.AuditRepository, retention time.Duration) *AuditRetentionJob { - return &AuditRetentionJob{ - repo: repo, - retention: retention, - } +// NewAuditRetentionJob creates a retention job for the audit trail. Without a +// wired settings source it prunes at the compiled default window. +func NewAuditRetentionJob(repo repository.AuditRepository) *AuditRetentionJob { + return &AuditRetentionJob{repo: repo} +} + +// WireRetention attaches the instance settings the window is read from. +func (j *AuditRetentionJob) WireRetention(src RetentionSource) *AuditRetentionJob { + j.retention = src + return j } // Run executes one pruning pass. func (j *AuditRetentionJob) Run(ctx context.Context) error { - if j.repo == nil || j.retention <= 0 { + if j.repo == nil { return nil } + days := config.AuditLogRetentionDaysDefault + if j.retention != nil { + days = j.retention.RetentionWindows(ctx).AuditLogDays + } - cutoff := time.Now().Add(-j.retention) + cutoff := time.Now().AddDate(0, 0, -days) if _, err := j.repo.PruneOlderThan(ctx, cutoff); err != nil { sentry.CaptureException(err) return err diff --git a/internal/jobs/form_events_retention.go b/internal/jobs/form_events_retention.go index d8aab24c..9d75f525 100644 --- a/internal/jobs/form_events_retention.go +++ b/internal/jobs/form_events_retention.go @@ -8,22 +8,34 @@ import ( "github.com/warmbly/warmbly/internal/repository" ) -// FormEventsRetentionJob prunes funnel events past the platform window; the -// forms analytics ranges top out at 90 days, so the fixed window keeps double -// coverage without a per-org setting. +// FormEventsRetentionJob prunes funnel events past the retention window. The +// window is an instance setting, read on every pass, so an operator who +// shortens it in the admin panel sees the next sweep honour it. type FormEventsRetentionJob struct { - repo repository.FormEventRepository + repo repository.FormEventRepository + retention RetentionSource } func NewFormEventsRetentionJob(repo repository.FormEventRepository) *FormEventsRetentionJob { return &FormEventsRetentionJob{repo: repo} } +// WireRetention attaches the instance settings the window is read from. Unset +// keeps the compiled default. +func (j *FormEventsRetentionJob) WireRetention(src RetentionSource) *FormEventsRetentionJob { + j.retention = src + return j +} + func (j *FormEventsRetentionJob) Run(ctx context.Context) error { if j.repo == nil { return nil } - before := time.Now().AddDate(0, 0, -config.FormEventsRetentionDays) + days := config.FormEventsRetentionDaysDefault + if j.retention != nil { + days = j.retention.RetentionWindows(ctx).FormEventDays + } + before := time.Now().AddDate(0, 0, -days) if _, xerr := j.repo.PruneBefore(ctx, before); xerr != nil { return xerr } diff --git a/internal/jobs/retention.go b/internal/jobs/retention.go new file mode 100644 index 00000000..7f08ad1f --- /dev/null +++ b/internal/jobs/retention.go @@ -0,0 +1,15 @@ +package jobs + +import ( + "context" + + "github.com/warmbly/warmbly/internal/app/instancesettings" +) + +// RetentionSource is the operator-editable retention section, satisfied by +// instancesettings.Service. The retention jobs read it on every pass rather +// than capturing a window at construction, so shortening one in the admin +// panel takes effect on the next sweep instead of at the next restart. +type RetentionSource interface { + RetentionWindows(ctx context.Context) instancesettings.Retention +} diff --git a/internal/updater/api.go b/internal/updater/api.go index 5fd6c9e3..e4e4f5e9 100644 --- a/internal/updater/api.go +++ b/internal/updater/api.go @@ -18,6 +18,10 @@ const ( // ModeCommand runs an operator-provided command (a build-and-restart // script) and leaves the rest to it. ModeCommand Mode = "command" + // ModeImage is the clone-free install: no git, no build. The release tag + // is pinned in the install's .env, images are pulled from the registry and + // the containers whose image changed are recreated. + ModeImage Mode = "image" ) // JobStatus is the lifecycle of one update run. @@ -56,18 +60,36 @@ type Job struct { Log []string `json:"log"` } +// Release describes an image-mode install: which tag its compose file +// resolves and where the images come from. It is the image-mode counterpart +// of Checkout, and exactly one of the two is ever set. +type Release struct { + // Tag is the value of WARMBLY_TAG in the install's .env, or the compose + // default when it names none. + Tag string `json:"tag"` + // Prefix is the registry namespace every image is pulled from. + Prefix string `json:"prefix"` + // Pinned is whether Tag is a fixed version rather than a moving channel + // tag. A moving tag republishes under the same name, so re-pulling it is + // a real update; a pinned one only moves when a release is chosen. + Pinned bool `json:"pinned"` +} + // Status is the updater's answer to GET /status. type Status struct { - Mode Mode `json:"mode"` - RepoDir string `json:"repo_dir"` - Version string `json:"version"` + Mode Mode `json:"mode"` + RepoDir string `json:"repo_dir"` + Version string `json:"version"` + // Checkout is set in compose and command mode, Release in image mode. Checkout *Checkout `json:"checkout,omitempty"` + Release *Release `json:"release,omitempty"` Job *Job `json:"job,omitempty"` LastJob *Job `json:"last_job,omitempty"` } // UpdateRequest is the body of POST /update. An empty Tag means: pull the -// tracked branch when the checkout is on one, otherwise refuse. +// tracked branch when the checkout is on one, otherwise refuse. In image mode +// it means re-pull whatever tag the install is already pinned to. type UpdateRequest struct { Tag string `json:"tag"` } diff --git a/internal/updater/compose.go b/internal/updater/compose.go index 0a6a7fc6..18fc6d12 100644 --- a/internal/updater/compose.go +++ b/internal/updater/compose.go @@ -97,9 +97,32 @@ func (r *Runner) servicesToRecreate(ctx context.Context) ([]string, error) { return out, nil } -// recreateSelf moves the updater onto the image it just built. It cannot run -// `compose up updater` in-process (that stops this container half way), so a -// detached one-off container from the new image does it a few seconds later. +// selfImageID is the image id the compose file resolves for the updater +// service, whether that image was just built or just pulled. It is asked of +// compose rather than assembled by hand, because the service carries an +// image: key now and `project-updater` is no longer its tag. +func (r *Runner) selfImageID(ctx context.Context) string { + ref, err := r.composeOutput(ctx, "config", "--images", selfService) + if err != nil || ref == "" { + // Older compose, or a file with neither key: fall back to the name a + // build without an image: key produces. + ref = r.cfg.ComposeProject + "-" + selfService + } + // --images answers one line per service; only one service was asked for. + if i := strings.IndexByte(ref, '\n'); i >= 0 { + ref = ref[:i] + } + out, err := exec.CommandContext(ctx, "docker", "image", "inspect", "-f", "{{.Id}}", strings.TrimSpace(ref)).Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// recreateSelf moves the updater onto the image it just built or pulled. It +// cannot run `compose up updater` in-process (that stops this container half +// way), so a detached one-off container from the new image does it a few +// seconds later. func (r *Runner) recreateSelf(ctx context.Context, job *Job) { running, err := r.composeOutput(ctx, "ps", "-q", selfService) if err != nil || running == "" { @@ -109,9 +132,8 @@ func (r *Runner) recreateSelf(ctx context.Context, job *Job) { if err != nil { return } - builtImage, err := exec.CommandContext(ctx, "docker", "image", "inspect", "-f", "{{.Id}}", - r.cfg.ComposeProject+"-"+selfService).Output() - if err != nil || strings.TrimSpace(string(currentImage)) == strings.TrimSpace(string(builtImage)) { + nextImage := r.selfImageID(ctx) + if nextImage == "" || strings.TrimSpace(string(currentImage)) == nextImage { return } r.logf(job, "updater image changed; recreating the updater itself") diff --git a/internal/updater/image.go b/internal/updater/image.go new file mode 100644 index 00000000..717b9cff --- /dev/null +++ b/internal/updater/image.go @@ -0,0 +1,201 @@ +package updater + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Image mode is the updater for an install that never had a checkout: the one +// `curl warmbly.com/install.sh | sh` creates, which holds a generated compose +// file and a .env and runs the published release images. There is nothing to +// pull with git and nothing to build, so an update is: pin the tag in .env, +// `docker compose pull`, recreate what changed. + +// tagVar is the .env key the generated compose file reads every image tag +// from, so pinning a release is one line the operator can also edit by hand. +const tagVar = "WARMBLY_TAG" + +// envFileName is the environment file next to the compose file. Compose reads +// it automatically, so writing the tag there is all it takes to move. +const envFileName = ".env" + +// composeFileNames are the compose files an image install may carry, in the +// order docker compose itself resolves them. +var composeFileNames = []string{"docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"} + +// composeFile finds the install's compose file, or reports that there is none. +func composeFile(dir string) (string, bool) { + for _, name := range composeFileNames { + p := filepath.Join(dir, name) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p, true + } + } + return "", false +} + +// readTag returns the tag the install is pinned to, or "prod" when .env does +// not name one (the compose default). +func (r *Runner) readTag() string { + if v := readEnvVar(filepath.Join(r.cfg.RepoDir, envFileName), tagVar); v != "" { + return v + } + return "prod" +} + +// releaseState is the image-mode answer to "what is this install running". +func (r *Runner) releaseState() *Release { + tag := r.readTag() + return &Release{ + Tag: tag, + Prefix: r.imagePrefix(), + // A moving tag republishes under the same name, so "pull again" is a + // real update for it and a no-op for a pinned version. + Pinned: tag != "prod" && tag != "dev" && tag != "latest", + } +} + +func (r *Runner) imagePrefix() string { + if v := readEnvVar(filepath.Join(r.cfg.RepoDir, envFileName), "WARMBLY_IMAGE_PREFIX"); v != "" { + return v + } + return "ghcr.io/warmbly/warmbly" +} + +// imageUpdate is the image-mode counterpart of composeUpdate: no git, no +// build. The tag is written to .env FIRST so the pull, the recreate and every +// later `docker compose up` by hand all resolve the same images. +func (r *Runner) imageUpdate(ctx context.Context, job *Job, tag string) error { + r.step(job, "resolve") + if tag == "" { + tag = r.readTag() + r.logf(job, "no release named; re-pulling the pinned tag %s", tag) + } + from := r.readTag() + envPath := filepath.Join(r.cfg.RepoDir, envFileName) + if err := setEnvVar(envPath, tagVar, tag); err != nil { + return fmt.Errorf("could not pin %s=%s in %s: %w", tagVar, tag, envPath, err) + } + if from == tag { + r.logf(job, "already pinned to %s; pulling it again", tag) + } else { + r.logf(job, "moved %s -> %s in %s", from, tag, envFileName) + } + + r.step(job, "pull") + r.logf(job, "pulling %s/*:%s", r.imagePrefix(), tag) + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", r.composeArgs("pull")...); err != nil { + // A pull that fails leaves .env pointing at a tag with no images, and + // the next `docker compose up` by hand would then fail the same way. + if rerr := setEnvVar(envPath, tagVar, from); rerr == nil { + r.logf(job, "pull failed; %s put back to %s", tagVar, from) + } + return err + } + + r.step(job, "restart") + services, err := r.servicesToRecreate(ctx) + if err != nil { + return err + } + r.logf(job, "recreating %s", strings.Join(services, ", ")) + args := append(r.composeArgs("up", "-d", "--no-build", "--remove-orphans"), services...) + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", args...); err != nil { + return err + } + + if r.cfg.Prune { + r.step(job, "prune") + if err := r.exec(ctx, job, r.cfg.RepoDir, nil, "docker", "image", "prune", "-f"); err != nil { + r.logf(job, "image prune failed (ignored): %v", err) + } + } + return nil +} + +// env file editing +// +// The file is read and rewritten whole rather than appended to, so a second +// update does not leave two WARMBLY_TAG lines with the loser still readable. + +// readEnvVar returns the last assignment of key in a KEY=VALUE file, or "". +// Last wins because that is what docker compose itself does. +func readEnvVar(path, key string) string { + b, err := os.ReadFile(path) //nolint:gosec // operator-owned install directory + if err != nil { + return "" + } + out := "" + for _, line := range strings.Split(string(b), "\n") { + k, v, ok := splitEnvLine(line) + if ok && k == key { + out = v + } + } + return out +} + +// setEnvVar rewrites key in place, keeping every other line and the file's +// position, and appends it when it is absent. The write is atomic and 0600: +// this file holds every secret the instance has. +func setEnvVar(path, key, value string) error { + raw, err := os.ReadFile(path) //nolint:gosec // operator-owned install directory + if err != nil && !os.IsNotExist(err) { + return err + } + lines := []string{} + if len(raw) > 0 { + lines = strings.Split(strings.TrimRight(string(raw), "\n"), "\n") + } + replaced := false + for i, line := range lines { + k, _, ok := splitEnvLine(line) + if !ok || k != key { + continue + } + if replaced { + // A duplicate earlier in the file would win nothing but confuse + // the next person to read it. + lines[i] = "# " + line + " # superseded by the updater" + continue + } + lines[i] = key + "=" + value + replaced = true + } + if !replaced { + lines = append(lines, key+"="+value) + } + body := strings.Join(lines, "\n") + "\n" + + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// splitEnvLine parses one KEY=VALUE line, skipping comments and blanks and +// unwrapping the quotes compose accepts. +func splitEnvLine(line string) (string, string, bool) { + s := strings.TrimSpace(line) + if s == "" || strings.HasPrefix(s, "#") { + return "", "", false + } + s = strings.TrimPrefix(s, "export ") + key, value, ok := strings.Cut(s, "=") + if !ok { + return "", "", false + } + key = strings.TrimSpace(key) + if key == "" { + return "", "", false + } + value = strings.TrimSpace(value) + if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] { + value = value[1 : len(value)-1] + } + return key, value, true +} diff --git a/internal/updater/runner.go b/internal/updater/runner.go index 9ba437a6..314cf551 100644 --- a/internal/updater/runner.go +++ b/internal/updater/runner.go @@ -66,7 +66,13 @@ func NewRunner(cfg Config) (*Runner, error) { if cfg.RepoDir == "" { return nil, errors.New("UPDATER_REPO_DIR is required") } - if _, err := os.Stat(filepath.Join(cfg.RepoDir, ".git")); err != nil { + if cfg.Mode == ModeImage { + // The clone-free install has no checkout to inspect; what it must have + // is the compose file the pull and the recreate address. + if _, ok := composeFile(cfg.RepoDir); !ok { + return nil, fmt.Errorf("%s holds no compose file; UPDATER_MODE=image needs the install directory", cfg.RepoDir) + } + } else if _, err := os.Stat(filepath.Join(cfg.RepoDir, ".git")); err != nil { return nil, fmt.Errorf("%s is not a git checkout: %w", cfg.RepoDir, err) } if cfg.Mode == ModeCommand && strings.TrimSpace(cfg.Command) == "" { @@ -103,8 +109,13 @@ func (r *Runner) Start(ctx context.Context) { }() } -// Refresh fetches the remote and re-reads the checkout. +// Refresh fetches the remote and re-reads the checkout. In image mode there is +// nothing to fetch: what a release exists is the backend's GitHub check, and +// what is installed is one line of .env, read live in Status. func (r *Runner) Refresh(ctx context.Context) *Checkout { + if r.cfg.Mode == ModeImage { + return nil + } fetchErr := r.git.fetch(ctx) c, err := r.git.inspect(ctx) if err != nil { @@ -130,6 +141,17 @@ func (r *Runner) Status(ctx context.Context) Status { last := cloneJob(r.lastJob) r.mu.Unlock() + if r.cfg.Mode == ModeImage { + return Status{ + Mode: r.cfg.Mode, + RepoDir: r.cfg.RepoDir, + Version: r.cfg.Version, + Release: r.releaseState(), + Job: job, + LastJob: last, + } + } + c, err := r.git.inspect(ctx) if err == nil && prev != nil { c.FetchedAt = prev.FetchedAt @@ -157,7 +179,12 @@ func (r *Runner) StartUpdate(req UpdateRequest) (*Job, error) { } target := strings.TrimSpace(req.Tag) if target == "" { + // What "no tag" means differs per mode, and the job label is what the + // admin panel shows while it runs. target = "branch" + if r.cfg.Mode == ModeImage { + target = r.readTag() + } } job := &Job{ ID: uuid.NewString(), @@ -202,7 +229,7 @@ func (r *Runner) execute(ctx context.Context, job *Job, req UpdateRequest) { r.mu.Unlock() r.saveState() - if err == nil && r.cfg.Mode == ModeCompose { + if err == nil && (r.cfg.Mode == ModeCompose || r.cfg.Mode == ModeImage) { // Last, because it may replace this very process: the outcome above is // already on disk for the successor to report. r.recreateSelf(ctx, job) @@ -210,6 +237,13 @@ func (r *Runner) execute(ctx context.Context, job *Job, req UpdateRequest) { } func (r *Runner) runSteps(ctx context.Context, job *Job, req UpdateRequest) error { + if r.cfg.Mode == ModeImage { + if err := r.imageUpdate(ctx, job, strings.TrimSpace(req.Tag)); err != nil { + return err + } + return r.waitForBackend(ctx, job) + } + from, err := r.git.head(ctx) if err != nil { return err @@ -278,14 +312,21 @@ func (r *Runner) runSteps(ctx context.Context, job *Job, req UpdateRequest) erro } } - if r.cfg.BackendHealthURL != "" { - r.step(job, "wait") - r.logf(job, "waiting for the backend at %s", r.cfg.BackendHealthURL) - if err := waitHealthy(ctx, r.cfg.BackendHealthURL, healthWait); err != nil { - return err - } - r.logf(job, "backend is answering") + return r.waitForBackend(ctx, job) +} + +// waitForBackend is the last step of every mode: the update is only finished +// once the API answers again. +func (r *Runner) waitForBackend(ctx context.Context, job *Job) error { + if r.cfg.BackendHealthURL == "" { + return nil } + r.step(job, "wait") + r.logf(job, "waiting for the backend at %s", r.cfg.BackendHealthURL) + if err := waitHealthy(ctx, r.cfg.BackendHealthURL, healthWait); err != nil { + return err + } + r.logf(job, "backend is answering") return nil } diff --git a/scripts/check-installer.sh b/scripts/check-installer.sh new file mode 100755 index 00000000..99bfab8a --- /dev/null +++ b/scripts/check-installer.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# +# Checks the one-command installer served at https://warmbly.com/install.sh. +# +# It is served verbatim out of site/public, so this runs against the exact +# bytes a `curl -fsSL https://warmbly.com/install.sh | sh` executes: +# +# * it parses as POSIX sh, in dash and not only in bash +# * shellcheck has nothing to say about it +# * --help, --print-env and --dry-run work without a terminal, a docker or a +# network, because that is how someone reads it before trusting it +# * every compose file it can generate is one docker compose accepts +# * the published checksum matches, so the documented +# "download, verify, read, run" path actually verifies +set -euo pipefail + +cd "$(dirname "$0")/.." +SCRIPT=site/public/install.sh +SUMFILE=site/public/install.sh.sha256 + +fail() { printf '\n\033[31m✗\033[0m %s\n' "$*" >&2; exit 1; } +pass() { printf '\033[32m✓\033[0m %s\n' "$*"; } + +[[ -f $SCRIPT ]] || fail "$SCRIPT is missing" + +# The script is executed by whatever /bin/sh is on the operator's box, which on +# Debian and Ubuntu is dash. Checking it with bash alone would let a bashism +# through to exactly the hosts this is aimed at. +if command -v dash >/dev/null 2>&1; then + dash -n "$SCRIPT" || fail "the installer is not valid POSIX sh (dash -n)" + pass "parses as POSIX sh" +else + sh -n "$SCRIPT" || fail "the installer does not parse" + pass "parses (dash not installed; POSIX check was approximate)" +fi + +if command -v shellcheck >/dev/null 2>&1; then + shellcheck -s sh "$SCRIPT" || fail "shellcheck found problems in the installer" + pass "shellcheck clean" +else + echo "· shellcheck not installed; skipped" +fi + +# --help must work before anything is set up, which is where an unbound +# variable under set -u would otherwise hide. +sh "$SCRIPT" --help >/dev/null || fail "--help failed" +pass "--help works" + +# --demo has to reach its end with no terminal, no Docker and no network, and +# above all it must not create the install directory it talks about. +demo_dir=$(mktemp -d)/opt-warmbly +sh "$SCRIPT" --demo --no-color --dir "$demo_dir" >/dev/null 2>&1 || fail "--demo failed" +[[ ! -e $demo_dir ]] || fail "--demo created $demo_dir; it must write nothing" +pass "--demo runs and writes nothing" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +sh "$SCRIPT" --print-env --no-color --dir "$work/inst" --version v0.0.0-test >"$work/env" || + fail "--print-env failed" +for key in WARMBLY_TAG AUTH_SECRET CREDENTIALS_ENCRYPTION_KEY KMS_LOCAL_MASTER_KEY \ + INTERNAL_API_TOKEN SECRET_KEY_BASE PRIMARY_DB WARMBLY_SETTINGS_BOOTSTRAP; do + grep -q "^${key}=." "$work/env" || fail "--print-env wrote no $key" +done +grep -q '^WARMBLY_TAG=v0.0.0-test$' "$work/env" || fail "--version was not pinned into .env" +pass "--print-env writes a complete .env" + +# Every shape of answer has to produce a compose file compose accepts. These +# are the four that change the file's structure rather than its values. +check_shape() { + local label=$1; shift + local dir="$work/shape" + rm -rf "$dir"; mkdir -p "$dir" + sh "$SCRIPT" --dry-run --no-color --dir "$dir/inst" --version v0.0.0-test "$@" >"$dir/out" || + fail "--dry-run failed for: $label" + python3 - "$dir" <<'PY' +import sys, os, re +d = sys.argv[1] +lines = open(os.path.join(d, "out")).read().split("\n") +def extract(marker, out): + idx = [i for i, l in enumerate(lines) if l.strip().startswith("── ") and marker in l] + if not idx: + return + body = [] + for l in lines[idx[0] + 1:]: + if l.strip().startswith("── ") and "(mode" in l: + break + body.append(l[2:] if l.startswith(" ") else l) + while body and (body[-1].strip() == "" or body[-1][:1] in "╭│╰"): + body.pop() + open(os.path.join(d, out), "w").write("\n".join(body) + "\n") +extract("/.env", ".env") +extract("docker-compose.yml", "docker-compose.yml") +PY + [[ -f "$dir/docker-compose.yml" ]] || fail "--dry-run printed no compose file for: $label" + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + ( cd "$dir" && docker compose config >/dev/null ) || fail "invalid compose file for: $label" + fi + pass "generates a valid stack: $label" +} + +check_shape "defaults" +check_shape "bundled TLS" --tls caddy --host warmbly.example.com +check_shape "core only" --components core +check_shape "named volumes" --data-root volumes +check_shape "external stores" --database-url postgres://u:p@db:5432/w --redis-url redis://cache:6379 --blobs s3 + +# Nothing drawn inside a redraw loop may be wider than the terminal. A wrapped +# line is two physical rows, every cursor-up counts logical ones, and the menu +# then draws over itself and over whatever was on screen before it. This is the +# regression that check exists for. +if command -v python3 >/dev/null 2>&1; then + python3 - "$SCRIPT" <<'PYEOF' || fail "the installer drew past the terminal width" +import fcntl, os, pty, re, select, struct, sys, termios, time + +script = sys.argv[1] +failures = [] +for cols in (80, 100): + master, slave = pty.openpty() + fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 45, cols, 0, 0)) + pid = os.fork() + if pid == 0: + os.setsid() + fcntl.ioctl(slave, termios.TIOCSCTTY, 0) + os.dup2(slave, 0); os.dup2(slave, 1); os.dup2(slave, 2) + os.close(master); os.close(slave) + os.environ["TERM"] = "xterm-256color" + os.environ["WARMBLY_DEMO_FAST"] = "1" + os.execvp("sh", ["sh", script, "--demo"]) + os._exit(1) + os.close(slave) + buf = b"" + start = last = time.time() + sent = 0 + while time.time() - start < 90: + r, _, _ = select.select([master], [], [], 0.25) + if r: + try: + chunk = os.read(master, 65536) + except OSError: + break + if not chunk: + break + buf += chunk + last = time.time() + elif time.time() - last > 0.4 and sent < 80: + tail = re.sub(r"\x1b\[[0-9;?]*[a-zA-Z]", "", buf.decode("utf-8", "replace"))[-400:] + os.write(master, b"copied\r" if "Type 'copied'" in tail else b"\r") + sent += 1 + last = time.time() + if b"That was the demo" in buf: + break + for fn in (lambda: os.close(master), lambda: os.waitpid(pid, 0)): + try: + fn() + except OSError: + pass + text = buf.decode("utf-8", "replace") + if "command not found" in text or "syntax error" in text: + failures.append(f"{cols} columns: the run produced shell errors") + if "That was the demo" not in text: + failures.append(f"{cols} columns: the demo did not reach its end") + plain = re.sub(r"\x1b\[[0-9;?]*[a-zA-Z]", "", text) + over = [l for l in plain.replace("\r", "\n").split("\n") if len(l) > cols] + if over: + failures.append(f"{cols} columns: {len(over)} line(s) too wide, first: {over[0][:cols + 20]!r}") + +for f in failures: + print(" " + f, file=sys.stderr) +sys.exit(1 if failures else 0) +PYEOF + pass "draws inside the terminal at 80 and 100 columns" +else + echo "· python3 not installed; the width check was skipped" +fi + +# The checksum is the whole answer to "why would I pipe this into a shell", so +# a stale one is a failure, not a warning. +if [[ ! -f $SUMFILE ]]; then + fail "$SUMFILE is missing. Regenerate it with: make installer-sha" +fi +expected=$(awk '{print $1}' "$SUMFILE") +actual=$(sha256sum "$SCRIPT" | awk '{print $1}') +if [[ $expected != "$actual" ]]; then + fail "$SUMFILE is stale. + published $expected + actual $actual + Regenerate it with: make installer-sha" +fi +pass "published checksum matches" + +printf '\n\033[32mThe installer is good.\033[0m\n' diff --git a/site/README.md b/site/README.md index a44fa923..920ce2d7 100644 --- a/site/README.md +++ b/site/README.md @@ -16,6 +16,32 @@ From the repo root you can also use `make site`, which is a shortcut for `cd site && pnpm dev`. `make app` does **not** start this site — it lives outside the docker compose stack and ships on its own cadence. +## The installer + +`public/install.sh` is the one-command self-host installer, served verbatim at +`https://warmbly.com/install.sh`. It is a static asset, so what ships is exactly +what a `curl | sh` executes. + +Two things the host has to get right: + +- serve it as `text/plain` (or `application/x-sh`), never `text/html`, so + reading it in a browser works and nothing tries to render it +- serve `public/install.sh.sha256` alongside it, unmodified + +Editing it means regenerating the checksum, and CI fails when the two disagree: + +```sh +make installer-sha # regenerate site/public/install.sh.sha256 +make installer-check # everything CI runs: POSIX parse, shellcheck, dry runs +make installer-demo # walk the wizard; installs nothing, needs no Docker +``` + +`make installer-demo` is the fastest way to see a UI change: it runs the real +questions and the real review with the pull, the container creation and the +health wait played, and writes nothing anywhere. It takes about half a minute; +`WARMBLY_DEMO_FAST=1 make installer-demo` cuts that to under ten seconds while +you iterate. + ## Layout ``` diff --git a/site/public/install.sh b/site/public/install.sh new file mode 100644 index 00000000..0d723039 --- /dev/null +++ b/site/public/install.sh @@ -0,0 +1,3071 @@ +#!/bin/sh +# +# curl -fsSL https://warmbly.com/install.sh | sh +# +# Installs Warmbly on this machine: pull the published release images, write a +# real .env and a compose file, start the stack, print the link that claims it. +# No git clone, no Go, Rust, Node or Elixir toolchain, nothing compiled here. +# +# sh install.sh --wizard ask where the data lives, what is kept and for +# how long, and how it is backed up +# sh install.sh --dry-run print the exact files it would write, touch nothing +# sh install.sh --help every flag, and the environment variable for each +# +# On reading this before running it: that is the right instinct, and it is why +# the checksum is published next to it. The short version of what it does: +# +# * checks for docker, and offers to install it rather than doing so quietly +# * creates one directory (default /opt/warmbly) and writes only inside it +# * generates five secrets with openssl, writes them 0600, prints the two +# that are unrecoverable and waits for you to say you have copied them +# * resolves the newest release once and PINS it in .env, so every later +# `docker compose up` in that directory is the same version. Never :latest +# * runs `docker compose pull` and `docker compose up -d` +# +# It asks for sudo only for the steps that need it, names them when it does, +# and never wants to be piped into sudo itself. Re-running it reconfigures in +# place: it does not regenerate secrets, does not touch an existing data root, +# and refuses a directory it did not create unless you pass --force. +# +# Verify before running, if you would rather: +# +# curl -fsSLO https://warmbly.com/install.sh +# curl -fsSLO https://warmbly.com/install.sh.sha256 +# sha256sum -c install.sh.sha256 +# less install.sh && sh install.sh +# +# https://docs.warmbly.com/development/install/ + +set -eu + +# ───────────────────────────────────────────────────────────────────────── +# Constants +# ───────────────────────────────────────────────────────────────────────── + +SCRIPT_NAME="Warmbly installer" +REPO="warmbly/warmbly" +DEFAULT_REGISTRY="ghcr.io/warmbly/warmbly" +DEFAULT_DIR="/opt/warmbly" +DOCS="https://docs.warmbly.com" +RELEASES_API="https://api.github.com/repos/${REPO}/releases" + +# The marker file that says this directory is ours. Its absence in a non-empty +# directory is what --force overrides, so the installer can never take over a +# path that belongs to something else by accident. +MARKER=".warmbly-install" + +# Ports the stack publishes, in the order they are checked for collisions. +# 3000 is the one that actually collides: half of self-hosted software wants it. +PORT_BACKEND=8080 +PORT_WEB=5173 +PORT_ADMIN=5174 +PORT_TRACKING=3000 +PORT_REALTIME=4000 +PORT_FORMS=8090 + +# ───────────────────────────────────────────────────────────────────────── +# Answers. Every one is a flag, an environment variable and a wizard question, +# and the defaults here are exactly what the fast path installs. +# ───────────────────────────────────────────────────────────────────────── + +DIR="${WARMBLY_DIR:-$DEFAULT_DIR}" +HOSTNAME_ANSWER="${WARMBLY_HOST:-localhost}" +TLS="${WARMBLY_TLS:-none}" +DATA_ROOT="${WARMBLY_DATA_ROOT:-}" # empty = /data; "volumes" = docker named volumes +BLOBS="${WARMBLY_BLOBS:-filesystem}" # filesystem | s3 +COMPONENTS="${WARMBLY_COMPONENTS:-full}" # core | full +VERSION="${WARMBLY_VERSION:-}" # empty = resolve the newest release +CHANNEL="${WARMBLY_CHANNEL:-stable}" +REGISTRY="${WARMBLY_IMAGE_PREFIX:-$DEFAULT_REGISTRY}" +EXTERNAL_DB="${WARMBLY_DATABASE_URL:-}" +EXTERNAL_REDIS="${WARMBLY_REDIS_URL:-}" +MAIL_MODE="${WARMBLY_MAIL:-log}" # log | smtp +REGISTRATION="${WARMBLY_REGISTRATION:-invite_only}" +UPDATE_CHECK="${WARMBLY_UPDATE_CHECK:-true}" +BOOTSTRAP_EMAIL="${WARMBLY_BOOTSTRAP_EMAIL:-}" +BOOTSTRAP_HASH="${WARMBLY_BOOTSTRAP_PASSWORD_HASH:-}" +BACKUP_DIR="${WARMBLY_BACKUP_DIR:-}" # empty = no scheduled backup +BACKUP_KEEP="${WARMBLY_BACKUP_KEEP:-14}" +BACKUP_SCHEDULE="${WARMBLY_BACKUP_SCHEDULE:-daily}" +BACKUP_KEYS="${WARMBLY_BACKUP_KEYS:-true}" +BACKUP_S3="${WARMBLY_BACKUP_S3:-}" + +# Retention and sync, the data-control answers. They are written into .env as +# one WARMBLY_SETTINGS_BOOTSTRAP document, applied on first boot and editable +# in Instance > Instance settings from then on. +SYNC_BACKFILL_DAYS="${WARMBLY_SYNC_BACKFILL_DAYS:-90}" +SYNC_BACKFILL_MESSAGES="${WARMBLY_SYNC_BACKFILL_MESSAGES:-5000}" +SYNC_DAILY_MAILBOX="${WARMBLY_SYNC_DAILY_MAILBOX:-2000}" +SYNC_DAILY_ORG="${WARMBLY_SYNC_DAILY_ORG:-25000}" +RET_ENGAGEMENT="${WARMBLY_RETENTION_ENGAGEMENT_DAYS:-365}" +RET_FORMS="${WARMBLY_RETENTION_FORM_DAYS:-180}" +RET_AUDIT="${WARMBLY_RETENTION_AUDIT_DAYS:-90}" + +# S3 blob answers, only read when BLOBS=s3. +S3_BUCKET="${WARMBLY_S3_BUCKET:-}" +S3_ENDPOINT="${WARMBLY_S3_ENDPOINT:-}" +S3_REGION="${WARMBLY_S3_REGION:-auto}" +S3_KEY="${WARMBLY_S3_ACCESS_KEY_ID:-}" +S3_SECRET="${WARMBLY_S3_SECRET_ACCESS_KEY:-}" + +# SMTP answers, only read when MAIL_MODE=smtp. +SMTP_HOST=""; SMTP_PORT=""; SMTP_USER=""; SMTP_PASS=""; SMTP_SECURITY="starttls" +SMTP_FROM="" + +# Reverse-proxy answer, only read when TLS=proxy. +PROXY_CIDRS="${WARMBLY_TRUSTED_PROXIES:-172.16.0.0/12,10.0.0.0/8,192.168.0.0/16}" + +# ───────────────────────────────────────────────────────────────────────── +# Modes +# ───────────────────────────────────────────────────────────────────────── + +MODE=install # install | uninstall +WIZARD=0 +DEMO=0 +ASSUME_YES=0 +DRY_RUN=0 +PRINT_ENV=0 +FORCE=0 +PURGE_DATA=0 +USE_COLOR=1 +# Appending is the default. The wizard reads as a transcript, nothing above it +# is touched, and there is no way for a redraw to land on top of something +# else. --clear opts into the full-screen version. +USE_CLEAR=0 +INTERACTIVE=0 +TTY=/dev/tty + +# Filled in as the run proceeds. +RESOLVED_TAG="" +COMPOSE="docker compose" +SUDO="" +EXISTING=0 + +# The palette starts empty so that anything reachable before setup_term (--help, +# an unknown flag) still prints under set -u. setup_term fills it in. +ESC=""; R=""; B=""; DIM=""; SKY=""; GREEN=""; AMBER="" +RED=""; GREY=""; WHITE=""; HIDE=""; SHOW="" +COLS=80; WIDTH=76 + +# ───────────────────────────────────────────────────────────────────────── +# Terminal: colour, cursor, and the drawing primitives everything else uses. +# +# Every escape sequence goes through these, so NO_COLOR, a pipe, a dumb +# terminal and --no-color all degrade to plain text in one place instead of +# leaking half-rendered ANSI into a log file. +# ───────────────────────────────────────────────────────────────────────── + +setup_term() { + # A wizard needs a keyboard. Piped into sh, stdin is the script itself, so + # the terminal is reopened explicitly; without one, only the answered + # (flag or environment) path can run. + if [ -t 0 ] && [ -t 1 ]; then + INTERACTIVE=1 + TTY=/dev/stdin + elif [ -c /dev/tty ] && ( exec 9<>/dev/tty ) 2>/dev/null && [ -t 1 ]; then + INTERACTIVE=1 + TTY=/dev/tty + fi + + if [ -n "${NO_COLOR:-}" ] || [ "${TERM:-dumb}" = "dumb" ] || [ ! -t 1 ]; then + USE_COLOR=0 + fi + if [ "$USE_COLOR" = 1 ]; then + ESC=$(printf '\033') + R="${ESC}[0m"; B="${ESC}[1m"; DIM="${ESC}[2m" + SKY="${ESC}[38;5;39m" + GREEN="${ESC}[38;5;42m"; AMBER="${ESC}[38;5;214m"; RED="${ESC}[38;5;203m" + GREY="${ESC}[38;5;245m"; WHITE="${ESC}[38;5;255m" + HIDE="${ESC}[?25l"; SHOW="${ESC}[?25h" + else + ESC=""; R=""; B=""; DIM=""; SKY=""; GREEN=""; AMBER="" + RED=""; GREY=""; WHITE=""; HIDE=""; SHOW="" + USE_CLEAR=0 + fi + + # Clearing a terminal that is not ours to clear is never right. + [ "$INTERACTIVE" = 1 ] || USE_CLEAR=0 + + # Width. Everything drawn inside a redraw loop is fitted to this, so + # getting it wrong is what makes a menu draw over itself. Three sources, + # because tput is missing on a minimal image and stty needs the terminal: + # an exported COLUMNS wins, since that is how a caller says so explicitly. + COLS="" + case "${COLUMNS:-}" in ''|*[!0-9]*) ;; *) COLS=$COLUMNS ;; esac + if [ -z "$COLS" ] && have tput; then + COLS=$(tput cols 2>/dev/null || true) + fi + if [ -z "$COLS" ] && [ "$INTERACTIVE" = 1 ]; then + COLS=$(stty size <"$TTY" 2>/dev/null | cut -d' ' -f2 || true) + fi + case "$COLS" in ''|*[!0-9]*) COLS=80 ;; esac + # Clamped: a 300 column terminal should not get a 300 column rule, and + # below the floor the boxes stop being boxes. + [ "$COLS" -lt 44 ] && COLS=44 + [ "$COLS" -gt 92 ] && COLS=92 + WIDTH=$((COLS - 4)) +} + +say() { printf '%s\n' "$*"; } +out() { printf '%b\n' "$*"; } +outn() { printf '%b' "$*"; } + +# Restore the cursor whatever happens: a Ctrl-C inside a spinner would +# otherwise leave the operator's terminal without one. +cleanup_term() { + [ "$USE_COLOR" = 1 ] && printf '%b' "$SHOW" + stty_sane +} +stty_sane() { + [ "$INTERACTIVE" = 1 ] || return 0 + stty sane <"$TTY" 2>/dev/null || true +} +trap 'cleanup_term' EXIT +trap 'cleanup_term; printf "\n"; die "cancelled"' INT + +# clear_screen is opt-in (--clear). Wiping someone's terminal is a thing to be +# asked for, not assumed, and a clear that the terminal declines leaves the +# wizard drawing over whatever was already there. ESC[3J is deliberately not +# sent: it erases the scrollback buffer, and the conversation above this +# command is not ours to delete. +clear_screen() { + [ "$USE_CLEAR" = 1 ] || return 0 + printf '%b' "${ESC}[H${ESC}[2J" +} + +repeat_char() { + # $1 char, $2 count. printf pads with spaces and sed swaps them for the + # character: no loop and no fork per character, which is what keeps the + # redraws smooth. sed rather than tr because tr maps byte to byte and every + # box-drawing character here is three bytes of UTF-8. + _n=$2 + [ "$_n" -lt 1 ] && { printf ''; return; } + printf "%${_n}s" "" | sed "s/ /$1/g" +} + +rule() { out "${DIM}$(repeat_char '─' "$WIDTH")${R}"; } + +# fit truncates plain text to n columns. +# +# It is what keeps the in-place redraws honest: a line longer than the terminal +# wraps onto a second physical row, and every cursor-up in this script counts +# LOGICAL rows, so one long option hint is enough to make a menu redraw over +# itself and over whatever was on screen before it. Applied to everything drawn +# inside a loop; prose printed once is free to wrap. +fit() { + _t=$1; _w=$2 + [ "$_w" -lt 12 ] && _w=12 + if [ "${#_t}" -le "$_w" ]; then + printf '%s' "$_t" + return 0 + fi + printf '%.*s...' "$((_w - 3))" "$_t" +} + +# A rounded box. Content comes on stdin, one line per row, already coloured. +# The border colour is $1. +box() { + _c="${1:-$DIM}" + out "${_c}╭$(repeat_char '─' $((WIDTH - 2)))╮${R}" + while IFS= read -r _line; do + _plain=$(strip_ansi "$_line") + # A content line wider than the box turns the border into a ragged + # edge. Truncating costs the line its colour, which is a better trade + # than a box that is not one. + if [ "${#_plain}" -gt $((WIDTH - 4)) ]; then + _line=$(fit "$_plain" $((WIDTH - 4))) + _plain=$_line + fi + _pad=$((WIDTH - 4 - ${#_plain})) + [ "$_pad" -lt 0 ] && _pad=0 + printf '%b %s%s %b\n' "${_c}│${R}" "$_line" "$(repeat_char ' ' "$_pad")" "${_c}│${R}" + done + out "${_c}╰$(repeat_char '─' $((WIDTH - 2)))╯${R}" +} + +# strip_ansi is what makes the borders line up: the padding is computed on the +# printable text, not on the byte length of a coloured string. ${#} counts +# bytes in a POSIX shell, so box CONTENT is kept ASCII; the border itself is +# printed rather than measured, and may be anything. +strip_ansi() { + printf '%s' "$1" | sed "s/$(printf '\033')\[[0-9;]*m//g" +} + +# ───────────────────────────────────────────────────────────────────────── +# The wordmark. Drawn once at the top of the run, one row at a time so it +# arrives rather than appears. Falls back to a single bold line when the +# terminal is not ours to animate. +# ───────────────────────────────────────────────────────────────────────── + +banner() { + if [ "$USE_COLOR" = 0 ] || [ "$COLS" -lt 60 ]; then + # The block wordmark is 56 columns wide, so on anything narrower it + # wraps into noise. One line says the same thing. + say "" + out " ${B}${SKY}WARMBLY${R} ${DIM}installer${R}" + say "" + return + fi + printf '%b' "$HIDE" + say "" + _i=0 + printf '%s\n' \ +"██ ██ █████ ██████ ███ ███ ██████ ██ ██ ██" \ +"██ █ ██ ██ ██ ██ ██ ████ ████ ██ ██ ██ ██ ██" \ +"██ ███ ██ ███████ ██████ ██ ████ ██ ██████ ██ ███ " \ +"████ ████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ " \ +" ██ ██ ██ ██ ██ ██ ██ ██ ██████ ██████ ██ " \ + | while IFS= read -r _row; do + _i=$((_i + 1)) + case "$_i" in + 1|2) _tone="${ESC}[38;5;33m" ;; + 3) _tone="${ESC}[38;5;39m" ;; + *) _tone="${ESC}[38;5;74m" ;; + esac + printf ' %b%s%b\n' "$_tone" "$_row" "$R" + nap 0.03 + done + printf ' %b%s%b\n\n' "$DIM" "email warmup and cold outreach, on your own machine" "$R" + printf '%b' "$SHOW" +} + +# nap sleeps a fraction of a second where the shell can, and not at all where +# it cannot. Animation is a nicety; a busybox sleep that rejects "0.03" must +# not turn into an error under set -e. +nap() { + [ "$USE_COLOR" = 1 ] || return 0 + sleep "$1" 2>/dev/null || true +} + +# ───────────────────────────────────────────────────────────────────────── +# Messages +# ───────────────────────────────────────────────────────────────────────── + +info() { out " ${SKY}·${R} $*"; } +ok() { out " ${GREEN}✓${R} $*"; } +warn() { out " ${AMBER}!${R} $*"; } +# note is the explanatory prose under a question. It is wrapped to the +# terminal rather than hand-wrapped to 72 columns, because the hand-wrapped +# version is ragged on anything narrower and this is the only text in the +# script long enough to care. +note() { + if [ "$COLS" -ge 78 ] || ! have fold; then + out " ${DIM}$*${R}" + return 0 + fi + printf '%s\n' "$*" | fold -s -w $((COLS - 6)) | while IFS= read -r _l; do + out " ${DIM}${_l}${R}" + done +} +step() { out "\n ${B}$*${R}"; } + +die() { + cleanup_term + out "\n ${RED}✗${R} ${B}$*${R}\n" >&2 + exit 1 +} + +# fail_with prints the error and, under it, what to do about it. An installer +# that stops without a next step is a support ticket. +fail_with() { + _msg=$1; shift + cleanup_term + out "\n ${RED}✗${R} ${B}${_msg}${R}" >&2 + for _l in "$@"; do out " ${DIM}${_l}${R}" >&2; done + out "" >&2 + exit 1 +} + +# ───────────────────────────────────────────────────────────────────────── +# The stepper: where the wizard is, drawn at the top of every question. +# ───────────────────────────────────────────────────────────────────────── + +STEP_TITLES="Where it lives|How it is reached|Where the data sits|Keys and secrets|What is kept|Backups|Who gets in|Footprint|Review" +STEP_TOTAL=9 +STEP_CURRENT=0 + +stepper() { + STEP_CURRENT=$1 + if [ "$USE_CLEAR" = 1 ]; then + clear_screen + else + # Appending, so the steps need their own separator or they run into + # each other and into whatever was on screen before the installer. + say "" + rule + fi + if [ "$USE_COLOR" = 0 ]; then + say "" + say "Step ${STEP_CURRENT} of ${STEP_TOTAL}: $(step_title "$STEP_CURRENT")" + say "" + return + fi + say "" + _line=" " + _i=1 + while [ "$_i" -le "$STEP_TOTAL" ]; do + if [ "$_i" -lt "$STEP_CURRENT" ]; then + _line="${_line}${GREEN}●${R}" + elif [ "$_i" -eq "$STEP_CURRENT" ]; then + _line="${_line}${SKY}◉${R}" + else + _line="${_line}${DIM}○${R}" + fi + [ "$_i" -lt "$STEP_TOTAL" ] && _line="${_line}${DIM}──${R}" + _i=$((_i + 1)) + done + out "$_line ${DIM}step ${STEP_CURRENT}/${STEP_TOTAL}${R}" + out " ${B}${WHITE}$(step_title "$STEP_CURRENT")${R}" + say "" +} + +step_title() { + printf '%s' "$STEP_TITLES" | cut -d'|' -f"$1" +} + +# ───────────────────────────────────────────────────────────────────────── +# Spinner. Runs a command in the background and animates until it finishes, +# then leaves one settled line behind: a tick, or a cross and the log. +# ───────────────────────────────────────────────────────────────────────── + +SPIN_FRAMES="⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" + +# spin