Merge remote-tracking branch 'origin/main' into fix/self-hosted-unsubscribe-domain

This commit is contained in:
Matthew Meszaros
2026-09-09 08:24:04 -07:00
137 changed files with 7919 additions and 10114 deletions
+2 -4
View File
@@ -463,7 +463,7 @@ BILLING_PROVIDER=none
# Workers hold no database. They reach organization keys over the backend's
# internal API. An unset URL or token lets a worker start, subscribe and NEVER
# register, with no log line. WORKER_BIND_IP, WORKER_PUBLIC_IP and
# WORKER_INSTALLER_PATH are not forwarded by the shipped docker-compose.yml.
# are not forwarded by the shipped docker-compose.yml.
# ENCRYPTED_KEYS_PROVIDER=http
# ENCRYPTED_KEYS_BACKEND_URL=http://backend:8080
# ENCRYPTED_KEYS_WORKER_TOKEN= # same value as INTERNAL_API_TOKEN
@@ -477,14 +477,12 @@ BILLING_PROVIDER=none
# WORKER_STATE_DIR=/data/state
# WORKER_BIND_IP=
# WORKER_PUBLIC_IP=
# WORKER_TIER=free # free | premium | dedicated
# WORKER_EGRESS_KIND=
# WARMBLY_NODE_REGION= # free-form egress location label, e.g. eu-central
# MAIL_TLS_INSECURE=false # skips cert verification on mailbox connections
#
# Set on the BACKEND: the image the remote installer pulls. The built-in default
# does not match what CI publishes.
# WORKER_IMAGE=ghcr.io/<owner>/warmbly/worker:prod
# WORKER_INSTALLER_PATH=
# === Updates ==================================================================
+3
View File
@@ -47,6 +47,9 @@ jobs:
- 'go.sum'
- 'internal/**'
- 'cmd/**'
# make lint runs the join-script checks, so a change to the
# checker itself has to trigger the job that runs it.
- 'scripts/check-join-script.sh'
migrations:
- 'internal/infrastructure/db/migrations/**'
- 'scripts/check-migrations.sh'
+74 -17
View File
@@ -186,7 +186,7 @@ Infra runs in docker; the Go services and frontends run natively on the host for
- `make dev` — the one-command stack: brings up the docker infra and waits for postgres, applies migrations, loads seed fixtures (skip with `SEED=false`), installs web + admin deps on first run, starts realtime and tracking as containers, then runs backend + forms + consumer + worker + dashboard + admin in one terminal. Login: dev@warmbly.com / password123, with the emailed login code in Mailpit at http://localhost:18025. Ctrl-C stops the app; infra stays up.
- `make infra` — start the backing services in docker (postgres, redis, nats, mailpit). Run once; leave running. Kafka, Schema Registry, localstack, cloud-tasks, and stripe-mock are gone; the stack is no-cloud by default (NATS, local KMS, filesystem blobs, in-process tasks).
- `make backend` — run the API natively on `:8080` (applies the embedded migrations on boot against the docker postgres).
- `make consumer` / `make worker` / `make worker-premium` — run those Go services natively, each in its own terminal. Two native workers exist because tier placement is strict: free-trial orgs place onto the free-tier worker (`make worker`), paid orgs onto the premium one (`make worker-premium`). The workers read encrypted DEKs through the backend's `/internal/dek` endpoint (the prod `http` provider, no worker DB), so `make backend` must be running and their `INTERNAL_API_TOKEN` must match (the targets are pre-wired to match).
- `make consumer` / `make worker` — run those Go services natively, each in its own terminal. Both register themselves as fleet nodes on their first heartbeat, so they show up in `warmblyctl fleet list` without any enrolment step in dev. Workers are interchangeable, so one is enough; run a second `make worker WORKER_ID=<uuid>` in another terminal when you want to watch placement spread mailboxes across a fleet. The workers read encrypted DEKs through the backend's `/internal/dek` endpoint (the prod `http` provider, no worker DB), so `make backend` must be running and their `INTERNAL_API_TOKEN` must match (the targets are pre-wired to match).
- `make run` — backend + forms + consumer + worker together in one terminal (Ctrl-C stops all).
- `make forms` — the public forms service natively on `:8090` (`cmd/forms`): builds the `forms/` TanStack app, then serves it plus the embed loader and public submissions. No database; it resolves forms and forwards submissions through the backend's internal API, so `make backend` must be running and `INTERNAL_API_TOKEN` must match (pre-wired). The backend's `FORMS_DOMAIN=localhost:8090` makes dashboard share links point at it; `make forms FORMS_PORT=8091` (matched on `make backend`) moves it when worktrees share the machine. `make forms-web` runs the Vite dev server (:5175) for iterating on the app itself.
- `make sandbox` — fully working demo environment: seeds the "Sunrise Labs" showcase org (live mailboxes: SMTP -> mailpit, IMAP -> dovecot, credentials sealed with `CREDENTIALS_ENCRYPTION_KEY`) and runs the simulator that plays the internet (delivers mail into dovecot inboxes, opens pixels, clicks tracked links, replies as contacts). Needs `make run` + `make tracking` alongside. Docs: `docs/content/docs/development/sandbox.mdx`.
@@ -294,26 +294,83 @@ API keys with the `REALTIME_SUBSCRIBE` permission (bit 11) can connect to the sa
Workers are intended to run distributed across many machines, with one worker process per machine.
That layout matters because it lets the system spread sending activity across different machine-level network identities and IP addresses instead of concentrating traffic through a single sender runtime.
**There is one kind of worker.** No tier, no type, no risk pool, no egress category. You stand a worker up, it heartbeats, and the control plane decides what runs on it. The only thing an operator may set is an optional free-form `WORKER_REGION` label, and leaving it blank is fine.
Do not reintroduce a worker category. The four that used to exist (`free_tier`, `worker_type`, `risk_pool`, `egress_kind`) were removed in migration `000140` because they all rested on a premise that is false for this architecture: that the worker's IP is the sending identity.
It is not. A worker never talks to a recipient's MX. It authenticates to the customer's own mailbox provider, and that provider delivers from its own outbound pool. So:
- **the worker IP is invisible to recipient spam filtering.** Google strips the submitting client's IP; Microsoft dropped `X-Originating-IP` years ago. A spam-prone mailbox therefore cannot contaminate a healthy neighbour on the same machine, which is why hard risk segregation of workers bought nothing
- **the worker IP is very visible to the mailbox provider**, where it drives sign-in risk challenges, per-IP auth throttles (`454 4.7.0`) and per-IP rate limits (`421 4.7.28`). Exchange Online also caps SMTP AUTH at ~3 concurrent connections and ~30 msg/min per mailbox, and IMAP at ~8 concurrent sessions
The practical inversion: **IP stability per mailbox beats IP diversity.** Moving a mailbox changes the client address its provider sees and buys a security challenge for nothing, so a migration is a cost, not a win. A fleet where nothing rotates is a healthy fleet.
In production, workers are treated as individually addressable executors:
- each worker has its own `worker_id`
- email accounts are assigned to a specific worker
- worker events are delivered through worker-specific Kafka topics
- the platform can rebalance or migrate accounts between workers
- the platform can rebalance or migrate accounts between workers, reluctantly
This repo already models three worker modes:
Placement is a score, never a filter (`internal/app/worker/placement.go`). Hard constraints cover only whether the work can be done: heartbeating, health in `healthy`/`watch`, and enough capacity headroom for the mailbox's weight. Everything else is a preference term: capacity headroom, incumbency (weighted highest), region match, tenant blast radius, per-provider crowding on one address, and foreign tenants for orgs entitled to isolated egress.
- shared free-tier workers
- shared premium workers
- dedicated workers assigned to a single paying organization
Capacity is one number for every worker in cold-mailbox equivalents, because each mailbox declares its own cost through `MailboxWeight`: `smtp_imap` = 1.0, `gmail`/`outlook` = 0.05, warmup-only = 0.4. Those are the `email_provider` enum values as stored; do not invent provider strings for them.
Rotation is gated separately (`internal/app/worker/rotation.go`) and is deliberately reluctant:
| Urgency | Trigger | Residency floor | Destination bar |
|---|---|---|---|
| Immediate | worker inactive, not heartbeating, blocked, quarantined | none | anything eligible |
| Elevated | worker throttled | 6h | anything eligible |
| Opportunistic | worker over 85% utilization, isolated-egress drift | 72h | must beat the incumbent by `RotationMinScoreGain` |
Isolated egress (the entitlement `plan.IsolatedEgress()`, still stored in `plans.dedicated_workers`) binds an org to a worker through `dedicated_worker_assignments`. It is a strong placement preference, not a pin: the worker carries no marking, so a reserved worker going down never strands the customer.
The relevant code paths are in:
- `internal/app/worker/assignment.go`
- `internal/repository/pg_worker.go`
- `internal/infrastructure/db/migrations/000015_worker_tiers.up.sql`
- `internal/app/fleetnode/service.go` (enrolment, heartbeat, desired version)
- `internal/app/worker/placement.go` (the score)
- `internal/app/worker/rotation.go` (when a move is allowed)
- `internal/app/worker/assignment.go` (the service that commits placements)
- `internal/app/fleet/rebalance.go` (the rotation loop)
- `internal/repository/pg_worker_placement.go`
- `internal/infrastructure/db/migrations/000140_worker_decategorization.up.sql`
## The Fleet Is Pull-Based
Every Warmbly process that runs on a machine you own is a **node**: `worker` (sends and syncs mail) or `consumer` (processes events). Both share one lifecycle and one registry.
A node joins by running one command with the instance join token, then heartbeats forever. **Nothing is ever pushed to a node.** Everything the control plane wants it to do comes back in the heartbeat reply, which today is exactly one instruction: what version to be running.
Do not reintroduce a push path. Migration `000142` deleted the whole of it — the Hetzner provider, `provisioning_templates`/`_jobs`/`_policy`, `worker_profiles`, `aws_credentials`, the SSH orchestrator and every `workers.ssh_*` column — because onboarding a machine you already own does not need a cloud API or a keypair, and an update does not need someone to shell in and run it.
Shape:
- `fleet_nodes` is the registry every role shares: identity, region, address, version, liveness, resource usage. `workers` is the placement extension and holds only `account_count`, `health_state`, `load_score`; `workers.id` IS the node id, enforced by a foreign key
- a node is created by enrolling, never by an admin form. `EnsureWorkerRow` adds the placement half when a node declares itself a worker
- liveness lives on `fleet_nodes.last_seen_at` and nowhere else. `models.Worker` is a flat view over `workers JOIN fleet_nodes`, so read it through `workerSelect` rather than adding a second source of truth
- `models.NodeLivenessWindow` is the one definition of live. The node paces its own beat at a third of it, from the value the server returns
Auto-update:
- `internal/app/releases` resolves the head of the configured channel from GitHub Releases and writes the tag to `admin_settings` under `fleet.release`. It updates nothing itself
- the heartbeat reply carries `desired_version`; the node writes it to a file and a systemd timer (`warmbly-node-update`, installed by the join script) pulls and restarts. The process being replaced is never the process doing the replacing
- an empty `desired_version` means "no opinion" and must never be read as "downgrade to nothing". A node that cannot be told what to run keeps running what it has
- a per-node `pinned_version` overrides the fleet target, for canarying or holding a machine back
- **the backend is deliberately excluded.** It is what tells everyone else their version; a self-update that goes wrong leaves nothing to recover with
The join script is `internal/api/handler/nodescript/join.sh`, embedded and served at `GET /join.sh` by the instance itself, so a self-hosted fleet never depends on a vendor host and always gets a script matching its backend. There is exactly one copy: do not add a mirror under `scripts/` or `site/public/`. All the POSIX-sh rules for published scripts apply to it (`sh -n`, `shellcheck -s sh`, everything in a function, `main "$@"` last).
Run `make join-check` before pushing a change to it; it is a prerequisite of `make lint`. It asserts on what `join.sh --print-unit` *renders*; an earlier version compared a heredoc copied into the checker itself and stayed green when the original bug was put back. It exists because nothing covered the script and three separate defects shipped into the branch as a result: a systemd unit built with `$(cat ...)`, which systemd never expands, so the machine restart-looped while the script printed "Done"; a missing bind mount, so the node wrote its update target inside the container and auto-update silently never ran; and an env file assembled by picking a multi-line value back out of JSON with sed, which appended a stray fragment. Assert on what the shell *renders*, not on the source text: every one of those parsed fine. The two invariants that leave no trace in the rendered unit (that `main` validates before writing anything, and that `install_units` prepares the blob root) are checked at their call sites instead, matched on the first field, which a mention inside a string or a comment cannot satisfy. That does mean those calls have to stay standalone statements, which `join.sh` notes above each set of asserted calls; a looser regex was tried and turned out to be satisfied by the name appearing inside a `warn` message, which is a far worse failure than a reformat that reports itself. Every assertion there was mutation-tested: the bug it guards was reintroduced and the check was watched to fail.
Two rules that follow from those:
- **systemd runs no shell.** No `$(...)`, no globbing, no word splitting in a unit. A value that has to vary comes from an `EnvironmentFile` as `${VAR}`, which expands to exactly one argument
- **What the node may write and what root reads are different directories.** The container runs as uid 1000; it gets `/var/lib/warmbly/node` and nothing else. `image-ref` lives one level up, root-owned, because systemd feeds it to a root `docker run --network host` and a node that could rewrite it would choose the image root executes
The env the join endpoint hands a node is rendered from the backend's own environment (`nodeEnvKeys` in `internal/api/handler/fleet_nodes.go`). `PRIMARY_DB` is deliberately absent: a worker reaches relational data through the internal API and nothing else, and shipping a DSN here would quietly undo that boundary.
Operator surface: `warmblyctl fleet` (join-token, list, show, remove, pin, version, channel) and the admin panel's Fleet section. There is no install, restart, logs or reboot action anywhere, because nothing reaches into a machine.
## Warmup Pool Model
@@ -430,14 +487,14 @@ Warmup posture:
### Worker-level distribution rule
For shared workers, distribute volume by mailbox budget and IP spread:
Distribute by mailbox budget, not by a per-worker sending target. Note what this rule is and is not for: spreading mailboxes across workers does **not** improve recipient-side deliverability, because the worker is not the sending identity (see Worker Topology). It limits blast radius and keeps any one address from crowding one provider's auth rate limits.
- no shared worker should become a concentration point for a large fraction of total cold-email traffic
- prefer adding more workers and spreading accounts rather than increasing per-worker density
- if one worker holds many active cold mailboxes, keep the total planned worker volume equal to the sum of those mailbox caps, not an independent higher target
- as a conservative planning heuristic, shared workers should usually stay near the equivalent of about `10` actively sending cold mailboxes at default settings, or roughly `500` cold campaign emails/day, unless there is explicit evidence that the worker/IP pool can safely sustain more
- no worker should become a concentration point for a large fraction of one customer's mailboxes, because losing it stops that fraction of their sending
- keep a worker's total planned volume equal to the sum of its mailboxes' caps, not an independent higher target
- avoid piling many mailboxes of the same provider onto one worker; that is the combination that earns a per-IP auth throttle (`providerSoftCap` in `placement.go`)
- prefer adding workers over increasing per-worker density, but do not churn existing mailboxes to achieve it
Dedicated workers may carry higher organization-specific volume, but those increases should come from more healthy mailboxes, not from forcing a small number of inboxes to send too much.
Increases in volume should come from more healthy mailboxes, never from forcing a small number of inboxes to send too much.
### Internet research constraints
@@ -859,7 +916,7 @@ If a new feature requires heavy joins, admin queries, billing checks, or complex
- do not add direct Postgres usage to `cmd/worker` or `internal/app/worker` unless explicitly required
- preserve worker-specific Kafka topic routing
- preserve separation between free, premium, and dedicated worker capacity
- do not reintroduce worker categories; placement is a score over live state
- preserve separation between free and premium warmup pools
- optimize for many-worker deployments, not a single giant worker
- document any change that alters worker assignment, pool membership, or network boundaries
+9 -3
View File
@@ -37,7 +37,7 @@ PROTOC_GEN_GO_GRPC_VERSION ?= v1.6.1
PROTO_DIR := internal/tasks/proto
PROTO_GEN_FILES := $(PROTO_DIR)/tasks.pb.go
.PHONY: poollink-dev poollink-dev-down poollink-dev-reset setup-tools fmt lint check-migrations proto check-proto \
.PHONY: poollink-dev poollink-dev-down poollink-dev-reset setup-tools fmt lint check-migrations join-check proto check-proto \
up upgrade claim doctor cli seed-demo seed seed-plan sandbox sandbox-seed sandbox-simulate reset logs status stop down test-seed \
restart restart-go restart-all infra infra-down app app-down app-logs \
backend forms forms-web consumer worker run dev tracking realtime web \
@@ -82,7 +82,7 @@ cli-check:
fmt:
gofmt -w ./cmd ./internal
lint: check-migrations
lint: check-migrations join-check
./scripts/check-forms-mirror.sh
$(GO_BIN)/golangci-lint run --timeout=5m
@@ -667,7 +667,6 @@ consumer:
worker:
$(WORKER_DEV_ENV) \
WORKER_ID=10c8f5e4-1c39-5b2a-9c8b-3d2f0a8b1a01 \
WORKER_TIER=shared \
ENCRYPTED_KEYS_PROVIDER=http \
ENCRYPTED_KEYS_BACKEND_URL=http://localhost:8080 \
ENCRYPTED_KEYS_WORKER_TOKEN=local-dev-internal-token \
@@ -704,6 +703,13 @@ installer-sha:
installer-check:
@./scripts/check-installer.sh
# The fleet join script is served verbatim from the backend at GET /join.sh and
# is what a stranger pipes into a root shell to add a machine. Nothing covered
# it, and a systemd unit that could never start shipped as a result. Part of
# `make lint`, like check-migrations.
join-check:
@./scripts/check-join-script.sh
# Every published image has to be pullable by a stranger, and nothing else we
# run proves it: a package on GHCR is created private, does not inherit the
# repository's visibility, and no API can change that, so every check that
+1 -1
View File
@@ -1,6 +1,6 @@
// Fleet: placement as the operator sees it. Capacity (per-worker load vs.
// effective capacity with the last hour's outcome counters), the decision log
// the control loops write, and the dedicated worker bindings. The tab lives
// the control loops write, and the isolated-egress reservations. The tab lives
// in ?tab= so links deep-link.
import { useSearchParams } from "react-router-dom";
+2 -2
View File
@@ -24,7 +24,7 @@ import { DataTable, type Column } from "@/components/data/DataTable";
import { useCursorPager } from "@/lib/useCursorPager";
import { emptyRange, rangeActive, rangeWithin, rangeAfter, rangeBefore, type DateRange } from "@/lib/dateRange";
import { searchMailboxes } from "@/lib/api/client/admin/mailboxes";
import { listManagedWorkers } from "@/lib/api/client/admin/workers";
import { listFleetNodes, type FleetNode } from "@/lib/api/client/admin/fleetNodes";
import type { AdminMailboxRow } from "@/lib/api/models/admin";
type StatusFilter = "active" | "inactive" | "all";
@@ -206,7 +206,7 @@ export default function MailboxesPage() {
if (qParam) setStatus("all");
}, [qParam]);
const { data: workersData } = useQuery({ queryKey: ["admin", "workers", "managed"], queryFn: listManagedWorkers, staleTime: 60_000 });
const { data: workersData } = useQuery({ queryKey: ["admin", "workers", "managed"], queryFn: () => listFleetNodes("worker"), staleTime: 60_000 });
const workerOptions = [
{ value: "any", label: "Any worker" },
...(workersData?.data ?? []).map((w) => ({ value: w.id, label: w.name || w.id.slice(0, 8) })),
File diff suppressed because it is too large Load Diff
+196 -350
View File
@@ -1,376 +1,222 @@
// Add a worker.
// Add a machine to the fleet.
//
// Two ways to attach a machine, both ending at the same place:
//
// SSH-managed — we mint a keypair, you paste the public key into the VPS,
// then Test and Install run from here.
// Enrollment — we mint a one-time token, you run one curl on the VPS and
// it configures itself. No inbound SSH needed.
//
// Only shared workers can be created: the backend rejects worker_type
// "dedicated" because dedicated capacity is allocated by the control plane.
// There is no form here worth filling in, because there is nothing to
// configure: you issue a token, run one command on a machine you already own,
// and it appears. Everything it needs — event bus, cache, keys, the version to
// run — is handed to it by the control plane at join time, so the only two
// choices are what the machine does (worker or consumer) and, optionally,
// where it is.
import { useState } from "react";
import { Link, useNavigate } from "react-router-dom";
import { useMemo, useState } from "react";
import { Link } from "react-router-dom";
import { useMutation } from "@tanstack/react-query";
import { toast } from "sonner";
import { ArrowLeft, CheckCircle2, Hammer, PlayCircle, Plug, XCircle } from "lucide-react";
import { ArrowLeft, Check, Copy, KeyRound, Server, Wrench } from "lucide-react";
import { PageHeader } from "@/components/layout/PageHeader";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import { Textarea } from "@/components/ui/textarea";
import { API_URL } from "@/lib/env";
import {
createWorker,
installWorker,
preflightWorker,
testWorker,
} from "@/lib/api/client/admin/workers";
import type { CreateWorkerResponse } from "@/lib/api/models/admin";
import { issueJoinToken, type NodeRole } from "@/lib/api/client/admin/fleetNodes";
type Method = "ssh" | "enroll";
// The instance a node should point at. API_URL carries the /api/v1 prefix the
// client uses; the join command wants the bare origin.
function instanceOrigin(): string {
try {
return new URL(API_URL, window.location.origin).origin;
} catch {
return window.location.origin;
}
}
function CopyButton({ value, label }: { value: string; label: string }) {
const [copied, setCopied] = useState(false);
return (
<Button
size="sm"
variant="outline"
onClick={async () => {
await navigator.clipboard.writeText(value);
setCopied(true);
toast.success(`${label} copied`);
window.setTimeout(() => setCopied(false), 1500);
}}
>
{copied ? <Check className="size-4" /> : <Copy className="size-4" />}
{copied ? "Copied" : "Copy"}
</Button>
);
}
export default function WorkerNewPage() {
const navigate = useNavigate();
const [role, setRole] = useState<NodeRole>("worker");
const [region, setRegion] = useState("");
const [token, setToken] = useState<string | null>(null);
const [method, setMethod] = useState<Method>("ssh");
const [name, setName] = useState("");
const [notes, setNotes] = useState("");
const [freeTier, setFreeTier] = useState(false);
const [host, setHost] = useState("");
const [port, setPort] = useState(22);
const [user, setUser] = useState("root");
const origin = instanceOrigin();
const [preflight, setPreflight] = useState<{
ok: boolean;
latency_ms?: number;
error?: string;
} | null>(null);
const [created, setCreated] = useState<CreateWorkerResponse | null>(null);
const preflightMut = useMutation({
mutationFn: () => preflightWorker(host, port),
onSuccess: (res) => setPreflight(res),
onError: (e: Error) => setPreflight({ ok: false, error: e.message }),
});
const createMut = useMutation({
mutationFn: () =>
createWorker({
name,
notes,
worker_type: "shared",
free_tier: freeTier,
ssh_host: host,
ssh_port: port,
ssh_user: user,
generate_enrollment_token: method === "enroll",
}),
const issue = useMutation({
mutationFn: issueJoinToken,
onSuccess: (res) => {
setCreated(res);
toast.success("Worker created");
setToken(res.token);
toast.success("Join token issued");
},
onError: (e: Error) => toast.error(e.message),
onError: (e: Error) => toast.error(e.message || "Could not issue a token"),
});
const testMut = useMutation({
mutationFn: () => testWorker(created!.id),
onSuccess: (res) =>
res.ok
? toast.success("SSH reachable")
: toast.error(res.error || "SSH unreachable"),
onError: (e: Error) => toast.error(e.message),
});
const installMut = useMutation({
mutationFn: () => installWorker(created!.id),
onSuccess: () => toast.success("Install kicked off"),
onError: (e: Error) => toast.error(e.message),
});
// The enrollment path does not need a reachable host up front, so only the
// SSH path gates on name + host.
const canCreate = name.trim() !== "" && host.trim() !== "";
const enrollCommand = created?.enrollment_token
? `curl -fsSL ${API_URL}/worker-install.sh | sudo bash -s -- \\\n --enroll ${created.enrollment_token} --api-base ${API_URL}`
: "";
const copy = (text: string, what: string) => {
navigator.clipboard.writeText(text);
toast.success(`${what} copied`);
};
if (created) {
return (
<div>
<Link
to="/workers"
className="inline-flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground mb-2"
>
<ArrowLeft className="size-3" />
All workers
</Link>
<PageHeader
title={created.name}
description="Worker created. Finish the handshake on the machine, then install."
/>
{created.enrollment_token ? (
<Card className="mb-4">
<CardHeader>
<CardTitle>Run this on the VPS</CardTitle>
<CardDescription>
One-time token, valid for{" "}
{Math.round((created.enrollment_token_ttl_seconds ?? 7200) / 3600)}{" "}
hours. It is shown once and cannot be retrieved later.
</CardDescription>
</CardHeader>
<CardContent className="pt-0">
<pre className="overflow-x-auto rounded bg-muted p-3 text-xs">
{enrollCommand}
</pre>
<Button
size="sm"
variant="outline"
className="mt-2"
onClick={() => copy(enrollCommand, "Command")}
>
Copy command
</Button>
</CardContent>
</Card>
) : (
<Card className="mb-4">
<CardHeader>
<CardTitle>Add this key to the VPS</CardTitle>
<CardDescription>
Append it to <code>~/.ssh/authorized_keys</code> for{" "}
<code>{created.ssh_user || user}</code>, then Test connection. The
first success pins the host fingerprint.
</CardDescription>
</CardHeader>
<CardContent className="pt-0">
<pre className="overflow-x-auto rounded bg-muted p-3 text-xs">
{created.ssh_public_key}
</pre>
<Button
size="sm"
variant="outline"
className="mt-2"
onClick={() => copy(created.ssh_public_key, "Public key")}
>
Copy public key
</Button>
</CardContent>
</Card>
)}
<div className="flex flex-wrap gap-2">
<Button
size="sm"
variant="outline"
onClick={() => testMut.mutate()}
disabled={testMut.isPending}
>
<PlayCircle className="size-4" />
{testMut.isPending ? "Testing…" : "Test connection"}
</Button>
<Button
size="sm"
variant="outline"
onClick={() => installMut.mutate()}
disabled={installMut.isPending}
>
<Hammer className="size-4" />
{installMut.isPending ? "Installing…" : "Install"}
</Button>
<Button size="sm" onClick={() => navigate(`/workers/${created.id}`)}>
Open worker
</Button>
</div>
</div>
);
}
const command = useMemo(() => {
const t = token ?? "<join-token>";
const parts = [
`curl -fsSL ${origin}/join.sh | sh -s -- \\`,
` --url ${origin} \\`,
` --token ${t} \\`,
` --role ${role}`,
];
if (region.trim()) parts[parts.length - 1] += ` \\`;
if (region.trim()) parts.push(` --region ${region.trim()}`);
return parts.join("\n");
}, [origin, token, role, region]);
return (
<div>
<Link
to="/workers"
className="inline-flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground mb-2"
>
<ArrowLeft className="size-3" />
All workers
</Link>
<div className="space-y-4">
<PageHeader
title="Add worker"
description="Attach a Linux machine you own. Outbound mail still leaves through each mailbox's own provider, so more workers means more parallelism, not more sending IPs."
/>
<div className="grid grid-cols-1 lg:grid-cols-2 gap-3">
<Card>
<CardHeader>
<CardTitle>How should we reach it?</CardTitle>
<CardDescription>
Both paths end with the same worker. Pick enrollment if the machine
cannot accept inbound SSH from here.
</CardDescription>
</CardHeader>
<CardContent className="pt-0 space-y-2">
<MethodOption
selected={method === "ssh"}
onSelect={() => setMethod("ssh")}
title="SSH-managed"
body="We generate a keypair. You paste the public key onto the machine, and every later action (install, restart, logs, reboot) runs from this panel."
/>
<MethodOption
selected={method === "enroll"}
onSelect={() => setMethod("enroll")}
title="Enrollment token"
body="We generate a one-time token. You run a single curl on the machine and it configures itself and starts reporting in."
/>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Machine</CardTitle>
<CardDescription>
The host is stored either way; it identifies the worker and is what SSH
actions dial.
</CardDescription>
</CardHeader>
<CardContent className="pt-0 space-y-3">
<div>
<Label htmlFor="w-name">Name</Label>
<Input
id="w-name"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="eu-west-1"
/>
</div>
<div className="grid grid-cols-3 gap-2">
<div className="col-span-2">
<Label htmlFor="w-host">SSH host</Label>
<Input
id="w-host"
value={host}
onChange={(e) => {
setHost(e.target.value);
setPreflight(null);
}}
placeholder="203.0.113.10"
/>
</div>
<div>
<Label htmlFor="w-port">Port</Label>
<Input
id="w-port"
type="number"
value={port}
onChange={(e) => {
setPort(Number(e.target.value) || 22);
setPreflight(null);
}}
/>
</div>
</div>
<div>
<Label htmlFor="w-user">SSH user</Label>
<Input
id="w-user"
value={user}
onChange={(e) => setUser(e.target.value)}
placeholder="root"
/>
</div>
<div>
<Label htmlFor="w-notes">Notes</Label>
<Textarea
id="w-notes"
value={notes}
onChange={(e) => setNotes(e.target.value)}
placeholder="Provider, region, anything worth remembering."
rows={2}
/>
</div>
<div className="flex items-center justify-between rounded-md border p-3">
<div className="min-w-0 pr-3">
<p className="text-sm font-medium">Free-tier worker</p>
<p className="text-xs text-muted-foreground">
Free-trial organizations place onto free-tier workers; paid
organizations place onto the rest.
</p>
</div>
<Switch checked={freeTier} onCheckedChange={setFreeTier} />
</div>
<div className="flex items-center gap-2">
<Button
size="sm"
variant="outline"
onClick={() => preflightMut.mutate()}
disabled={!host.trim() || preflightMut.isPending}
>
<Plug className="size-4" />
{preflightMut.isPending ? "Checking…" : "Check reachability"}
</Button>
{preflight && (
<span
className={`inline-flex items-center gap-1 text-xs ${preflight.ok ? "text-emerald-600" : "text-red-600"}`}
>
{preflight.ok ? (
<CheckCircle2 className="size-3.5" />
) : (
<XCircle className="size-3.5" />
)}
{preflight.ok
? `Port open${preflight.latency_ms ? ` · ${preflight.latency_ms}ms` : ""}`
: preflight.error || "Unreachable"}
</span>
)}
</div>
</CardContent>
</Card>
</div>
<div className="mt-4 flex gap-2">
<Button onClick={() => createMut.mutate()} disabled={!canCreate || createMut.isPending}>
{createMut.isPending ? "Creating…" : "Create worker"}
title="Add a machine"
description="Run one command on any machine you own. Nothing connects back to it."
>
<Button asChild size="sm" variant="outline">
<Link to="/workers">
<ArrowLeft className="size-4" />
Fleet
</Link>
</Button>
<Button variant="outline" onClick={() => navigate("/workers")}>
Cancel
</Button>
</div>
</PageHeader>
<Card>
<CardHeader>
<CardTitle className="text-base">What should it do?</CardTitle>
<CardDescription>
Both roles enrol, report themselves and stay on the version you choose.
The difference is only what work they pick up.
</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
<div className="grid gap-2 sm:grid-cols-2">
<button
type="button"
onClick={() => setRole("worker")}
className={`rounded-md border p-3 text-left transition ${
role === "worker"
? "border-[var(--admin-accent-strong)] bg-[var(--admin-accent-weak)]"
: "hover:bg-muted/50"
}`}
>
<span className="flex items-center gap-2 text-sm font-medium">
<Server className="size-4" />
Worker
</span>
<span className="mt-1 block text-xs text-muted-foreground">
Connects to customer mailboxes to send and sync. Add these when
capacity runs low.
</span>
</button>
<button
type="button"
onClick={() => setRole("consumer")}
className={`rounded-md border p-3 text-left transition ${
role === "consumer"
? "border-[var(--admin-accent-strong)] bg-[var(--admin-accent-weak)]"
: "hover:bg-muted/50"
}`}
>
<span className="flex items-center gap-2 text-sm font-medium">
<Wrench className="size-4" />
Consumer
</span>
<span className="mt-1 block text-xs text-muted-foreground">
Processes events and keeps platform state current. They share work
automatically, so more of them just works.
</span>
</button>
</div>
{role === "worker" && (
<div className="space-y-1.5">
<Label htmlFor="region">Region (optional)</Label>
<Input
id="region"
value={region}
onChange={(e) => setRegion(e.target.value)}
placeholder="eu-central"
/>
<p className="text-xs text-muted-foreground">
Where this machine egresses from. Placement prefers a worker near
where a mailbox&apos;s provider expects sign-ins, which means fewer
security challenges. Leave it blank and it scores neutral.
</p>
</div>
)}
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle className="text-base">Run this on the machine</CardTitle>
<CardDescription>
Needs Docker, systemd and root. The machine must be able to reach this
instance; nothing needs to reach it.
</CardDescription>
</CardHeader>
<CardContent className="space-y-3">
{!token && (
<Button size="sm" onClick={() => issue.mutate()} disabled={issue.isPending}>
<KeyRound className="size-4" />
{issue.isPending ? "Issuing…" : "Issue a join token"}
</Button>
)}
<div className="relative">
<pre className="overflow-x-auto rounded-md border bg-muted/40 p-3 pr-24 font-mono text-[12px] leading-relaxed">
{command}
</pre>
<div className="absolute right-2 top-2">
<CopyButton value={command} label="Command" />
</div>
</div>
{token ? (
<p className="text-xs text-amber-700">
This token is shown once and is not recoverable. Issuing another one
revokes it; machines that already joined are unaffected.
</p>
) : (
<p className="text-xs text-muted-foreground">
Issue a token to fill in the command. One token can add as many
machines as you like until you replace it.
</p>
)}
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle className="text-base">Then what</CardTitle>
</CardHeader>
<CardContent className="space-y-2 text-[13px] text-muted-foreground">
<p>
The machine appears in the fleet within a minute or two. A worker starts
taking mailboxes on its own; you never assign them by hand.
</p>
<p>
It also keeps itself on whatever version the fleet is set to, so there is
nothing to do when a release lands. Set that under Fleet.
</p>
<p>
Re-running the same command on the same machine re-joins it under the same
identity, keeping its history and its mailboxes.
</p>
</CardContent>
</Card>
</div>
);
}
function MethodOption({
selected,
onSelect,
title,
body,
}: {
selected: boolean;
onSelect: () => void;
title: string;
body: string;
}) {
return (
<button
type="button"
onClick={onSelect}
className={`w-full rounded-md border p-3 text-left transition ${
selected ? "border-primary bg-primary/5" : "hover:bg-muted/50"
}`}
>
<p className="text-sm font-medium">{title}</p>
<p className="mt-1 text-xs text-muted-foreground">{body}</p>
</button>
);
}
+82 -32
View File
@@ -1,8 +1,9 @@
// Workers explorer — the managed-worker control plane as a faceted browser.
// Data is fetched all-at-once from /admin/workers/managed (small N), so search,
// faceting, and sort run client-side; the Explorer rail mirrors the Users /
// Organizations / Mailboxes browsers. SSH lifecycle (install / restart /
// uninstall) lives in each worker's detail page.
// Fleet explorer — every machine running Warmbly, as a faceted browser.
//
// Both roles are here: a worker sends and syncs mail, a consumer processes
// events. They share one lifecycle (enrol, heartbeat, self-update) so they
// share one table. Data is fetched all-at-once (small N), so search, faceting
// and sort run client-side, mirroring the Users / Organizations browsers.
import { useMemo, useState } from "react";
import { useQuery } from "@tanstack/react-query";
@@ -22,8 +23,7 @@ import {
} from "@/components/data/Explorer";
import { DataTable, type Column } from "@/components/data/DataTable";
import { emptyRange, rangeActive, type DateRange } from "@/lib/dateRange";
import { listManagedWorkers } from "@/lib/api/client/admin/workers";
import type { ManagedWorker } from "@/lib/api/models/admin";
import { listFleetNodes, nodeNeedsUpdate, type FleetNode } from "@/lib/api/client/admin/fleetNodes";
const OFFLINE_MS = 5 * 60_000;
const DAY_MS = 24 * 60 * 60 * 1000;
@@ -43,7 +43,7 @@ function inDateRange(iso: string | undefined, r: DateRange): boolean {
type LiveKey = "online" | "stale" | "offline" | "none";
function liveKey(w: ManagedWorker): LiveKey {
function liveKey(w: FleetNode): LiveKey {
if (!w.last_seen_at) return "none";
const age = Date.now() - new Date(w.last_seen_at).getTime();
if (age < 90_000) return "online";
@@ -58,7 +58,7 @@ const LIVE_LABEL: Record<LiveKey, { label: string; cls: string }> = {
none: { label: "no heartbeat", cls: "text-zinc-400" },
};
const columns: Column<ManagedWorker>[] = [
const columns: Column<FleetNode>[] = [
{
id: "name",
header: "Worker",
@@ -75,16 +75,28 @@ const columns: Column<ManagedWorker>[] = [
csv: (w) => w.name || w.id,
},
{
id: "host",
header: "Host",
id: "role",
header: "Role",
sortable: true,
sortKey: "role",
cell: (w) => (
<span className="font-mono text-[11px]">
{w.ssh_user ? `${w.ssh_user}@` : ""}
{w.ssh_host || w.ip_addr}
{w.ssh_port ? `:${w.ssh_port}` : ""}
</span>
<Badge variant="outline" className="text-[10px]">
{w.role}
</Badge>
),
csv: (w) => w.ssh_host || w.ip_addr,
csv: (w) => w.role,
},
{
id: "address",
header: "Address",
cell: (w) => <span className="font-mono text-[11px]">{w.address || "—"}</span>,
csv: (w) => w.address,
},
{
id: "region",
header: "Region",
cell: (w) => <span className="font-mono text-[11px]">{w.region || "—"}</span>,
csv: (w) => w.region,
},
{
id: "live",
@@ -95,12 +107,48 @@ const columns: Column<ManagedWorker>[] = [
},
csv: (w) => LIVE_LABEL[liveKey(w)].label,
},
{ id: "mailboxes", header: "Mailboxes", align: "right", sortable: true, sortKey: "mailboxes", cell: (w) => <span className="tabular-nums">{w.account_count}</span>, csv: (w) => w.account_count },
{
id: "image",
header: "Image",
cell: (w) => (w.image_version ? <span className="font-mono text-xs">{w.image_version}</span> : <span className="text-xs text-muted-foreground"></span>),
csv: (w) => w.image_version || "",
id: "mailboxes",
header: "Mailboxes",
align: "right",
sortable: true,
sortKey: "mailboxes",
cell: (w) =>
w.mailbox_count === undefined ? (
<span className="text-xs text-muted-foreground"></span>
) : (
<span className="tabular-nums">{w.mailbox_count}</span>
),
csv: (w) => w.mailbox_count ?? "",
},
{
// The pending update is shown inline, because "which machines are
// behind" is the question this table exists to answer.
id: "version",
header: "Version",
cell: (w) =>
nodeNeedsUpdate(w) ? (
<span className="font-mono text-xs">
{w.version || "—"}
<span className="text-muted-foreground"> </span>
<span className="text-amber-600">{w.desired_version}</span>
</span>
) : (
<span className="font-mono text-xs">{w.version || "—"}</span>
),
csv: (w) => w.version || "",
},
{
id: "memory",
header: "Memory",
align: "right",
cell: (w) =>
w.usage?.memory_mb === undefined ? (
<span className="text-xs text-muted-foreground"></span>
) : (
<span className="tabular-nums text-xs">{w.usage.memory_mb} MB</span>
),
csv: (w) => w.usage?.memory_mb ?? "",
defaultHidden: true,
},
{
@@ -140,12 +188,14 @@ const columns: Column<ManagedWorker>[] = [
},
];
function compare(a: ManagedWorker, b: ManagedWorker, by: string): number {
function compare(a: FleetNode, b: FleetNode, by: string): number {
switch (by) {
case "name":
return (a.name || a.id).localeCompare(b.name || b.id);
case "role":
return a.role.localeCompare(b.role);
case "mailboxes":
return a.account_count - b.account_count;
return (a.mailbox_count ?? -1) - (b.mailbox_count ?? -1);
case "seen":
return new Date(a.last_seen_at || 0).getTime() - new Date(b.last_seen_at || 0).getTime();
case "created":
@@ -159,7 +209,7 @@ export default function WorkersPage() {
const nav = useNavigate();
const { data, isLoading, error, refetch } = useQuery({
queryKey: ["admin", "workers", "managed"],
queryFn: listManagedWorkers,
queryFn: () => listFleetNodes(),
});
const [query, setQuery] = useState("");
@@ -179,20 +229,20 @@ export default function WorkersPage() {
const q = query.trim().toLowerCase();
if (q) {
all = all.filter((w) =>
`${w.name} ${w.id} ${w.ssh_host ?? ""} ${w.ip_addr} ${w.image_version ?? ""} ${(w.tags || []).join(" ")}`
`${w.name} ${w.id} ${w.role} ${w.address} ${w.region} ${w.version} ${(w.tags || []).join(" ")}`
.toLowerCase()
.includes(q),
);
}
if (live) all = all.filter((w) => liveKey(w) === live);
if (activeOnly) all = all.filter((w) => w.active);
if (hasMailboxes) all = all.filter((w) => w.account_count > 0);
if (hasMailboxes) all = all.filter((w) => (w.mailbox_count ?? 0) > 0);
if (hasError) all = all.filter((w) => !!w.last_error && w.last_error !== "");
if (hasTags) all = all.filter((w) => (w.tags?.length ?? 0) > 0);
if (mbMin !== undefined) all = all.filter((w) => w.account_count >= mbMin);
if (mbMax !== undefined) all = all.filter((w) => w.account_count <= mbMax);
if (mbMin !== undefined) all = all.filter((w) => (w.mailbox_count ?? 0) >= mbMin);
if (mbMax !== undefined) all = all.filter((w) => (w.mailbox_count ?? 0) <= mbMax);
if (rangeActive(created)) all = all.filter((w) => inDateRange(w.created_at, created));
if (rangeActive(lastSeen)) all = all.filter((w) => inDateRange(w.last_seen_at, lastSeen));
if (rangeActive(lastSeen)) all = all.filter((w) => inDateRange(w.last_seen_at ?? undefined, lastSeen));
if (sort.by) {
all = [...all].sort((a, b) => compare(a, b, sort.by) * (sort.desc ? -1 : 1));
}
@@ -227,8 +277,8 @@ export default function WorkersPage() {
return (
<div>
<PageHeader
title="Workers"
description="Physical worker processes managed over SSH. One worker = one machine running the Warmbly worker binary."
title="Fleet"
description="Every machine running Warmbly. They enrol themselves, report what they are running, and stay on the version you set."
>
<Button size="sm" asChild>
<Link to="/workers/new">
+6 -11
View File
@@ -7,10 +7,10 @@ import { Link } from "react-router-dom";
import { Badge } from "@/components/ui/badge";
import { DataTable, type Column } from "@/components/data/DataTable";
import { StateLegend } from "@/components/StateLegend";
import { WORKER_HEALTH_LEGEND, WORKER_RISK_POOL_LEGEND } from "@/lib/legends";
import { WORKER_HEALTH_LEGEND } from "@/lib/legends";
import { getFleetCapacity, type AdminFleetWorkerRow } from "@/lib/api/client/admin/fleet";
import { cn } from "@/lib/utils";
import { HealthPill, LiveDot, RiskPoolPill, TierPill, TypePill } from "./tones";
import { HealthPill, LiveDot } from "./tones";
import { fmtAgo } from "./format";
function UtilizationBar({ row }: { row: AdminFleetWorkerRow }) {
@@ -83,15 +83,11 @@ const columns: Column<AdminFleetWorkerRow>[] = [
),
csv: (w) => w.name || w.worker_id,
},
{ id: "tier", header: "Tier", cell: (w) => <TierPill freeTier={w.free_tier} />, csv: (w) => (w.free_tier ? "free" : "premium") },
{ id: "type", header: "Type", cell: (w) => <TypePill type={w.worker_type} />, csv: (w) => w.worker_type },
{ id: "pool", header: "Risk pool", cell: (w) => <RiskPoolPill pool={w.risk_pool} />, csv: (w) => w.risk_pool },
{
id: "egress",
header: "Egress",
cell: (w) => <span className="font-mono text-[11px]">{w.egress_kind}</span>,
csv: (w) => w.egress_kind,
defaultHidden: true,
id: "region",
header: "Region",
cell: (w) => <span className="font-mono text-[11px]">{w.region || "—"}</span>,
csv: (w) => w.region,
},
{ id: "health", header: "Health", cell: (w) => <HealthPill state={w.health_state} />, csv: (w) => w.health_state },
{
@@ -197,7 +193,6 @@ export function CapacityTab() {
)}
</span>
<span className="flex flex-wrap gap-3">
<StateLegend label="Risk pools" entries={WORKER_RISK_POOL_LEGEND} />
<StateLegend label="Health states" entries={WORKER_HEALTH_LEGEND} />
</span>
</div>
@@ -1,6 +1,9 @@
// Convert a shared worker into a dedicated one bound to a workspace. The
// backend refuses a worker that still holds mailboxes unless a drain target
// is named, so the dialog requires one whenever the chosen worker is loaded.
// Reserve a worker for one workspace, so its mailboxes authenticate to their
// providers from an address nobody else uses.
//
// Reserving only writes the binding. Mailboxes already on the worker are not
// evicted here: the rotation loop moves other tenants off on its own schedule,
// which is why there is no drain step to fill in.
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
@@ -24,14 +27,14 @@ import {
SelectValue,
} from "@/components/ui/select";
import { convertWorkerToDedicated } from "@/lib/api/client/admin/fleet";
import { listManagedWorkers } from "@/lib/api/client/admin/workers";
import type { ManagedWorker } from "@/lib/api/models/admin";
import { listFleetNodes, nodeState, type FleetNode } from "@/lib/api/client/admin/fleetNodes";
import { OrgPicker, type PickedOrg } from "./OrgPicker";
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
function workerLabel(w: ManagedWorker): string {
return `${w.name || w.id.slice(0, 8)} · ${w.free_tier ? "free" : "premium"} · ${w.health_state} · ${w.account_count} mailbox${w.account_count === 1 ? "" : "es"}`;
function workerLabel(w: FleetNode): string {
const region = w.region ? ` · ${w.region}` : "";
return `${w.name || w.id.slice(0, 8)}${region} · ${nodeState(w)} · ${(w.mailbox_count ?? 0)} mailbox${(w.mailbox_count ?? 0) === 1 ? "" : "es"}`;
}
export function ConvertDedicatedDialog({
@@ -45,36 +48,30 @@ export function ConvertDedicatedDialog({
const [workerId, setWorkerId] = useState("");
const [org, setOrg] = useState<PickedOrg | null>(null);
const [subscriptionId, setSubscriptionId] = useState("");
const [drainTo, setDrainTo] = useState("");
const workersQ = useQuery({
queryKey: ["admin", "workers", "managed"],
queryFn: listManagedWorkers,
queryFn: () => listFleetNodes("worker"),
enabled: open,
staleTime: 30_000,
});
const workers = workersQ.data?.data ?? [];
const shared = workers.filter((w) => w.worker_type === "shared");
const worker = workers.find((w) => w.id === workerId) ?? null;
const needsDrain = !!worker && worker.account_count > 0;
// Mailboxes keep their tier when drained, so the target must match it.
const drainTargets = workers.filter((w) => w.id !== workerId && (!worker || w.free_tier === worker.free_tier));
// Any worker can be reserved: there is no category to check.
const shared = workers;
const subOk = UUID_RE.test(subscriptionId.trim());
const canSubmit = !!workerId && !!org && subOk && (!needsDrain || !!drainTo);
const canSubmit = !!workerId && !!org && subOk;
const mutation = useMutation({
mutationFn: () =>
convertWorkerToDedicated(workerId, {
organization_id: org!.id,
subscription_id: subscriptionId.trim(),
drain_to_worker_id: drainTo || null,
}),
onSuccess: (res) => {
toast.success(
res.new_assignment
? `Worker is now dedicated to ${org?.name}${res.accounts_drained ? ` (${res.accounts_drained} mailboxes drained)` : ""}`
: "Binding already existed; worker type set to dedicated",
res.new_reservation
? `Worker reserved for ${org?.name}. Other tenants drift off it on the rotation loop.`
: "That workspace already had this worker reserved.",
);
qc.invalidateQueries({ queryKey: ["admin", "workers"] });
qc.invalidateQueries({ queryKey: ["admin", "fleet"] });
@@ -88,7 +85,6 @@ export function ConvertDedicatedDialog({
setWorkerId("");
setOrg(null);
setSubscriptionId("");
setDrainTo("");
}
return (
@@ -107,23 +103,24 @@ export function ConvertDedicatedDialog({
}}
>
<DialogHeader>
<DialogTitle>Convert a worker to dedicated</DialogTitle>
<DialogTitle>Reserve a worker</DialogTitle>
<DialogDescription>
The worker leaves the shared pool and only this workspace's mailboxes are placed on it.
Its existing mailboxes must be drained to another worker of the same tier first.
This workspace&apos;s mailboxes will sign in from an address no other
tenant uses. Placement prefers the reserved worker for them, and moves
other tenants off it over the following passes.
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="space-y-1.5">
<Label className="text-xs">Shared worker</Label>
<Select value={workerId || undefined} onValueChange={(v) => { setWorkerId(v); setDrainTo(""); }}>
<Label className="text-xs">Worker</Label>
<Select value={workerId || undefined} onValueChange={setWorkerId}>
<SelectTrigger className="h-8 w-full text-[12.5px]">
<SelectValue placeholder={workersQ.isLoading ? "Loading workers…" : "Pick a shared worker"} />
<SelectValue placeholder={workersQ.isLoading ? "Loading workers…" : "Pick a worker"} />
</SelectTrigger>
<SelectContent>
{shared.length === 0 && (
<div className="px-2 py-1.5 text-xs text-muted-foreground">No shared workers.</div>
<div className="px-2 py-1.5 text-xs text-muted-foreground">No workers.</div>
)}
{shared.map((w) => (
<SelectItem key={w.id} value={w.id} className="text-[12.5px]">
@@ -158,29 +155,11 @@ export function ConvertDedicatedDialog({
</p>
</div>
<div className="space-y-1.5">
<Label className="text-xs">
Drain mailboxes to{" "}
<span className="font-normal text-muted-foreground">
{needsDrain ? `(required: ${worker!.account_count} assigned)` : "(optional)"}
</span>
</Label>
<Select value={drainTo || undefined} onValueChange={setDrainTo} disabled={!workerId}>
<SelectTrigger className="h-8 w-full text-[12.5px]">
<SelectValue placeholder={needsDrain ? "Pick where the current mailboxes go" : "Leave as is"} />
</SelectTrigger>
<SelectContent>
{drainTargets.length === 0 && (
<div className="px-2 py-1.5 text-xs text-muted-foreground">No other worker in this tier.</div>
)}
{drainTargets.map((w) => (
<SelectItem key={w.id} value={w.id} className="text-[12.5px]">
{workerLabel(w)}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<p className="text-[11px] text-muted-foreground">
Mailboxes already on this worker are not evicted here. The rotation loop
moves other tenants off it on its own schedule, so the reservation
becomes exclusive without re-authenticating every mailbox at once.
</p>
</div>
<DialogFooter>
@@ -195,7 +174,7 @@ export function ConvertDedicatedDialog({
Cancel
</Button>
<Button onClick={() => mutation.mutate()} disabled={!canSubmit || mutation.isPending}>
{mutation.isPending ? "Converting…" : "Convert to dedicated"}
{mutation.isPending ? "Reserving…" : "Reserve worker"}
</Button>
</DialogFooter>
</DialogContent>
@@ -11,7 +11,7 @@ import { Skeleton } from "@/components/ui/skeleton";
import { ErrorState } from "@/components/ErrorState";
import { SelectFilter } from "@/components/data/Explorer";
import { listFleetDecisions, type AdminFleetDecision } from "@/lib/api/client/admin/fleet";
import { listManagedWorkers } from "@/lib/api/client/admin/workers";
import { listFleetNodes, type FleetNode } from "@/lib/api/client/admin/fleetNodes";
import { fmtAgo, fmtDateTime, shortId } from "./format";
const LIMIT = 200;
@@ -110,7 +110,7 @@ export function DecisionsTab() {
const workersQ = useQuery({
queryKey: ["admin", "workers", "managed"],
queryFn: listManagedWorkers,
queryFn: () => listFleetNodes("worker"),
staleTime: 60_000,
});
const workers = workersQ.data?.data ?? [];
@@ -32,8 +32,7 @@ export function DedicatedTab() {
mutationFn: (orgId: string) => releaseDedicatedWorker(orgId),
onSuccess: (res) => {
toast.success(
`${res.accounts_moved} mailbox${res.accounts_moved === 1 ? "" : "es"} moved to shared workers` +
(res.returned_to_shared ? "; worker returned to the shared pool" : "; worker still bound to another workspace"),
`Reservation released; ${res.accounts_remaining} mailbox${res.accounts_remaining === 1 ? "" : "es"} still on that worker`,
);
qc.invalidateQueries({ queryKey: ["admin", "workers"] });
qc.invalidateQueries({ queryKey: ["admin", "fleet"] });
@@ -44,7 +43,7 @@ export function DedicatedTab() {
async function onRelease(a: AdminDedicatedAssignment) {
const ok = await confirm({
title: `Release ${a.worker_name || shortId(a.worker_id)} from ${a.organization_name}?`,
description: `The workspace's ${a.account_count} mailbox${a.account_count === 1 ? "" : "es"} move back onto shared premium workers and the worker returns to the shared pool once no other workspace binds it. A mailbox with no live shared target stays where it is.`,
description: `The worker rejoins the general fleet. The workspace's ${a.account_count} mailbox${a.account_count === 1 ? "" : "es"} stay where they are: moving one changes the address its provider sees signing in, so the rotation loop only re-places them when it has a reason to.`,
confirmLabel: "Release",
destructive: true,
});
@@ -113,12 +112,14 @@ export function DedicatedTab() {
<div>
<div className="mb-3 flex flex-wrap items-center justify-between gap-2">
<p className="text-[12.5px] text-muted-foreground max-w-2xl">
A dedicated worker carries one workspace's mailboxes and nothing else, so its IP reputation is that
workspace's alone. Placement still respects the mailbox tier and the warmup pool policy.
A reserved worker carries one workspace's mailboxes and nothing else, so that workspace always
authenticates to its mailbox providers from an address no other tenant sends from. That is what
the entitlement buys: fewer sign-in challenges and no shared per-IP auth throttle. It is a strong
placement preference, not a pin, so a worker going down never strands the workspace.
</p>
<Button size="sm" onClick={() => setConvertOpen(true)}>
<Plus className="size-4" />
Convert a worker to dedicated
Reserve a worker
</Button>
</div>
<DataTable
@@ -132,8 +133,8 @@ export function DedicatedTab() {
storageKey="admin.fleet.dedicated"
csvName="warmbly-dedicated-workers"
noun="assignments"
emptyTitle="No dedicated workers"
emptyHint="Every worker is shared. Convert one above to reserve it for a single workspace."
emptyTitle="No reserved workers"
emptyHint="Every worker is shared by the whole fleet. Reserve one above to give a workspace its own sending address."
/>
<ConvertDedicatedDialog open={convertOpen} onOpenChange={setConvertOpen} />
</div>
+1 -38
View File
@@ -2,11 +2,10 @@
// pills here read the same as everywhere else.
import { Badge } from "@/components/ui/badge";
import { WORKER_HEALTH_LEGEND, WORKER_RISK_POOL_LEGEND } from "@/lib/legends";
import { WORKER_HEALTH_LEGEND } from "@/lib/legends";
import { cn } from "@/lib/utils";
const HEALTH_TONE = Object.fromEntries(WORKER_HEALTH_LEGEND.map((e) => [e.term, e.tone ?? ""]));
const RISK_TONE = Object.fromEntries(WORKER_RISK_POOL_LEGEND.map((e) => [e.term, e.tone ?? ""]));
const FALLBACK = "border-zinc-300 text-zinc-600";
export function HealthPill({ state }: { state: string }) {
@@ -17,42 +16,6 @@ export function HealthPill({ state }: { state: string }) {
);
}
export function RiskPoolPill({ pool }: { pool: string }) {
return (
<Badge variant="outline" className={cn("text-[10px]", RISK_TONE[pool] ?? FALLBACK)}>
{pool || "—"}
</Badge>
);
}
export function TierPill({ freeTier }: { freeTier: boolean }) {
return (
<Badge
variant="outline"
className={cn(
"text-[10px]",
freeTier ? "border-zinc-300 text-zinc-700" : "border-purple-300 bg-purple-50 text-purple-700",
)}
>
{freeTier ? "free" : "premium"}
</Badge>
);
}
export function TypePill({ type }: { type: string }) {
return (
<Badge
variant="outline"
className={cn(
"text-[10px]",
type === "dedicated" ? "border-sky-300 bg-sky-50 text-sky-700" : "border-zinc-300 text-zinc-700",
)}
>
{type}
</Badge>
);
}
export function LiveDot({ live, title }: { live: boolean; title?: string }) {
return (
<span className="inline-flex items-center gap-1.5 text-xs" title={title}>
+2 -2
View File
@@ -1,5 +1,5 @@
// Inline "what do these states mean?" legend for enum badges (risk pools,
// health states, job statuses). Renders a small help trigger that reveals a
// Inline "what do these states mean?" legend for enum badges (health states,
// job statuses). Renders a small help trigger that reveals a
// term → definition list on hover/focus, so tables stay compact but no state
// name is ever left unexplained.
@@ -30,7 +30,7 @@ import { AdminPerm, hasAdminPerm } from "@/lib/auth/permissions";
import { searchUsers } from "@/lib/api/client/admin/users";
import { listOrganizations } from "@/lib/api/client/admin/organizations";
import { searchMailboxes } from "@/lib/api/client/admin/mailboxes";
import { listManagedWorkers } from "@/lib/api/client/admin/workers";
import { listFleetNodes, type FleetNode } from "@/lib/api/client/admin/fleetNodes";
import { checkForUpdates } from "@/lib/api/client/admin/updates";
import { visibleNavGroups } from "./Sidebar";
@@ -153,7 +153,7 @@ export function CommandPalette() {
});
const workersQ = useQuery({
queryKey: ["admin", "workers", "managed"],
queryFn: listManagedWorkers,
queryFn: () => listFleetNodes("worker"),
enabled: searching && canWorkers,
staleTime: 30_000,
});
@@ -165,8 +165,7 @@ export function CommandPalette() {
(w) =>
includes(w.name, debounced) ||
includes(w.id, debounced) ||
includes(w.ip_addr ?? "", debounced) ||
includes(w.ssh_host ?? "", debounced),
includes(w.address ?? "", debounced),
)
.slice(0, SEARCH_LIMIT);
}, [workersQ.data, debounced, searching]);
@@ -328,7 +327,7 @@ export function CommandPalette() {
<Server className="size-4" />
<span className="truncate">{w.name}</span>
<span className="ml-auto truncate font-mono text-[11px] text-muted-foreground">
{w.ssh_host || w.ip_addr}
{w.address}
</span>
</CommandItem>
))}
+18 -26
View File
@@ -1,25 +1,17 @@
// /admin/fleet/* — placement as the operator sees it: every worker against
// its capacity row, the decision log the control loops write, and the
// dedicated bindings. Shapes mirror the fleet section of
// isolated-egress reservations. Shapes mirror the fleet section of
// internal/models/admin_ops.go (snake_case, as the backend serializes them).
import { Request } from "@/lib/api/client";
import type {
WorkerEgressKind,
WorkerHealthState,
WorkerRiskPool,
WorkerType,
} from "@/lib/api/models/admin";
import type { WorkerHealthState } from "@/lib/api/models/admin";
export interface AdminFleetWorkerRow {
worker_id: string;
name: string;
ip_addr: string;
active: boolean;
free_tier: boolean;
worker_type: WorkerType;
risk_pool: WorkerRiskPool;
egress_kind: WorkerEgressKind;
region: string;
health_state: WorkerHealthState;
install_state: string;
last_seen_at?: string | null;
@@ -32,7 +24,7 @@ export interface AdminFleetWorkerRow {
health_multiplier: number;
age_multiplier: number;
effective_capacity: number;
/** Load over effective capacity; the rebalancer calls a worker hot above 0.8 and cold below 0.5. */
/** Load over effective capacity; the rotation loop calls a worker hot above 0.85. */
utilization: number;
sends_attempted_1h: number;
sends_succeeded_1h: number;
@@ -69,24 +61,23 @@ export interface AdminDedicatedAssignment {
account_count: number;
}
export interface AdminConvertDedicatedRequest {
export interface AdminReserveWorkerRequest {
organization_id: string;
subscription_id: string;
drain_to_worker_id?: string | null;
}
export interface AdminConvertDedicatedResponse {
// Mirrors the /reserve handler. Reserving no longer drains anything: the
// rotation loop moves other tenants off on its own schedule.
export interface AdminReserveWorkerResponse {
ok: boolean;
accounts_drained: number;
new_assignment: boolean;
worker_id: string;
new_reservation: boolean;
}
export interface AdminReleaseDedicatedResponse {
ok: boolean;
worker_id: string;
accounts_moved: number;
accounts_remaining: number;
returned_to_shared: boolean;
}
export interface FleetDecisionsParams {
@@ -126,8 +117,9 @@ export function listDedicatedAssignments(): Promise<{ data: AdminDedicatedAssign
});
}
// Moves the workspace's mailboxes back to shared premium workers, releases the
// binding, and returns the worker to the shared pool when nothing else binds it.
// Releases the reservation. Nothing migrates: the worker carries no category to
// reset, and the workspace's mailboxes stay put until the rotation loop finds
// them a better home on its own schedule.
export function releaseDedicatedWorker(orgId: string): Promise<AdminReleaseDedicatedResponse> {
return Request({
method: "POST",
@@ -136,15 +128,15 @@ export function releaseDedicatedWorker(orgId: string): Promise<AdminReleaseDedic
});
}
// The backend refuses a worker that still carries mailboxes unless
// drain_to_worker_id names where they go first.
// Reserves a worker for one organization. It only writes the binding; the
// mailboxes already on it drift away on the rotation loop.
export function convertWorkerToDedicated(
workerId: string,
body: AdminConvertDedicatedRequest,
): Promise<AdminConvertDedicatedResponse> {
body: AdminReserveWorkerRequest,
): Promise<AdminReserveWorkerResponse> {
return Request({
method: "POST",
url: `/admin/workers/${workerId}/convert-dedicated`,
url: `/admin/workers/${workerId}/reserve`,
authorization: true,
data: body,
});
@@ -0,0 +1,115 @@
// /admin/fleet/* — the fleet as the operator sees it.
//
// Nodes are pull-based: they enrol with the join token, heartbeat, and ask what
// version they should be running. Nothing here reaches into a machine, so there
// is no install, restart, logs or reboot call to make.
import { Request } from "@/lib/api/client";
export type NodeRole = "worker" | "consumer";
export interface NodeUsage {
cpu_percent?: number;
memory_mb?: number;
goroutines?: number;
uptime_seconds?: number;
}
export interface FleetNode {
id: string;
role: NodeRole;
name: string;
notes: string;
region: string;
address: string;
/** What the node reports it is running. */
version: string;
/** Set when this one node is held at a version, overriding the fleet target. */
pinned_version?: string;
/** What the control plane wants it to run. Empty means "no opinion". */
desired_version?: string;
active: boolean;
last_seen_at?: string | null;
enrolled_at: string;
usage: NodeUsage;
/** How much mail this node carries. Workers only; absent for a consumer. */
mailbox_count?: number;
last_error?: string;
tags?: string[] | null;
created_at: string;
updated_at: string;
}
export function listFleetNodes(role?: NodeRole): Promise<{ data: FleetNode[] }> {
const q = role ? `?role=${role}` : "";
return Request({
method: "GET",
url: `/admin/fleet/nodes${q}`,
authorization: true,
});
}
// The token is returned once and never again: only its hash is stored. Issuing
// a new one revokes the previous token; nodes already enrolled are unaffected.
export function issueJoinToken(): Promise<{ token: string; note: string }> {
return Request({
method: "POST",
url: "/admin/fleet/join-token",
authorization: true,
});
}
/** Liveness matches the server's window: a node is live if it beat recently. */
export const NODE_LIVENESS_MS = 5 * 60 * 1000;
export function nodeIsLive(n: FleetNode): boolean {
if (!n.active || !n.last_seen_at) return false;
return Date.now() - new Date(n.last_seen_at).getTime() <= NODE_LIVENESS_MS;
}
export type NodeState = "live" | "unreachable" | "stopped";
export function nodeState(n: FleetNode): NodeState {
if (!n.active) return "stopped";
return nodeIsLive(n) ? "live" : "unreachable";
}
/** True when the node is running something other than what it should be. */
export function nodeNeedsUpdate(n: FleetNode): boolean {
if (!n.desired_version) return false;
return n.version !== n.desired_version;
}
export interface FleetRelease {
channel: "stable" | "dev" | "pinned";
/** The resolved tag. Empty means nothing has been resolved yet. */
tag: string;
resolved_at: string;
source?: string;
}
export function getFleetRelease(): Promise<FleetRelease> {
return Request({ method: "GET", url: "/admin/fleet/release", authorization: true });
}
// Setting a tag also pins the channel, so a release landing later does not
// silently undo a deliberate rollback.
export function setFleetRelease(body: {
channel?: FleetRelease["channel"];
tag?: string;
}): Promise<FleetRelease> {
return Request({ method: "PUT", url: "/admin/fleet/release", data: body, authorization: true });
}
export function patchFleetNode(
id: string,
body: { name?: string; notes?: string; pinned_version?: string },
): Promise<FleetNode> {
return Request({ method: "PATCH", url: `/admin/fleet/nodes/${id}`, data: body, authorization: true });
}
// Forgetting a node does not stop it: a process still running re-joins on its
// next heartbeat. Stop the service on the machine too.
export function deleteFleetNode(id: string): Promise<{ ok: boolean; note: string }> {
return Request({ method: "DELETE", url: `/admin/fleet/nodes/${id}`, authorization: true });
}
+28 -178
View File
@@ -1,13 +1,11 @@
// /admin/workers/* — the SSH-managed-worker control plane.
// /admin/workers/* — what a worker is carrying.
//
// The machine half of a worker (liveness, version, usage, enrolment) lives in
// the fleet node API instead; see ./fleetNodes.ts. There is nothing here that
// reaches into a machine, because nothing does any more.
import { Request } from "@/lib/api/client";
import type {
AdminWorkerEmailsResult,
CreateWorkerInput,
CreateWorkerResponse,
ManagedWorker,
WorkerLiveStatus,
} from "@/lib/api/models/admin";
import type { AdminWorkerEmailsResult } from "@/lib/api/models/admin";
// getWorkerEmails returns the mailboxes assigned to a worker (paginated), with
// per-mailbox risk band + warmup health so the detail page can show how healthy
@@ -24,171 +22,8 @@ export function getWorkerEmails(
});
}
export function listManagedWorkers(): Promise<{ data: ManagedWorker[] }> {
return Request({
method: "GET",
url: "/admin/workers/managed",
authorization: true,
});
}
export function getManagedWorker(id: string): Promise<ManagedWorker> {
return Request({
method: "GET",
url: `/admin/workers/${id}/managed`,
authorization: true,
});
}
export function createWorker(input: CreateWorkerInput): Promise<CreateWorkerResponse> {
return Request({
method: "POST",
url: "/admin/workers",
data: input,
authorization: true,
});
}
// Reachability probe before the row is created, so a typo'd host fails fast
// instead of after the keypair is minted.
export function preflightWorker(
host: string,
port: number,
): Promise<{ ok: boolean; latency_ms?: number; error?: string }> {
return Request({
method: "POST",
url: "/admin/workers/preflight",
data: { host, port },
authorization: true,
});
}
export function testWorker(id: string): Promise<{ ok: boolean; error?: string }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/test`,
authorization: true,
});
}
export function installWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/install`,
authorization: true,
});
}
export function restartWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/restart`,
authorization: true,
});
}
export function uninstallWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/uninstall`,
authorization: true,
});
}
export function getWorkerLiveStatus(id: string): Promise<WorkerLiveStatus> {
return Request({
method: "GET",
url: `/admin/workers/${id}/live-status`,
authorization: true,
});
}
export function getWorkerLogs(id: string, lines = 200): Promise<{ logs: string }> {
return Request({
method: "GET",
url: `/admin/workers/${id}/logs?lines=${lines}`,
authorization: true,
});
}
// Pulls the newest image and restarts the unit. "Apply config" below only
// rewrites the env file, so this is the only path that changes the image.
export function upgradeWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/upgrade`,
authorization: true,
});
}
// Rewrites /etc/warmbly/worker.env over SSH and restarts, without pulling.
export function applyWorkerConfig(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/apply`,
authorization: true,
});
}
// Mints a fresh keypair and returns the new public key, which has to be pasted
// into the VPS before anything else will authenticate again.
export function rotateWorkerKeys(id: string): Promise<{ ssh_public_key: string }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/rotate-keys`,
authorization: true,
});
}
export function systemUpdateWorker(
id: string,
): Promise<{ output: string; reboot_required: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/system-update`,
authorization: true,
});
}
export function rebootWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${id}/reboot`,
authorization: true,
});
}
export function deleteWorker(id: string): Promise<{ ok: boolean }> {
return Request({
method: "DELETE",
url: `/admin/workers/${id}`,
authorization: true,
});
}
export function setWorkerTags(
workerID: string,
tags: string[],
): Promise<{ ok: boolean; tags: string[] }> {
return Request({
method: "PUT",
url: `/admin/workers/${workerID}/tags`,
data: { tags },
authorization: true,
});
}
export function listAllWorkerTags(): Promise<{ data: string[] }> {
return Request({
method: "GET",
url: "/admin/workers/tags",
authorization: true,
});
}
// ---- capacity and reassignment (used by the worker detail page) ----
// WorkerStats in internal/models/admin.go.
// Mirrors models.WorkerStats. Renaming a field here does not rename it on the
// wire; it just renders blank.
export interface WorkerStats {
worker_id: string;
total_emails_sent: number;
@@ -207,17 +42,32 @@ export function getWorkerStats(id: string): Promise<WorkerStats> {
});
}
// POST /admin/workers/:id/reassign moves email_ids onto the worker in the
// URL. ReassignEmailsRequest also requires new_worker_id; the handler ignores
// it in favour of the path, so both name the target.
// Moves mailboxes onto the worker in the URL. Placement would get there on its
// own, so this is for when you know something it does not; the rotation loop
// will move them again if it disagrees once their residency window passes.
export function reassignWorkerEmails(
targetWorkerId: string,
emailIds: string[],
): Promise<{ message: string }> {
): Promise<{ ok: boolean }> {
return Request({
method: "POST",
url: `/admin/workers/${targetWorkerId}/reassign`,
authorization: true,
// The target is in the URL and in the body: models.ReassignEmailsRequest
// marks new_worker_id required, so omitting it is a 400.
data: { email_ids: emailIds, new_worker_id: targetWorkerId },
authorization: true,
});
}
export function setWorkerTags(id: string, tags: string[]): Promise<{ ok: boolean; tags: string[] }> {
return Request({
method: "PUT",
url: `/admin/workers/${id}/tags`,
data: { tags },
authorization: true,
});
}
export function listWorkerTags(): Promise<{ data: string[] }> {
return Request({ method: "GET", url: "/admin/workers/tags", authorization: true });
}
+4 -10
View File
@@ -9,10 +9,6 @@ export type WorkerInstallState =
| "uninstalling"
| "uninstalled";
export type WorkerType = "shared" | "dedicated";
export type WorkerRiskPool = "clean" | "risky" | "quarantine";
export type WorkerEgressKind = "cold_smtp" | "oauth_api" | "warmup_only";
export type WorkerHealthState =
| "healthy"
| "watch"
@@ -26,11 +22,9 @@ export interface ManagedWorker {
notes: string;
ip_addr: string;
active: boolean;
free_tier: boolean;
worker_type: WorkerType;
account_count: number;
risk_pool: WorkerRiskPool;
egress_kind: WorkerEgressKind;
/** Sign-in geography hint the placer scores on. Empty is fine. */
region: string;
health_state: WorkerHealthState;
load_score: number;
@@ -82,8 +76,8 @@ export interface AdminWorkerEmailsResult {
export interface CreateWorkerInput {
name: string;
notes?: string;
worker_type: WorkerType;
free_tier: boolean;
/** Optional sign-in geography hint. The only placement input an operator sets. */
region?: string;
ssh_host: string;
ssh_port?: number;
ssh_user?: string;
-26
View File
@@ -4,32 +4,6 @@
import type { LegendEntry } from "@/components/StateLegend";
// Shared workers are bucketed by acceptable mailbox risk so one bad tenant
// can't burn the IP reputation healthy senders depend on.
export const WORKER_RISK_POOL_LEGEND: LegendEntry[] = [
{
term: "clean",
tone: "border-emerald-300 bg-emerald-50 text-emerald-700",
description:
"Carries only mailboxes with healthy reputation. Protect this pool's IPs first.",
},
{
term: "risky",
tone: "border-amber-300 bg-amber-50 text-amber-700",
description:
"Accepts unproven or recovering mailboxes. Expect noisier deliverability here.",
},
{
term: "quarantine",
tone: "border-red-300 bg-red-50 text-red-700",
description:
"Isolation pool after abuse or deliverability incidents. Never place healthy traffic here.",
},
];
// Worker health is the rolled-up label maintained by the assignment loop.
// It gates whether a worker can accept new mailboxes; it mirrors the mailbox
// health vocabulary but applies to the whole machine.
export const WORKER_HEALTH_LEGEND: LegendEntry[] = [
{
term: "healthy",
+31 -141
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"net/url"
@@ -55,6 +54,7 @@ import (
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
"github.com/warmbly/warmbly/internal/app/feature"
"github.com/warmbly/warmbly/internal/app/fleet"
"github.com/warmbly/warmbly/internal/app/fleetnode"
"github.com/warmbly/warmbly/internal/app/form"
"github.com/warmbly/warmbly/internal/app/group"
"github.com/warmbly/warmbly/internal/app/guardrail"
@@ -77,7 +77,6 @@ import (
"github.com/warmbly/warmbly/internal/app/passkey"
"github.com/warmbly/warmbly/internal/app/placement"
"github.com/warmbly/warmbly/internal/app/poollink"
"github.com/warmbly/warmbly/internal/app/provisioning"
"github.com/warmbly/warmbly/internal/app/ratelimit"
"github.com/warmbly/warmbly/internal/app/referral"
"github.com/warmbly/warmbly/internal/app/releases"
@@ -106,13 +105,10 @@ import (
"github.com/warmbly/warmbly/internal/app/webhook"
"github.com/warmbly/warmbly/internal/app/websitetracking"
"github.com/warmbly/warmbly/internal/app/worker"
"github.com/warmbly/warmbly/internal/app/worker_orchestrator"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/events"
"github.com/warmbly/warmbly/internal/infrastructure/apns"
"github.com/warmbly/warmbly/internal/infrastructure/cache"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider/hetzner"
"github.com/warmbly/warmbly/internal/infrastructure/codec"
"github.com/warmbly/warmbly/internal/infrastructure/db"
"github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys"
@@ -181,10 +177,9 @@ func main() {
var passkeyService passkey.Service
var encryptedKeys encryptedkeys.Store
var storageBackendRepo repository.StorageBackendRepository
var cloudCredentialRepo repository.CloudCredentialRepository
var provisioningTemplateRepo repository.ProvisioningTemplateRepository
var provisioningJobRepo repository.ProvisioningJobRepository
var provisioningPolicyRepo repository.ProvisioningPolicyRepository
var fleetNodeRepo repository.FleetNodeRepository
var fleetSettingsRepo repository.FleetSettingsRepository
var fleetNodeService *fleetnode.Service
var tasksService tasks.TasksService
var advancedService advanced.Service
var unsubSigner *unsublink.Signer
@@ -237,9 +232,7 @@ func main() {
var dailyThrottleService dailythrottle.Service
// Worker orchestrator (SSH-driven admin worker lifecycle)
var workerOrchestrator *worker_orchestrator.Orchestrator
var workerRepoForHandler repository.WorkerRepository
var credentialsRepository repository.CredentialsRepository
var releasesService *releases.Service
var updatesService *updates.Service
@@ -1031,10 +1024,9 @@ func main() {
// were chosen via env vars and changing them at runtime would orphan
// existing ciphertext / DEKs.
storageBackendRepo = repository.NewStorageBackendRepository(primaryDB)
cloudCredentialRepo = repository.NewCloudCredentialRepository(primaryDB)
provisioningTemplateRepo = repository.NewProvisioningTemplateRepository(primaryDB)
provisioningJobRepo = repository.NewProvisioningJobRepository(primaryDB)
provisioningPolicyRepo = repository.NewProvisioningPolicyRepository(primaryDB)
fleetNodeRepo = repository.NewFleetNodeRepository(primaryDB)
fleetSettingsRepo = repository.NewFleetSettingsRepository(primaryDB)
fleetNodeService = fleetnode.New(fleetNodeRepo, workerRepository, fleetSettingsRepo)
settingsRegistrar := settings.NewRegistrar(storageBackendRepo)
if err := settingsRegistrar.RegisterAll(ctx, []settings.Backend{
{Kind: "kms", Provider: kms.Name(), Display: kms.Name(), ReadOnly: true},
@@ -1051,123 +1043,26 @@ func main() {
// the root context on shutdown.
decisionLogRepo := repository.NewDecisionLogRepository(primaryDB)
// Refresh worker_capacity_view every minute so the assignment loop +
// rebalance + scale + quarantine evaluators see fresh rolling
// metrics. The materialized view is what aggregates the 1h windows
// across all workers.
// Refresh worker_capacity_view every minute so placement, rotation,
// scale and quarantine all see fresh rolling metrics. The materialized
// view is what aggregates the 1h windows across all workers.
go jobrun.Loop(ctx, "worker_capacity_refresh", time.Minute, false, workerRepository.RefreshWorkerCapacityView)
go (&fleet.Rebalancer{
go (&fleet.Rotator{
WorkerRepo: workerRepository,
Assignment: workerAssignmentService,
Decisions: decisionLogRepo,
}).Run(ctx)
go (&fleet.Scaler{
WorkerRepo: workerRepository,
PolicyRepo: provisioningPolicyRepo,
TemplateRepo: provisioningTemplateRepo,
JobRepo: provisioningJobRepo,
Decisions: decisionLogRepo,
WorkerRepo: workerRepository,
Decisions: decisionLogRepo,
}).Run(ctx)
go (&fleet.QuarantineEvaluator{
WorkerRepo: workerRepository,
Decisions: decisionLogRepo,
}).Run(ctx)
// Provisioning runner. Drives provisioning_jobs rows to completion —
// without it a job created from the admin UI sits in "pending" forever.
//
// Real Hetzner calls only happen when PROVISIONING_DRY_RUN=false. A real
// SSH installer adapter (over worker_orchestrator) is not wired yet, so
// until it is we force dry-run: real-mode would otherwise create servers
// it could not provision, leaving orphaned, billed machines. Dry-run runs
// the full state machine against a simulated provider so the admin flow
// works end-to-end in dev without spending money.
if getenvDefault("PROVISIONING_RUNNER_ENABLED", "true") == "true" {
provDryRun := getenvDefault("PROVISIONING_DRY_RUN", "true") != "false"
if !provDryRun {
log.Printf("PROVISIONING_DRY_RUN=false but no real installer is wired; forcing dry-run to avoid orphaned servers")
provDryRun = true
}
credRepoForResolver := cloudCredentialRepo
provService := &provisioning.Service{
Jobs: provisioningJobRepo,
Installer: &provisioning.StubInstaller{},
ProviderResolver: func(rctx context.Context, job *repository.ProvisioningJob) (cloudprovider.Provider, error) {
if provDryRun {
return provisioning.DryRunProvider{}, nil
}
if credRepoForResolver == nil {
return nil, fmt.Errorf("no cloud credential repo configured")
}
cred, err := credRepoForResolver.GetByProvider(rctx, job.Provider)
if err != nil {
return nil, err
}
if cred == nil {
return nil, fmt.Errorf("no cloud credential for provider %q", job.Provider)
}
switch cred.Provider {
case "hetzner":
return hetzner.New(cred.EncryptedToken)
default:
return nil, fmt.Errorf("unsupported provider %q", cred.Provider)
}
},
}
go (&provisioning.Runner{
Jobs: provisioningJobRepo,
Svc: provService,
DryRun: provDryRun,
}).Run(ctx)
}
// Worker orchestrator. The env config below is the FALLBACK that gets
// written into /etc/warmbly/worker.env when a worker has no profile
// assigned. Production workers should reference a worker_profile row;
// dev/sim can rely on the fallback so docker-compose still works.
workerRepoForHandler = workerRepository
credentialsRepository = repository.NewCredentialsRepository(primaryDB.Pool)
workerOrchestrator = worker_orchestrator.New(
workerRepository,
credentialsRepository,
cipherService,
worker_orchestrator.WorkerEnvConfig{
AppEnv: os.Getenv("APP_ENV"),
WorkerImage: getenvDefault("WORKER_IMAGE", "ghcr.io/warmbly/worker:latest"),
KafkaBootstrap: os.Getenv("KAFKA_BOOTSTRAP_SERVERS"),
KafkaSASLUsername: os.Getenv("KAFKA_SASL_USERNAME"),
KafkaSASLPassword: os.Getenv("KAFKA_SASL_PASSWORD"),
SchemaRegistryURL: os.Getenv("SCHEMA_REGISTRY_URL"),
SchemaRegistryKey: os.Getenv("SCHEMA_REGISTRY_KEY"),
SchemaRegistrySecret: os.Getenv("SCHEMA_REGISTRY_SECRET"),
RedisURL: os.Getenv("REDIS"),
AWSRegion: os.Getenv("AWS_REGION"),
AWSAccessKeyID: os.Getenv("WORKER_AWS_ACCESS_KEY_ID"),
AWSSecretAccessKey: os.Getenv("WORKER_AWS_SECRET_ACCESS_KEY"),
// A remote worker reaches the internal API over the network, so
// fall back to the public API URL. ENCRYPTED_KEYS_BACKEND_URL is
// typically only set on workers themselves (compose points it at
// the in-network hostname), leaving it empty here and shipping a
// config the worker cannot use.
EncryptedKeysBackendURL: getenvDefault("ENCRYPTED_KEYS_BACKEND_URL", os.Getenv("API_PUBLIC_URL")),
EncryptedKeysWorkerToken: os.Getenv("INTERNAL_API_TOKEN"),
KMSProvider: getenvDefault("KMS_PROVIDER", "local"),
KMSLocalMasterKey: os.Getenv("KMS_LOCAL_MASTER_KEY"),
KMSAWSKeyID: os.Getenv("KMS_AWS_KEY_ID"),
CredentialsEncryptionKey: os.Getenv("CREDENTIALS_ENCRYPTION_KEY"),
BlobProvider: getenvDefault("BLOB_PROVIDER", "filesystem"),
BlobBucket: os.Getenv("BLOB_BUCKET"),
BlobFSRoot: os.Getenv("BLOB_FS_ROOT"),
EventBusProvider: os.Getenv("EVENTBUS_PROVIDER"),
NATSURL: os.Getenv("NATS_URL"),
CodecProvider: os.Getenv("CODEC_PROVIDER"),
BoxGoogleClientID: os.Getenv("BOX_GOOGLE_CLIENT_ID"),
BoxGoogleClientSecret: os.Getenv("BOX_GOOGLE_CLIENT_SECRET"),
BoxOutlookClientID: os.Getenv("BOX_OUTLOOK_CLIENT_ID"),
BoxOutlookClientSecret: os.Getenv("BOX_OUTLOOK_CLIENT_SECRET"),
},
getenvDefault("WORKER_INSTALLER_PATH", "/app/scripts/install-worker.sh"),
)
// Releases service. Off by default for self-host (no vendor image
// auto-roll, no GitHub polling on boot); set RELEASES_ENABLED=true to
@@ -1180,9 +1075,7 @@ func main() {
WebhookSecret: os.Getenv("RELEASES_WEBHOOK_SECRET"),
GithubToken: os.Getenv("RELEASES_GITHUB_TOKEN"),
},
credentialsRepository,
workerRepository,
workerOrchestrator,
fleetSettingsRepo,
)
releasesService.RunBootCheck(ctx)
@@ -2028,10 +1921,8 @@ func main() {
AdminOutreachService: adminOutreachService,
// SSH-managed worker lifecycle
WorkerOrchestrator: workerOrchestrator,
WorkerRepo: workerRepoForHandler,
CredentialsRepo: credentialsRepository,
UpdatesService: updatesService,
WorkerRepo: workerRepoForHandler,
UpdatesService: updatesService,
// Notifications
EmailNotificationService: emailNotificationService,
@@ -2087,21 +1978,20 @@ func main() {
// Object storage + direct repository handles for handlers
// without a dedicated service layer (avatars, etc.).
Storage: s3ForHandler,
EncryptedKeys: encryptedKeys,
EmailMessageMap: emailMessageMapForHandler,
EmailSyncState: emailSyncStateRepository,
TrackedLinks: trackedLinkRepository,
WebsiteTrackingService: websiteTrackingService,
UserRepo: userRepoForHandler,
OrgRepo: organizationRepoForHandler,
AttachmentRepo: attachmentRepoForHandler,
EmailImageRepo: emailImageRepoForHandler,
StorageBackendRepo: storageBackendRepo,
CloudCredentialRepo: cloudCredentialRepo,
ProvisioningTemplateRepo: provisioningTemplateRepo,
ProvisioningJobRepo: provisioningJobRepo,
ProvisioningPolicyRepo: provisioningPolicyRepo,
Storage: s3ForHandler,
EncryptedKeys: encryptedKeys,
EmailMessageMap: emailMessageMapForHandler,
EmailSyncState: emailSyncStateRepository,
TrackedLinks: trackedLinkRepository,
WebsiteTrackingService: websiteTrackingService,
UserRepo: userRepoForHandler,
OrgRepo: organizationRepoForHandler,
AttachmentRepo: attachmentRepoForHandler,
EmailImageRepo: emailImageRepoForHandler,
StorageBackendRepo: storageBackendRepo,
FleetNodeRepo: fleetNodeRepo,
FleetSettingsRepo: fleetSettingsRepo,
FleetNodes: fleetNodeService,
// Danger zone
DangerZoneService: dangerZoneService,
+68 -4
View File
@@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
awsconf "github.com/aws/aws-sdk-go-v2/config"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/advanced"
"github.com/warmbly/warmbly/internal/app/cipher"
jobs "github.com/warmbly/warmbly/internal/app/consumer"
@@ -52,6 +53,7 @@ import (
"github.com/warmbly/warmbly/internal/pkg/encrypt"
"github.com/warmbly/warmbly/internal/pkg/generation"
"github.com/warmbly/warmbly/internal/pkg/geo"
"github.com/warmbly/warmbly/internal/pkg/nodeagent"
"github.com/warmbly/warmbly/internal/repository"
)
@@ -423,6 +425,7 @@ func main() {
WarmupEngagementRepo: repository.NewWarmupEngagementRepository(primaryDB.Pool),
WarmupService: warmupService,
WorkerRepo: workerRepo,
FleetNodeRepo: repository.NewFleetNodeRepository(primaryDB),
LifecycleRepo: repository.NewSendLifecycleRepository(primaryDB),
Publisher: eventsPublisher,
StreamingPublisher: streamingPublisher,
@@ -478,11 +481,11 @@ func main() {
// so the admin dashboard can render liveness without touching Redis.
go jobsService.StartWorkerHeartbeatSync(ctx, 60*time.Second)
// Re-evaluate per-mailbox risk bands hourly and migrate to a matching
// risk_pool worker when the band changes. Skipped if AssignmentService
// or WorkerRepo are nil.
// Re-evaluate per-mailbox risk bands hourly. The band feeds warmup partner
// selection and pacing; it does not move mailboxes between workers, because
// the worker is not the sending identity.
go jobsService.StartRiskRebalancer(ctx, 1*time.Hour)
// Same cadence, different question: risk_band picks the worker, the
// Same cadence, different question: the band describes reputation, the
// lifecycle picks whether the mailbox is in cold rotation at all.
go jobsService.StartLifecycleRebalancer(ctx, 1*time.Hour)
// The abuse sweep: cross-account shape, plus what each organization's mail
@@ -544,7 +547,68 @@ func main() {
log.Println("Tracking consumer started, listening on", trackingCfg.Topic)
}
// The consumer is a fleet node like any other: it enrols, heartbeats,
// reports what it is running and what it is using, and picks up the
// version the control plane wants. Before this it was anonymous, so a
// dead one stayed invisible until work started backing up.
agentDone := make(chan struct{})
go func() {
defer close(agentDone)
newConsumerAgent().Run(ctx)
}()
log.Println("Consumer started, listening on", kafka.TopicWorkerEvents)
jobsService.Start(ctx)
// Give the farewell beat a moment to land, bounded so a wedged backend
// cannot stop the consumer exiting.
select {
case <-agentDone:
case <-time.After(8 * time.Second):
log.Println("timed out waiting for the shutdown heartbeat")
}
log.Println("Consumer stopped")
}
// newConsumerAgent builds the fleet agent for this consumer.
//
// Identity resolution mirrors the worker's: an explicit WARMBLY_NODE_ID wins,
// otherwise it is derived from the hostname so a container recreate keeps the
// same identity instead of leaving a dead row behind on every restart.
func newConsumerAgent() *nodeagent.Agent {
id := resolveConsumerID()
return nodeagent.New(nodeagent.Config{
NodeID: id,
Role: models.NodeRoleConsumer,
Name: os.Getenv("WARMBLY_NODE_NAME"),
Region: os.Getenv("WARMBLY_NODE_REGION"),
Version: os.Getenv("WARMBLY_VERSION"),
BaseURL: consumerBackendURL(),
Token: os.Getenv("INTERNAL_API_TOKEN"),
TargetVersionPath: os.Getenv("WARMBLY_TARGET_VERSION_PATH"),
})
}
func resolveConsumerID() uuid.UUID {
if raw := os.Getenv("WARMBLY_NODE_ID"); raw != "" {
if id, err := uuid.Parse(raw); err == nil {
return id
}
log.Printf("WARMBLY_NODE_ID is not a valid uuid; deriving one from the hostname instead")
}
host, err := os.Hostname()
if err != nil || host == "" {
// Last resort. A random id means this process shows up as a new node
// on every restart, which is visible in the dashboard rather than
// silent, so it is a better failure than refusing to start.
return uuid.New()
}
return uuid.NewSHA1(uuid.NameSpaceURL, []byte("warmbly-consumer:"+host))
}
func consumerBackendURL() string {
if v := os.Getenv("WARMBLY_BACKEND_URL"); v != "" {
return v
}
return os.Getenv("ENCRYPTED_KEYS_BACKEND_URL")
}
+26 -22
View File
@@ -133,9 +133,9 @@ func seedBaseline(ctx context.Context, pool *pgxpool.Pool) error {
return err
}
// Worker for dev accounts. Matches the docker-compose shared worker
// hostname so the running worker container can pick the assignments up.
if err := upsertWorker(ctx, pool, workerShared, "shared-1", "10.0.0.11", "shared", true, true); err != nil {
// Worker for dev accounts. Matches the docker-compose worker hostname so
// the running worker container can pick the assignments up.
if err := upsertWorker(ctx, pool, workerShared, "worker-1", "10.0.0.11", "", true); err != nil {
return fmt.Errorf("dev worker: %w", err)
}
@@ -254,19 +254,18 @@ func seedRich(ctx context.Context, pool *pgxpool.Pool) error {
// workers
workers := []struct {
id uuid.UUID
name string
ip string
wtype string
freeTier bool
active bool
id uuid.UUID
name string
ip string
region string
active bool
}{
{workerShared, "shared-1", "10.0.0.11", "shared", true, true},
{workerPremium, "premium-1", "10.0.0.12", "shared", false, true},
{workerDedicated, "dedicated-1", "10.0.0.13", "dedicated", false, true},
{workerShared, "worker-1", "10.0.0.11", "eu-central", true},
{workerPremium, "worker-2", "10.0.0.12", "eu-central", true},
{workerDedicated, "worker-3", "10.0.0.13", "us-east", true},
}
for _, w := range workers {
if err := upsertWorker(ctx, pool, w.id, w.name, w.ip, w.wtype, w.freeTier, w.active); err != nil {
if err := upsertWorker(ctx, pool, w.id, w.name, w.ip, w.region, w.active); err != nil {
return fmt.Errorf("worker %s: %w", w.name, err)
}
}
@@ -406,18 +405,23 @@ func upsertOrg(ctx context.Context, pool *pgxpool.Pool, id uuid.UUID, name, slug
return tx.Commit(ctx)
}
func upsertWorker(ctx context.Context, pool *pgxpool.Pool, id uuid.UUID, name, ip, wtype string, freeTier, active bool) error {
_, err := pool.Exec(ctx, `
INSERT INTO workers (id, name, ip_addr, worker_type, free_tier, active)
VALUES ($1, $2, $3, $4, $5, $6)
func upsertWorker(ctx context.Context, pool *pgxpool.Pool, id uuid.UUID, name, ip, region string, active bool) error {
if _, err := pool.Exec(ctx, `
INSERT INTO fleet_nodes (id, role, name, address, region, active, last_seen_at)
VALUES ($1, 'worker', $2, $3, $4, $5, now())
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
ip_addr = EXCLUDED.ip_addr,
worker_type = EXCLUDED.worker_type,
free_tier = EXCLUDED.free_tier,
address = EXCLUDED.address,
region = EXCLUDED.region,
active = EXCLUDED.active,
updated_at = NOW()`,
id, name, ip, wtype, freeTier, active)
last_seen_at = now(),
updated_at = now()`,
id, name, ip, region, active); err != nil {
return err
}
_, err := pool.Exec(ctx, `
INSERT INTO workers (id, account_count) VALUES ($1, 0)
ON CONFLICT (id) DO NOTHING`, id)
return err
}
+361
View File
@@ -0,0 +1,361 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"text/tabwriter"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/fleetnode"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func fleetUsage(w *os.File) {
fmt.Fprint(w, `Manage the machines running Warmbly.
warmblyctl fleet join-token Issue a join token. Shown once.
warmblyctl fleet list Every node: role, version, liveness, usage.
warmblyctl fleet show <node-id> One node in full.
warmblyctl fleet remove <node-id> Forget a node. Its mailboxes re-place themselves.
warmblyctl fleet pin <node-id> <ver> Hold one node at a version ("" clears the pin).
warmblyctl fleet version What version the fleet should be running.
warmblyctl fleet version <tag> Pin the whole fleet to a tag.
warmblyctl fleet channel <name> Follow stable, dev, or pinned.
Adding a machine is two commands: issue a token here, then on that machine run
curl -fsSL https://<your-instance>/join.sh | sh -s -- \
--url https://<your-instance> --token <token> --role worker
Roles are worker (sends and syncs mail) and consumer (processes events).
`)
}
func runFleet(ctx context.Context, args []string) error {
if len(args) == 0 {
fleetUsage(os.Stderr)
return errors.New("`fleet` needs a subcommand. Pick one from the list above.")
}
switch args[0] {
case "help", "-h", "--help":
fleetUsage(os.Stdout)
return nil
case "join-token":
return runFleetJoinToken(ctx, args[1:])
case "list":
return runFleetList(ctx, args[1:])
case "show":
return runFleetShow(ctx, args[1:])
case "remove":
return runFleetRemove(ctx, args[1:])
case "pin":
return runFleetPin(ctx, args[1:])
case "version":
return runFleetVersion(ctx, args[1:])
case "channel":
return runFleetChannel(ctx, args[1:])
}
fleetUsage(os.Stderr)
return fmt.Errorf("unknown fleet subcommand %q", args[0])
}
// fleetDeps is the small slice of the object graph the fleet commands need.
func fleetDeps(ctx context.Context) (*conn, repository.FleetNodeRepository, repository.FleetSettingsRepository, *fleetnode.Service, error) {
c, err := connect(ctx)
if err != nil {
return nil, nil, nil, nil, err
}
nodes := repository.NewFleetNodeRepository(c.db)
settings := repository.NewFleetSettingsRepository(c.db)
workers := repository.NewWorkerRepository(c.db.Pool)
return c, nodes, settings, fleetnode.New(nodes, workers, settings), nil
}
func runFleetJoinToken(ctx context.Context, args []string) error {
fs := newFlagSet("fleet join-token")
if err := fs.Parse(args); err != nil {
return err
}
if err := noExtraArgs(fs); err != nil {
return err
}
c, _, _, svc, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
token, err := svc.IssueJoinToken(ctx)
if err != nil {
return err
}
fmt.Println(token)
fmt.Fprintln(os.Stderr, "\nShown once. Issuing another token revokes this one; nodes already joined are unaffected.")
return nil
}
func runFleetList(ctx context.Context, args []string) error {
fs := newFlagSet("fleet list")
role := fs.String("role", "", "only worker or consumer")
asJSON := fs.Bool("json", false, "machine-readable output")
if err := fs.Parse(args); err != nil {
return err
}
if err := noExtraArgs(fs); err != nil {
return err
}
c, _, _, svc, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
nodes, err := svc.List(ctx, models.NodeRole(*role))
if err != nil {
return err
}
if *asJSON {
return json.NewEncoder(os.Stdout).Encode(nodes)
}
if len(nodes) == 0 {
fmt.Println("No nodes have joined yet. Issue a token with `warmblyctl fleet join-token`.")
return nil
}
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "ROLE\tNAME\tSTATE\tVERSION\tREGION\tMEM\tSEEN\tID")
for _, n := range nodes {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
n.Role, orDash(n.Name), nodeState(&n), versionCell(&n),
orDash(n.Region), memCell(&n), seenCell(&n), n.ID)
}
return w.Flush()
}
// nodeState is the one word that matters: can this node do work right now.
func nodeState(n *models.FleetNode) string {
switch {
case !n.Active:
return "stopped"
case n.Live():
return "live"
default:
return "unreachable"
}
}
// versionCell shows the pending update inline, because "which of my machines
// are behind" is the question this table exists to answer.
func versionCell(n *models.FleetNode) string {
cur := orDash(n.Version)
if n.NeedsUpdate() {
return cur + " -> " + n.DesiredVersion
}
return cur
}
func memCell(n *models.FleetNode) string {
if n.Usage.MemoryMB == nil {
return "-"
}
return fmt.Sprintf("%dMB", *n.Usage.MemoryMB)
}
func seenCell(n *models.FleetNode) string {
if n.LastSeenAt == nil {
return "never"
}
d := time.Since(*n.LastSeenAt).Round(time.Second)
if d < time.Minute {
return fmt.Sprintf("%ds ago", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm ago", int(d.Minutes()))
}
return fmt.Sprintf("%dh ago", int(d.Hours()))
}
func orDash(s string) string {
if s == "" {
return "-"
}
return s
}
func runFleetShow(ctx context.Context, args []string) error {
fs := newFlagSet("fleet show")
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() != 1 {
return errors.New("`fleet show` needs one node id")
}
id, err := uuid.Parse(fs.Arg(0))
if err != nil {
return fmt.Errorf("%q is not a node id", fs.Arg(0))
}
c, _, _, svc, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
node, err := svc.Get(ctx, id)
if err != nil {
return err
}
if node == nil {
return fmt.Errorf("no node with id %s", id)
}
return json.NewEncoder(os.Stdout).Encode(node)
}
func runFleetRemove(ctx context.Context, args []string) error {
fs := newFlagSet("fleet remove")
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() != 1 {
return errors.New("`fleet remove` needs one node id")
}
id, err := uuid.Parse(fs.Arg(0))
if err != nil {
return fmt.Errorf("%q is not a node id", fs.Arg(0))
}
c, nodes, _, _, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
if err := nodes.Delete(ctx, id); err != nil {
return err
}
// Mailboxes are released by the foreign key, not stranded: the rotation
// loop places them on a live worker on its next pass.
fmt.Printf("Removed %s. Any mailboxes it carried will be re-placed within a few minutes.\n", id)
fmt.Println("Stop the service on that machine too, or it will re-join on its next heartbeat.")
return nil
}
func runFleetPin(ctx context.Context, args []string) error {
fs := newFlagSet("fleet pin")
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() < 1 || fs.NArg() > 2 {
return errors.New("usage: warmblyctl fleet pin <node-id> [version] (omit the version to clear)")
}
id, err := uuid.Parse(fs.Arg(0))
if err != nil {
return fmt.Errorf("%q is not a node id", fs.Arg(0))
}
version := ""
if fs.NArg() == 2 {
version = fs.Arg(1)
}
c, nodes, _, _, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
if err := nodes.SetPinnedVersion(ctx, id, version); err != nil {
return err
}
if version == "" {
fmt.Printf("Cleared the pin on %s; it will follow the fleet version again.\n", id)
return nil
}
fmt.Printf("Pinned %s to %s. It will hold there until the pin is cleared.\n", id, version)
return nil
}
func runFleetVersion(ctx context.Context, args []string) error {
fs := newFlagSet("fleet version")
if err := fs.Parse(args); err != nil {
return err
}
c, _, settings, _, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
if fs.NArg() == 0 {
state, err := settings.GetRelease(ctx)
if err != nil {
return err
}
if state == nil || state.Tag == "" {
fmt.Println("No fleet version resolved yet. Nodes are leaving themselves alone.")
return nil
}
fmt.Printf("%s (channel %s, resolved %s)\n", state.Tag, state.Channel,
state.ResolvedAt.Format(time.RFC3339))
return nil
}
if fs.NArg() != 1 {
return errors.New("usage: warmblyctl fleet version [tag]")
}
tag := strings.TrimSpace(fs.Arg(0))
next := &models.FleetReleaseState{
Channel: models.FleetChannelPinned,
Tag: tag,
ResolvedAt: time.Now(),
Source: "warmblyctl",
}
if err := settings.SetRelease(ctx, next); err != nil {
return err
}
fmt.Printf("Fleet target is now %s. Every node moves to it within a couple of minutes.\n", tag)
fmt.Println("The channel is now `pinned`, so a new release will not override this. `fleet channel stable` resumes following releases.")
return nil
}
func runFleetChannel(ctx context.Context, args []string) error {
fs := newFlagSet("fleet channel")
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() != 1 {
return errors.New("usage: warmblyctl fleet channel <stable|dev|pinned>")
}
channel := fs.Arg(0)
switch channel {
case models.FleetChannelStable, models.FleetChannelDev, models.FleetChannelPinned:
default:
return fmt.Errorf("channel must be stable, dev or pinned, got %q", channel)
}
c, _, settings, _, err := fleetDeps(ctx)
if err != nil {
return err
}
defer c.close()
state, err := settings.GetRelease(ctx)
if err != nil {
return err
}
if state == nil {
state = &models.FleetReleaseState{}
}
state.Channel = channel
if err := settings.SetRelease(ctx, state); err != nil {
return err
}
fmt.Printf("Fleet now follows the %s channel.\n", channel)
if channel != models.FleetChannelPinned {
fmt.Println("The backend resolves the head of that channel on its next release check.")
}
return nil
}
+8
View File
@@ -75,6 +75,8 @@ func dispatch(ctx context.Context, args []string) error {
return runRestore(ctx, args[1:])
case "api":
return runAPI(ctx, args[1:])
case "fleet":
return runFleet(ctx, args[1:])
}
if _, ok := apiFamilies[args[0]]; ok {
return runAPIResource(ctx, args[0], args[1:])
@@ -108,6 +110,12 @@ var commands = []command{
{"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"},
{"org export", "Write a whole workspace to a portable archive file", composeExec + "org export --org you@example.com --out /tmp/workspace.warmbly.zip"},
{"org import", "Apply an archive to a workspace on this instance", composeExec + "org import --org you@example.com --file /tmp/workspace.warmbly.zip"},
{"fleet join-token", "Issue the token a machine needs to join the fleet. Shown once", composeExec + "fleet join-token"},
{"fleet list", "Every worker and consumer: role, version, liveness and usage", composeExec + "fleet list"},
{"fleet version", "Show or set the version every node should be running", composeExec + "fleet version v1.4.2"},
{"fleet channel", "Follow stable or dev releases, or hold the fleet where it is", composeExec + "fleet channel stable"},
{"fleet pin", "Hold one node at a version, to canary or to hold it back", composeExec + "fleet pin <node-id> v1.4.1"},
{"fleet remove", "Forget a node. Its mailboxes re-place themselves", composeExec + "fleet remove <node-id>"},
}
func usage(w *os.File) {
+34 -66
View File
@@ -1,11 +1,8 @@
package main
import (
"bytes"
"context"
"encoding/json"
"log"
"net/http"
"os"
"os/signal"
"strings"
@@ -25,7 +22,9 @@ import (
"github.com/warmbly/warmbly/internal/infrastructure/kafka"
"github.com/warmbly/warmbly/internal/infrastructure/kms"
"github.com/warmbly/warmbly/internal/infrastructure/storage"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/observability"
"github.com/warmbly/warmbly/internal/pkg/nodeagent"
"github.com/warmbly/warmbly/internal/repository"
)
@@ -199,7 +198,7 @@ func main() {
heartbeatDone := make(chan struct{})
go func() {
defer close(heartbeatDone)
runInternalHeartbeat(ctx, workerID, bindIP)
newNodeAgent(workerID, bindIP).Run(ctx)
}()
// Graceful shutdown
@@ -228,75 +227,44 @@ func main() {
log.Println("Worker stopped")
}
func runInternalHeartbeat(ctx context.Context, workerID uuid.UUID, bindIP string) {
baseURL := strings.TrimRight(os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"), "/")
token := os.Getenv("ENCRYPTED_KEYS_WORKER_TOKEN")
if baseURL == "" || token == "" {
return
}
// newNodeAgent builds the shared fleet agent. It replaces the worker's own
// heartbeat loop: identity, usage reporting and self-update are node concerns,
// identical for every role, so they live in one place.
func newNodeAgent(workerID uuid.UUID, bindIP string) *nodeagent.Agent {
reportedIP := os.Getenv("WORKER_PUBLIC_IP")
if reportedIP == "" && bindIP != "default route" {
reportedIP = bindIP
}
if reportedIP == "" {
reportedIP = "unknown"
}
return nodeagent.New(nodeagent.Config{
NodeID: workerID,
Role: models.NodeRoleWorker,
Name: os.Getenv("WARMBLY_NODE_NAME"),
Region: nodeRegion(),
Address: reportedIP,
Version: buildVersion(),
BaseURL: os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"),
Token: os.Getenv("ENCRYPTED_KEYS_WORKER_TOKEN"),
// Written for the host-side updater installed by `warmbly join`.
TargetVersionPath: os.Getenv("WARMBLY_TARGET_VERSION_PATH"),
})
}
client := &http.Client{Timeout: 10 * time.Second}
// reqCtx is separate from ctx so the farewell beat still sends after ctx is
// cancelled by the shutdown signal.
send := func(reqCtx context.Context, booted, stopping bool) {
payload := map[string]any{
"worker_id": workerID.String(),
"bind_ip": reportedIP,
"tier": os.Getenv("WORKER_TIER"),
"egress_kind": os.Getenv("WORKER_EGRESS_KIND"),
}
if booted {
// Mailboxes live in memory only, so a fresh process holds none.
// The backend reloads this worker's mailboxes on this beat
// instead of leaving them to the reconciler's next pass.
payload["booted"] = true
}
if stopping {
payload["stopping"] = true
}
body, _ := json.Marshal(payload)
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, baseURL+"/api/v1/internal/worker/heartbeat", bytes.NewReader(body))
if err != nil {
log.Println("failed to build internal heartbeat:", err)
return
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Println("failed internal heartbeat:", err)
return
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
log.Println("internal heartbeat returned status", resp.StatusCode)
}
// nodeRegion reads the sign-in geography hint. WARMBLY_NODE_REGION is what the
// join script writes and what every role uses; WORKER_REGION is the older
// worker-only name, kept as a fallback so a machine configured by hand before
// the join flow existed keeps reporting its region.
func nodeRegion() string {
if v := os.Getenv("WARMBLY_NODE_REGION"); v != "" {
return v
}
return os.Getenv("WORKER_REGION")
}
send(ctx, true, false)
ticker := time.NewTicker(90 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
// Farewell beat on a fresh context: ctx is already cancelled, and
// without this the row stays selectable until the heartbeat ages
// out, so placement keeps picking a worker that has exited.
byeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
send(byeCtx, false, true)
cancel()
return
case <-ticker.C:
send(ctx, false, false)
}
}
// buildVersion is the image tag this build reports. Set by the join script
// from the tag it pulled; empty means unknown, which the control plane must
// not read as "needs updating".
func buildVersion() string {
return os.Getenv("WARMBLY_VERSION")
}
// uuidNamespaceURL is the RFC 4122 URL namespace, matching the value used by
+1 -2
View File
@@ -483,8 +483,7 @@ services:
# so ids survive container recreates and --scale still works.
WORKER_ID: ${WORKER_ID:-}
WORKER_STATE_DIR: ${WORKER_STATE_DIR:-/data/state}
WORKER_TIER: ${WORKER_TIER:-}
WORKER_EGRESS_KIND: ${WORKER_EGRESS_KIND:-}
WARMBLY_NODE_REGION: ${WARMBLY_NODE_REGION:-}
BOX_GOOGLE_CLIENT_ID: ${BOX_GOOGLE_CLIENT_ID:-}
BOX_GOOGLE_CLIENT_SECRET: ${BOX_GOOGLE_CLIENT_SECRET:-}
BOX_OUTLOOK_CLIENT_ID: ${BOX_OUTLOOK_CLIENT_ID:-}
+5 -1
View File
@@ -696,6 +696,10 @@ Returns a `data` array plus a `pagination` envelope.
}
```
`email_account_id`, `email_account_email` and `email_account_name` are the mailbox the email was actually sent from, which is what a campaign rotating across several mailboxes needs: the row names the mailbox that sent this email, not the campaign's pool or the one that sent the previous step.
`opened_at` is a person's open, as it is in `engagement` and in campaign analytics. A fetch by a mail client's prefetch or a security gateway is reported as `machine_opened_at` instead, so a row is never presented as read by the recipient when only a machine touched it. At most one of the two is present.
## List a contact's timeline
`GET /contacts/:id/timeline`
@@ -839,7 +843,7 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
}
```
`lead_status` uses the same values as the campaign Leads view: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `unsubscribed`, or `undeliverable`. Each step carries whichever of `sent_at`, `opened_at`, `clicked_at`, `replied_at`, `bounced_at` and `failed_at` apply, plus `attempts` and `in_flight` (reserved for a worker whose result has not come back). While a branch condition is undecided, `next.step_id` is absent and `next.step_label` says the step depends on the contact's response.
`lead_status` uses the same values as the campaign Leads view: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `unsubscribed`, or `undeliverable`. Each step carries whichever of `sent_at`, `opened_at`, `clicked_at`, `replied_at`, `bounced_at` and `failed_at` apply, plus `attempts` and `in_flight` (reserved for a worker whose result has not come back). `opened_at` is a person's open, as it is in the Leads view: a step a mail client prefetched or a security gateway scanned carries no `opened_at`. While a branch condition is undecided, `next.step_id` is absent and `next.step_label` says the step depends on the contact's response.
## List a contact's activities
@@ -23,8 +23,8 @@ The landing page. Instance-wide counters and their trends, acquisition by channe
| Page | What it answers |
|---|---|
| **Workers** | Every worker, its heartbeat, tier and the mailboxes placed on it. Detail pages carry stats, the assigned mailbox list, and reassignment |
| **Fleet** | Capacity per worker with utilization, the control loops' decision log (what the placement and rebalance loops did and why), and dedicated worker bindings, with convert and release |
| **Workers** | Every worker, its heartbeat, region and the mailboxes placed on it. Detail pages carry stats, the assigned mailbox list, and reassignment |
| **Fleet** | Capacity per worker with utilization, the control loops' decision log (what the placement and rotation loops did and why), and isolated-egress reservations, with reserve and release |
| **Mailboxes** | Every connected mailbox across every workspace, with its state and the worker it sits on |
| **Sync** | The platform copy of each mailbox's backfill progress and fair-use throttle, with clear-throttle and restart-backfill. The budgets it enforces are the four `sync.*` values under [Configuration > Settings](/development/configuration/#settings-stored-in-the-database); the worker-side engine is the sync governor described under [anti-abuse layers](/development/architecture/#anti-abuse-layers) |
| **Warmup** | The pools, the blocked list, abuse signals (invalid warmup-token attempts), and the block and unblock history |
+42 -2
View File
@@ -90,7 +90,7 @@ From then on, every lifecycle operation (restart, update image, uninstall, rotat
### Why per-VPS instead of a Kubernetes DaemonSet
A k8s `DaemonSet` was the previous shape. It was removed because k8s nodes typically NAT all pods through a small set of egress IPs, defeating the IP-diversity goal. Pods churn but IPs accumulate reputation, so the unit of identity needs to be the IP, not the pod. The worker also has no Postgres dependency, so cluster-level service discovery and RBAC buy us nothing.
A k8s `DaemonSet` was the previous shape. It was removed because k8s nodes typically NAT all pods through a small set of egress IPs, and pods churn while IPs do not. What a mailbox provider remembers is the address an account signs in from, so the unit of identity has to be the IP, not the pod: a mailbox whose client address changes every deploy collects sign-in risk challenges for no reason. The worker also has no Postgres dependency, so cluster-level service discovery and RBAC buy us nothing.
### Identity from IP
@@ -100,6 +100,46 @@ Worker UUID is `UUIDv5(URL_namespace, public_ipv4)`. Properties:
- new IP → new worker (fresh identity, no inherited history)
- deterministic, no state needed at the control plane to recover it
## Worker placement
There is one kind of worker. You install it, it heartbeats, and the control plane decides what runs on it. Workers carry no tier, type, risk pool or egress category, and nothing about a mailbox has to match anything about a machine.
That follows from where the sending identity actually lives. A worker never talks to a recipient's MX. It authenticates to the customer's own mailbox provider (Gmail, Microsoft Graph, or a customer SMTP host) and that provider delivers the message from its own outbound pool. Two consequences shape the whole model:
- **The worker's IP is invisible to recipient spam filtering.** Google strips the submitting client's IP from outgoing messages and Microsoft dropped `X-Originating-IP` years ago. So co-locating a spam-prone mailbox next to a healthy one cannot contaminate the healthy one's sending reputation, and segregating workers by customer tier buys nothing.
- **The worker's IP is very visible to the mailbox provider.** It is what drives sign-in risk challenges, per-IP authentication throttles (`454 4.7.0`) and per-IP rate limits (`421 4.7.28`).
So the levers invert: IP *stability* per mailbox beats IP diversity, and a migration is a cost rather than a win.
### Choosing a worker
Placement scores every live worker and takes the best (`internal/app/worker/placement.go`). Hard constraints are only about whether the work can be done at all: the worker has to be heartbeating, in `healthy` or `watch`, and have capacity headroom for the mailbox's weight. Everything else is a preference:
| Term | Why |
|---|---|
| Capacity headroom | Fill the fleet evenly |
| Incumbency | Staying put keeps one client IP in front of the provider. Weighted highest |
| Region match | Sign-ins from where the provider expects them raise fewer challenges |
| Tenant blast radius | Spread one customer across workers so a single failure does not stop their sending |
| Provider crowding | Many accounts of one provider signing in from one address is what earns a per-IP throttle |
| Foreign tenants | Only for organizations entitled to isolated egress |
Capacity is one number for every worker, in cold-mailbox equivalents, because each mailbox already declares its own cost: an `smtp_imap` mailbox weighs `1.0`, a Gmail or Outlook API mailbox `0.05`, and a warmup-only assignment `0.4`.
### Moving a mailbox
Rotation (`internal/app/worker/rotation.go`) is deliberately reluctant, and a fleet where nothing moves is a healthy one. Only mailboxes whose worker is dead, degraded, over capacity, or on the wrong side of an isolated-egress reservation are considered at all:
| Urgency | Trigger | Residency floor | Destination bar |
|---|---|---|---|
| Immediate | Worker inactive, not heartbeating, blocked or quarantined | none | anything eligible |
| Elevated | Worker throttled | 6 hours | anything eligible |
| Opportunistic | Worker over 85% utilization, or isolated-egress drift | 72 hours | must score materially better |
### Isolated egress
Plans that reserve egress bind an organization to a worker (`dedicated_worker_assignments`). It is a strong placement preference, not a pin: the worker carries no marking, so if it dies the organization's mailboxes place normally instead of stranding, and the rotation loop pulls them onto a replacement once one exists. Mailboxes belonging to other tenants drift off a reserved worker on the same loop.
## Credentials and profiles
Workers never carry hardcoded credentials. Two reusable entities in the admin dashboard:
@@ -141,7 +181,7 @@ Cold-email throughput is mailbox-first, not worker-first. A worker's safe outbou
- default warmup ceiling per mailbox: 40/day
- default warmup ramp: +1/day
Pool model: warmup traffic is segregated into `free` and `premium` pools (the `warmup_pools` tables in `internal/infrastructure/db/migrations/000001_baseline.up.sql`). Dedicated-worker customers still pick a pool explicitly; tier and pool are orthogonal.
Pool model: warmup traffic is segregated into `free` and `premium` pools (the `warmup_pools` tables in `internal/infrastructure/db/migrations/000001_baseline.up.sql`). Pool membership is a property of the mailbox, independent of which worker happens to be sending for it.
Warmup content generation is an offline control-plane workload. The autonomous controller runs every six hours, derives a shared-bank target from seven-day send demand, maintains a 200-thread floor, scales up to 5,000 threads, and submits at most 250 replacements per batch within a 1,000-thread daily cap. It deliberately does not split generation by customer mailbox tags, which would create unbounded queues and smaller, more repetitive content cohorts. Scheduled top-ups use `gpt-5-mini` through durable provider batch jobs rather than synchronous model calls, with a database uniqueness guard that prevents duplicate in-flight scheduled batches when several backend replicas are running. Content with at least 20 sampled sends is automatically archived when it has at least 3 spam placements and a placement rate of 15% or more. The send path atomically selects and increments a least-used active conversation through an indexed query, then falls back to the static reviewed library if no generated content is available. Model availability therefore never gates a warmup send.
+4 -4
View File
@@ -377,7 +377,7 @@ sudo tee /etc/warmbly/worker.env >/dev/null <<EOF
APP_ENV=prod
AWS_CONFIG_ENABLED=false
WORKER_ID=$(uuidgen)
WORKER_TIER=shared_premium
WARMBLY_NODE_REGION=eu-central
EVENTBUS_PROVIDER=nats
NATS_URL=nats://127.0.0.1:4222
@@ -529,7 +529,7 @@ warmblyctl status
The point of workers is to spread sending across machine identities, so most installs eventually add a worker on another host. Two routes, neither needing Docker on the control plane:
**The enrollment installer**, which is what the admin panel's Add Worker flow produces, runs the worker as a container on the remote host. It needs Docker only there. The backend serves it at `GET /worker-install.sh` from `WORKER_INSTALLER_PATH`, which the shipped unit already points at the checkout.
**The join script**, which is what the admin panel's Add a machine flow produces, runs the node as a container on the remote host. It needs Docker and systemd only there. The backend serves it at `GET /join.sh`, and it is embedded in the binary, so there is no path to configure.
**A native worker** is the same binary and env file as above, with the addresses changed. The backend renders a complete env file for an enrollment token, so nothing has to be copied by hand:
@@ -559,9 +559,9 @@ tls {
}
```
Then set `NATS_URL=tls://<token>@nats.yourdomain.com:4222` in `warmbly.env` and `worker.env`; the token in the URL is what every service authenticates with. Redis gets `requirepass` and `REDIS=redis://:<password>@redis.yourdomain.com:6379`, ideally over a private network or a tunnel. Restrict both ports at the firewall to the worker's address as well. Use `BLOB_PROVIDER=s3` with a bucket both sides can reach; a remote worker on the filesystem provider writes to its own disk. The [self-hosting guide](/development/deployment-guide/#remote-workers) has the same caveats in more detail.
Then set `NATS_URL=tls://<token>@nats.yourdomain.com:4222` in `warmbly.env` and `worker.env`; the token in the URL is what every service authenticates with. Redis gets `requirepass` and `REDIS=redis://:<password>@redis.yourdomain.com:6379`, ideally over a private network or a tunnel. Restrict both ports at the firewall to the worker's address as well. Use `BLOB_PROVIDER=s3` with a bucket both sides can reach: a worker reads the message body the backend wrote, so the filesystem provider only works when the two share storage and the permissions line up. The [self-hosting guide](/development/deployment-guide/#adding-a-machine) has the same caveats in more detail.
Because the worker is not in a container, the admin panel's SSH-driven day-two actions (pull image, restart container) do not apply to it. Manage it with `systemctl` and the update steps below.
Because the worker is not in a container, the join script's systemd service and update timer do not apply to it. Manage it with `systemctl` and the update steps below, and set its version by hand rather than through `warmblyctl fleet version`.
## Upgrading
@@ -321,10 +321,8 @@ The variable must be set on the backend in every environment. The file itself is
| `WORKER_ID` | Stable uuid for this worker. Leave unset when running scaled replicas, which share one environment | derived, then random | yes |
| `WORKER_BIND_IP` | Source address to bind outbound connections to, and the seed for a derived `WORKER_ID` | unset | yes |
| `WORKER_PUBLIC_IP` | The address the worker reports to the control plane | detected | yes |
| `WORKER_TIER` | `free`, `premium` or `dedicated`. Tier placement is strict | `free` | yes |
| `WORKER_EGRESS_KIND` | Label describing the worker's egress path | unset | yes |
| `WARMBLY_NODE_REGION` | Free-form label for where this node egresses from, e.g. `eu-central`. Placement prefers a worker near where a mailbox's provider expects sign-ins; unset scores neutral. Written by the join script from `--region` | unset | yes |
| `WORKER_IMAGE` | Image the remote installer pulls. The built-in default does not match what CI publishes, so set it | built-in | yes |
| `WORKER_INSTALLER_PATH` | Path to the installer script the backend serves | built-in | yes |
| `ENCRYPTED_KEYS_BACKEND_URL` | Backend base the worker fetches organization keys from | unset | yes |
| `ENCRYPTED_KEYS_WORKER_TOKEN` | The worker's copy of `INTERNAL_API_TOKEN` | unset | yes |
| `MAIL_TLS_INSECURE` | Skips certificate verification on mailbox connections | `false` | yes |
@@ -547,7 +545,6 @@ Changing them is audited, and every value is validated and clamped server side o
|---|---|
| `KAFKA_CLUSTER` | Nothing. A loader exists but no caller does. Remove it |
| `SENTRY_DSN_API` | Nothing, for the same reason. Superseded by `SENTRY_DSN` |
| `PROVISIONING_DRY_RUN` | It is read, but it cannot be turned off. No real installer adapter is wired yet, so `false` logs a line and is forced back to dry-run rather than creating servers nothing could finish provisioning. `PROVISIONING_RUNNER_ENABLED=false` stops the runner entirely |
| `CAPTCHA_PROVIDER` | Read, but derived when unset rather than defaulting to a constant. See [captcha](#captcha) |
## See also
@@ -695,53 +695,85 @@ WORKER_ID=<a uuid you generate once> # uuidgen
Set it only when you run a single worker per host. Scaled replicas share one environment, so a pinned `WORKER_ID` would make them collide; leave it unset when using `--scale` and let the state volume handle it.
The remote installer handles this for you: it derives the UUID from the machine's public IPv4, so a reinstall on the same IP keeps the same identity and reputation.
The join script handles this for you: it records the node id the control plane assigned and reuses it when you re-run the command, so a rebuild on the same machine keeps the same identity and its mailboxes.
### Remote workers
### Adding a machine
A remote worker VPS must be able to reach the backend URL, NATS (or Kafka), and Redis.
Every Warmbly process that runs on a machine you own is a node. Adding one is two commands: issue a join token, then run one command on the machine.
<Callout type="warn" title="The enrolled worker inherits the backend's addresses">
The config handed to a worker is generated from the backend's own environment. On a stock local install that means `ENCRYPTED_KEYS_BACKEND_URL=http://localhost:8080`, `NATS_URL=nats://nats:4222`, and `REDIS=redis://redis:6379`, none of which resolve on another machine. Before enrolling a remote worker, set `API_PUBLIC_URL`, `NATS_URL`, and `REDIS` on the backend to addresses the VPS can actually reach, and open those ports.
```bash
# On the instance
warmblyctl fleet join-token
# On the new machine (needs Docker, systemd and root)
curl -fsSL https://api.example.com/join.sh | sh -s -- \
--url https://api.example.com \
--token <join-token> \
--role worker \
--region eu-central
```
`--role` is `worker` (sends and syncs mail) or `consumer` (processes events). `--region` is optional and only affects worker placement: a mailbox scores better on a worker near where its provider expects sign-ins.
Nothing connects back to the machine, before or after. It needs no inbound port, no SSH key and no cloud account; the only credential involved is the join token, which is used once and never stored. Re-running the same command on the same machine re-joins it under the same identity, keeping its history and its mailboxes.
The script enrols the node, writes the config the control plane hands back to `/etc/warmbly/node.env`, installs a systemd service and an update timer, and starts it. Add `--dry-run` to see what it would write without changing anything.
<Callout type="warn" title="The node inherits the backend's addresses">
The config handed to a node is generated from the backend's own environment. On a stock local install that means `ENCRYPTED_KEYS_BACKEND_URL=http://localhost:8080`, `NATS_URL=nats://nats:4222`, and `REDIS=redis://redis:6379`, none of which resolve on another machine. Before adding a remote node, set `API_PUBLIC_URL`, `NATS_URL`, and `REDIS` on the backend to addresses the machine can actually reach, and open those ports.
</Callout>
The generated `worker.env` carries the decryption material the worker needs: the internal API token, `KMS_LOCAL_MASTER_KEY`, and `CREDENTIALS_ENCRYPTION_KEY`. It is returned exactly once, over the one-time enrollment token, and the token is rejected on any reuse. Serve the API over HTTPS before enrolling anything across a network you do not control.
That config carries the decryption material the node needs: the internal API token, `KMS_LOCAL_MASTER_KEY`, and `CREDENTIALS_ENCRYPTION_KEY`. Serve the API over HTTPS before adding a node across a network you do not control. It deliberately does not include `PRIMARY_DB`: a worker reaches relational data through the internal API and nothing else.
On `BLOB_PROVIDER=filesystem`, a remote worker writes blobs to its own local disk rather than a volume the backend shares. Use `BLOB_PROVIDER=s3` with a bucket both sides can reach when you run workers off-host.
`BLOB_PROVIDER=filesystem` does not survive a fleet. A worker reads the message body the backend wrote, so the two need the same storage with permissions that let both reach it, and a node on another machine has neither. The join script creates and mounts `BLOB_FS_ROOT` so the node starts, and warns you, but sends will fail when the worker cannot read the body. Use `BLOB_PROVIDER=s3` with a bucket both sides can reach before running nodes off-host.
<Mermaid
chart={`
sequenceDiagram
participant A as Admin panel
participant O as Operator
participant B as Backend
participant V as VPS
A->>B: Create worker (one-time enrollment token)
V->>B: GET /worker-install.sh
V->>B: POST /api/v1/workers/enroll (token)
B-->>V: full worker config (worker.env)
V->>V: systemd unit + docker run
V-->>B: heartbeat every 90s
participant M as Machine
O->>B: warmblyctl fleet join-token
M->>B: GET /join.sh
M->>B: POST /api/v1/fleet/join (token, role)
B-->>M: node id + config + version to run
M->>M: systemd service + update timer
M-->>B: heartbeat
B-->>M: version you should be running
`}
/>
**Token enrollment, one command on the VPS.** Creating a worker through the admin API with `generate_enrollment_token` returns a one-time `wmenroll_...` token. Then:
### Keeping nodes current
Nodes update themselves. Each heartbeat asks the control plane what version it should be running; when the answer differs from what it is running, the host-side timer pulls that image and restarts the service. Nothing is pushed.
```bash
curl -fsSL https://api.example.com/worker-install.sh | sudo bash -s -- \
--enroll wmenroll_... --api-base https://api.example.com
warmblyctl fleet version # what the fleet should be on
warmblyctl fleet version v1.4.2 # move the whole fleet
warmblyctl fleet channel stable # follow releases again
warmblyctl fleet pin <node> v1.4.1 # hold or canary one machine
```
**Or SSH-managed from the admin panel.** Go to Workers, **Add Worker** (host, port, user), paste the generated SSH public key into the VPS's `~/.ssh/authorized_keys`, click **Test connection** (the first success pins the host fingerprint), then **Install**.
Setting a tag also pins the channel, so a release landing later does not silently undo a deliberate rollback. `fleet channel stable` resumes following releases.
Either way the installer installs Docker if it is missing, derives the worker's UUID from the machine's public IPv4 (same IP, same identity and reputation), writes `/etc/warmbly/worker.env`, creates the `warmbly-worker.service` systemd unit, and adds a daily self-update timer. Run it with `--help` for every flag: `--ips` for multi-IP machines, plus `--update`, `--uninstall`, and `--purge`.
The backend is deliberately excluded: it is the thing that tells every node what version to be, so it is upgraded the same way as the rest of your infrastructure. Upgrade the backend, and the fleet follows.
Set `WORKER_IMAGE` on the backend explicitly, for example `ghcr.io/<owner>/warmbly/worker:prod`. Its built-in default does not match what CI publishes.
### Watching the fleet
### Day-2 operations
```bash
warmblyctl fleet list
```
From each worker's detail page in the admin panel: Test connection, Install (idempotent), Restart, Pull latest and restart, Apply config and restart (environment only, no image pull), Live status, Logs, Update OS packages, Reboot, Rotate SSH keys, Uninstall, and Delete.
```
ROLE NAME STATE VERSION REGION MEM SEEN ID
worker box-1 live v1.4.2 eu-central 128MB 12s ago de434ce4-...
consumer events-1 live v1.0.0 -> v1.4.2 - 96MB 30s ago 2b334317-...
```
`STATE` is `live` (beating), `unreachable` (enrolled but silent) or `stopped` (told us it was shutting down). A version shown as `a -> b` is a node that has not picked up the target yet. The same view is in the admin panel under Fleet.
A node leaves rotation the moment it stops: on shutdown it sends a farewell beat that marks it inactive. If it dies without one (hard kill, machine loss), placement stops considering it once its heartbeat goes stale. Either way nothing is placed onto a machine that is no longer answering.
"Apply config" only rewrites the environment file. Use "Pull latest and restart" to move a worker onto a new image.
## Images and releases
@@ -83,6 +83,12 @@ A command that would set a password refuses on a non-TTY unless you passed `--pa
| [`user revoke-admin`](#user-revoke-admin) | Takes platform admin away from an account |
| [`user disable-2fa`](#user-disable-2fa) | Clears an account's authenticator enrolment |
| [`hash-password`](#hash-password) | Prints an argon2 hash for unattended provisioning |
| [`fleet join-token`](#fleet) | Issues the token a machine needs to join the fleet |
| [`fleet list`](#fleet) | Every worker and consumer: role, version, liveness and usage |
| [`fleet version`](#fleet) | Shows or sets the version every node should run |
| [`fleet channel`](#fleet) | Follows stable or dev releases, or holds the fleet |
| [`fleet pin`](#fleet) | Holds one node at a version, to canary or hold it back |
| [`fleet remove`](#fleet) | Forgets a node; its mailboxes re-place themselves |
| [`backup`](#backup) | Writes the whole instance to one restorable bundle |
| [`restore`](#restore) | Restores a bundle onto this instance, replacing everything on it |
| [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size |
@@ -507,3 +513,52 @@ Every one of these runs `warmblyctl` inside the backend container of a compose i
- [Export and import](/guides/workspace-export-import/) for what an archive contains and what deliberately does not travel
- [Configuration reference](/development/configuration/) for every variable named here
- [API authentication](/api/authentication/) and [permissions](/api/permissions/) for the keys and scopes the API commands run on
## fleet
Manages the machines running Warmbly. Every process on a machine you own is a node: a `worker` sends and syncs mail, a `consumer` processes events. Both enrol with a token, heartbeat, and pull the version the control plane tells them to run.
Adding a machine is two commands:
```bash
warmblyctl fleet join-token
```
Then on the machine itself:
```bash
curl -fsSL https://<your-instance>/join.sh | sh -s -- \
--url https://<your-instance> --token <token> --role worker
```
The token is shown once. Issuing another revokes the previous one; machines that already joined are unaffected.
```bash
warmblyctl fleet list
```
```
ROLE NAME STATE VERSION REGION MEM SEEN ID
worker box-1 live v1.4.2 eu-central 128MB 12s ago de434ce4-...
consumer events-1 live v1.0.0 -> v1.4.2 - 96MB 30s ago 2b334317-...
```
`STATE` is `live`, `unreachable` (enrolled but silent) or `stopped`. A version shown as `a -> b` has not picked up the target yet. `--role worker` narrows it; `--json` is machine-readable.
Moving the fleet:
```bash
warmblyctl fleet version # what the fleet should be on
warmblyctl fleet version v1.4.2 # move everything to a tag
warmblyctl fleet channel stable # follow releases again
warmblyctl fleet pin <node-id> v1.4.1 # hold or canary one machine
warmblyctl fleet pin <node-id> # clear that pin
```
Setting a tag also pins the channel, so a release landing later does not undo a deliberate rollback.
```bash
warmblyctl fleet remove <node-id>
```
Forgets a node. Any mailboxes it carried are re-placed within a few minutes. It does not stop the process: a machine still running re-joins on its next heartbeat, so stop the service there too.
+13 -9
View File
@@ -221,19 +221,23 @@ Everything belonging to that mailbox goes with it: its imported mail in the unib
## Worker assignment
You never pick a worker. Warmbly assigns each mailbox to a sending worker automatically and can move it later. Workers are the machines that send and sync, so spreading mailboxes across them spreads sending across network identities and IPs.
You never pick a worker. Warmbly assigns each mailbox to a sending worker automatically and can move it later.
| Workspace | Placement |
|-----------|-----------|
| Free / trial | Shared free-tier workers |
| Paid | Shared premium workers |
| Plans including dedicated workers | A dedicated worker bound to the workspace, allocated on demand, falling back to shared premium and retrying later |
Workers are the machines that connect to your mailbox provider. They are not the address your recipients see: your provider delivers the mail from its own infrastructure, and it strips the connecting client's address on the way out. What the worker's address does decide is how your provider sees the sign-in, which is why Warmbly optimises for **keeping a mailbox on the same worker** rather than spreading it around. A mailbox whose connecting address keeps changing collects security challenges and authentication throttles for nothing.
Within a tier, Warmbly picks the least-loaded healthy worker with capacity rather than packing onto the first one. Each mailbox contributes a load weight, so a worker's planned volume tracks the sum of its mailboxes' budgets.
Every worker is the same kind of worker. There are no tiers to qualify for and no pools to be sorted into. Placement scores the fleet on live signals and picks the best fit:
**Risk is segregated.** A degraded, risky, or quarantined mailbox is never placed on a clean worker beside trusted inboxes, so one struggling mailbox cannot drag down the reputation others depend on.
- how much capacity a worker has left, so the fleet fills evenly
- whether the mailbox is already there, which counts for more than anything else
- whether the worker sits near where your provider expects your sign-ins
- how much of your workspace is already on that one machine, so a single failure never stops all of your sending
- how many other mailboxes on the same provider already sign in from that address
Mailboxes migrate automatically when a trial upgrades, a subscription ends, a workspace moves to or from a dedicated worker, or a risk band changes. No action needed.
Each mailbox contributes a load weight, so a worker's planned volume tracks the sum of its mailboxes' budgets rather than a flat machine limit.
**Mailboxes move only when there is a reason.** A worker going offline, getting blocked, or running out of capacity will move yours. Ordinary rebalancing will not: a settled mailbox has to have been in place for three days before it is even considered, and the alternative has to be meaningfully better. A mailbox that never moves is the ideal, not a stuck one.
Plans that include reserved sending give your workspace a worker no other customer sends from, so your mailboxes always sign in from an address that is yours alone. It is a preference, not a lock: if that machine goes down your mailboxes keep working on the rest of the fleet and return when a replacement is ready.
## Resting a tired mailbox
+2
View File
@@ -47,6 +47,8 @@ Search (press `/`) searches within your current scope, across subjects and the t
Each row is a conversation, not a message, with a badge showing how many messages are inside. Opening it shows every message in order with participants and the owning mailbox. Replies and new inbound mail land in the same thread. Rows group under `Today`, `Yesterday`, `This week`, and `Earlier` headers.
The list loads more conversations as you reach the end of it, and it keeps its place: opening a conversation, or leaving the inbox and coming back, returns you to the row you were on rather than the top of the list.
Threading applies on both sides. A reply you send carries the provider's thread id, which nests it in your own mailbox, and an `In-Reply-To` header naming the last message in the conversation, which is what nests it for the recipient. Their mail client only ever sees the header: a thread id means nothing outside the mailbox that issued it.
<Callout type="info" title="Keyboard navigation">
+7 -7
View File
@@ -34,21 +34,21 @@ Free pools are useful for testing the mechanics but include higher-risk members.
- Keep a separate recovery pool for senders coming back from quarantine.
- Never silently mix free-tier mailboxes into the premium recipient set.
In Warmbly, free and premium pools are physically separated. Dedicated-worker customers still participate in the premium pool by default: dedicated infrastructure is for isolation and IP control, not a reputation shortcut.
In Warmbly, free and premium pools are physically separated. Reserved sending infrastructure does not change that: it is for isolation, not a reputation shortcut.
## What "dedicated" should mean
## What reserved sending means
A dedicated worker is a worker process and IP allocated to your organisation. It gives you:
Plans that include it reserve a worker for your organisation, so your mailboxes always authenticate to their providers from an address no other customer uses. It gives you:
- Predictable sending IP.
- Isolation from other tenants' bad days.
- The ability to negotiate IP allowlisting with specific partners.
- A predictable sign-in address for every one of your mailboxes.
- Isolation from another tenant triggering a per-address authentication throttle.
- The ability to negotiate allowlisting with specific partners.
It does _not_:
- Change the address your recipients see. That belongs to your mailbox provider, not to Warmbly.
- Bypass per-mailbox health rules.
- Let you ignore complaint or bounce thresholds.
- Magically warm a brand-new IP. Dedicated IPs need their own warmup ramp.
## Re-entry after quarantine
-79
View File
@@ -1,79 +0,0 @@
// Admin endpoints binding a worker to a reusable worker profile.
//
// /admin/workers/:id/profile assign / unassign a profile to a worker
// /admin/workers/:id/apply re-write env + restart for a single worker
package handler
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// worker → profile binding
type assignProfileBody struct {
ProfileID *string `json:"profile_id"`
}
func (h *Handler) AdminAssignWorkerProfile(c *gin.Context) {
id, ok := parseUUID(c, "id")
if !ok {
return
}
var body assignProfileBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
var pid *uuid.UUID
if body.ProfileID != nil && *body.ProfileID != "" {
parsed, err := uuid.Parse(*body.ProfileID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid profile_id"))
return
}
pid = &parsed
}
if err := h.WorkerRepo.AssignWorkerProfile(c.Request.Context(), id, pid); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
meta := map[string]string{}
if pid != nil {
meta["profile_id"] = pid.String()
} else {
meta["profile_id"] = "(none)"
}
h.audit(c, models.AuditActionAssign, models.AuditEntityWorker, &id, meta)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
// AdminApplyWorkerConfig re-writes env and restarts the service for ONE
// worker. Useful when the admin wants to pick up the latest profile values
// without re-running the full installer.
func (h *Handler) AdminApplyWorkerConfig(c *gin.Context) {
id, ok := parseUUID(c, "id")
if !ok {
return
}
if err := h.WorkerOrchestrator.ApplyConfig(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionApply, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func parseUUID(c *gin.Context, param string) (uuid.UUID, bool) {
id, err := uuid.Parse(c.Param(param))
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid "+param))
return uuid.Nil, false
}
return id, true
}
+19 -57
View File
@@ -80,17 +80,22 @@ func (h *Handler) AdminFleetDedicated(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"data": rows})
}
// AdminFleetReleaseDedicated is the inverse of AdminConvertWorkerToDedicated:
// the org's mailboxes go back to shared premium workers, the binding is
// released, and the worker re-enters the shared pool once nothing binds it.
// AdminFleetReleaseIsolatedEgress releases an organization's reserved worker
// back to the fleet.
//
// It only deletes the binding. Nothing migrates: the worker carries no
// category to reset, and the org's mailboxes stay where they are until the
// rotation loop finds them a better home on its own schedule. Forcing them to
// move here would re-authenticate every one of them from a new address for no
// deliverability gain.
//
// POST /admin/fleet/dedicated/:orgId/release
func (h *Handler) AdminFleetReleaseDedicated(c *gin.Context) {
func (h *Handler) AdminFleetReleaseIsolatedEgress(c *gin.Context) {
orgID, ok := parseUUIDParam(c, "orgId")
if !ok {
return
}
if h.WorkerAssignmentService == nil || h.WorkerRepo == nil {
if h.WorkerRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "worker placement is not available on this instance"))
return
}
@@ -102,77 +107,34 @@ func (h *Handler) AdminFleetReleaseDedicated(c *gin.Context) {
return
}
if assignment == nil {
errx.JSON(c, errx.New(errx.NotFound, "organization has no active dedicated worker"))
errx.JSON(c, errx.New(errx.NotFound, "organization has no reserved worker"))
return
}
workerID := assignment.WorkerID
before, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID)
// Release the exact row read above by id, so a reservation created
// meanwhile is never touched.
released, err := h.WorkerRepo.ReleaseDedicatedAssignmentByID(ctx, assignment.ID)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error()))
return
}
// Moves every org mailbox onto a live shared premium worker and releases
// the binding; a mailbox with no shared target stays put rather than failing.
if err := h.WorkerAssignmentService.MigrateOrgToShared(ctx, orgID); err != nil {
errx.JSON(c, errx.New(errx.Internal, "migrate to shared: "+err.Error()))
return
}
// MigrateOrgToShared swallows its own release error, so release the exact
// row read above by id: a binding created meanwhile is never touched.
if _, err := h.WorkerRepo.ReleaseDedicatedAssignmentByID(ctx, assignment.ID); err != nil {
errx.JSON(c, errx.New(errx.Internal, "release assignment: "+err.Error()))
return
}
after, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID)
remaining, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error()))
return
}
moved := len(before) - len(after)
if moved < 0 {
moved = 0
}
// The worker only returns to the shared pool when no other org binds it.
stillBound := false
if h.AdminFleetRepo != nil {
active, err := h.AdminFleetRepo.DedicatedAssignments(ctx)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "list assignments: "+err.Error()))
return
}
for _, a := range active {
if a.WorkerID == workerID {
stillBound = true
break
}
}
}
returnedToShared := false
if !stillBound {
if err := h.WorkerRepo.SetWorkerType(ctx, workerID, models.WorkerTypeShared); err != nil {
errx.JSON(c, errx.New(errx.Internal, "set type: "+err.Error()))
return
}
returnedToShared = true
}
h.audit(c, "release_dedicated", models.AuditEntityWorker, &workerID, map[string]string{
h.audit(c, "release_isolated_egress", models.AuditEntityWorker, &workerID, map[string]string{
"organization_id": orgID.String(),
"subscription_id": assignment.SubscriptionID.String(),
"assignment_id": assignment.ID.String(),
"accounts_moved": itoa(moved),
"accounts_remaining": itoa(len(after)),
"returned_to_shared": boolStr(returnedToShared),
"accounts_remaining": itoa(len(remaining)),
})
c.JSON(http.StatusOK, gin.H{
"ok": true,
"ok": released,
"worker_id": workerID,
"accounts_moved": moved,
"accounts_remaining": len(after),
"returned_to_shared": returnedToShared,
"accounts_remaining": len(remaining),
})
}
+66
View File
@@ -0,0 +1,66 @@
package handler
import (
"strconv"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/api/middleware"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// Small helpers shared by the admin handlers. They used to live in the
// SSH worker file, which no longer exists.
func itoa(n int) string { return strconv.Itoa(n) }
func boolStr(b bool) string {
if b {
return "true"
}
return "false"
}
// parseID reads the :id path parameter as a uuid.
func (h *Handler) parseID(c *gin.Context) (uuid.UUID, bool) {
id, err := uuid.Parse(c.Param("id"))
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid id"))
return uuid.Nil, false
}
return id, true
}
// audit fires an admin audit log entry. Writes to admin_audit_log (the table
// the /admin/audit-logs viewer queries) so every operator action is browsable
// alongside ban/unban and the rest.
//
// Fire-and-forget: AdminService spawns its own goroutine, so the response is
// never blocked. Safe to call with a nil entityID and/or nil metadata.
func (h *Handler) audit(c *gin.Context, action models.AuditAction, entity models.AuditEntityType, entityID *uuid.UUID, metadata map[string]string) {
if h.AdminService == nil {
return
}
adminID := middleware.GetAdminUserID(c)
if adminID == nil {
return
}
var details map[string]any
if len(metadata) > 0 {
details = make(map[string]any, len(metadata))
for k, v := range metadata {
details[k] = v
}
}
h.AdminService.LogAdminAction(
c.Request.Context(),
*adminID,
string(action),
string(entity),
entityID,
details,
c.ClientIP(),
c.GetHeader("User-Agent"),
)
}
-692
View File
@@ -1,692 +0,0 @@
// Admin endpoints for the SSH-managed worker lifecycle.
//
// Flow:
// 1. Admin POSTs to /admin/workers with host/port/user + name.
// Backend generates an ed25519 keypair, encrypts the private key via the
// cipher service under the platform identity, stores the row in
// `pending` state, and returns the public key to paste into the VPS's
// ~/.ssh/authorized_keys.
// 2. Admin pastes the pubkey, then POSTs /admin/workers/:id/test.
// Backend opens an SSH session and runs `true`. First successful connect
// pins the host fingerprint (TOFU).
// 3. Admin POSTs /admin/workers/:id/install. Backend uploads the project's
// install-worker.sh + a per-worker env file and runs the installer.
// install_state moves pending → provisioning → installed.
// 4. Admin can then restart / update / uninstall / rotate-keys / get logs /
// get a live status snapshot.
//
// The encrypted private key is never returned over the API.
package handler
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/api/middleware"
"github.com/warmbly/warmbly/internal/app/worker_orchestrator"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// DTOs
type adminCreateWorkerRequest struct {
Name string `json:"name" binding:"required"`
Notes string `json:"notes"`
WorkerType string `json:"worker_type" binding:"required,oneof=shared dedicated"`
FreeTier bool `json:"free_tier"`
SSHHost string `json:"ssh_host" binding:"required"`
SSHPort int `json:"ssh_port"`
SSHUser string `json:"ssh_user"`
GenerateEnrollURL bool `json:"generate_enrollment_token"`
}
type adminCreateWorkerResponse struct {
*models.Worker
// SSHPublicKey is what the admin pastes into the VPS's authorized_keys.
SSHPublicKey string `json:"ssh_public_key"`
EnrollmentToken string `json:"enrollment_token,omitempty"`
EnrollmentTokenTTL int `json:"enrollment_token_ttl_seconds,omitempty"`
}
// handlers
// AdminCreateWorker creates a new SSH-managed worker.
//
// POST /admin/workers
func (h *Handler) AdminCreateWorker(c *gin.Context) {
if h.WorkerOrchestrator == nil || h.WorkerRepo == nil {
errx.JSON(c, errx.New(errx.Internal, "worker orchestrator not configured"))
return
}
var req adminCreateWorkerRequest
if err := c.ShouldBindJSON(&req); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
// A worker is never born dedicated: dedicated capacity is allocated by the
// control plane (by promoting a spare shared worker). Admins create shared
// workers and may convert one later via AdminConvertWorkerToDedicated.
if !repository.IsClientRequestableTier(req.WorkerType) {
errx.JSON(c, errx.New(errx.BadRequest, "dedicated workers cannot be created directly; dedicated capacity is allocated automatically by the control plane — create a shared worker and convert it if needed"))
return
}
if req.SSHPort == 0 {
req.SSHPort = 22
}
if req.SSHUser == "" {
req.SSHUser = "root"
}
pub, priv, err := keypairGen()
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to generate keypair"))
return
}
encPriv, err := h.WorkerOrchestrator.EncryptPrivateKey(c.Request.Context(), priv)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to encrypt key"))
return
}
workerID := uuid.New()
var (
enrollToken string
enrollHash string
enrollExpiresAt *time.Time
)
if req.GenerateEnrollURL {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to generate enrollment token"))
return
}
enrollToken = "wmenroll_" + hex.EncodeToString(raw)
sum := sha256.Sum256([]byte(enrollToken))
enrollHash = hex.EncodeToString(sum[:])
exp := time.Now().Add(2 * time.Hour)
enrollExpiresAt = &exp
}
if err := h.WorkerRepo.CreateWorker(c.Request.Context(), repository.CreateWorkerInput{
ID: workerID,
Name: req.Name,
Notes: req.Notes,
IPAddr: req.SSHHost,
WorkerType: models.WorkerType(req.WorkerType),
FreeTier: req.FreeTier,
SSHHost: req.SSHHost,
SSHPort: req.SSHPort,
SSHUser: req.SSHUser,
SSHPublicKey: pub,
SSHPrivateKeyEncrypted: encPriv,
EnrollmentTokenHash: enrollHash,
EnrollmentTokenExpires: enrollExpiresAt,
}); err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to create worker: "+err.Error()))
return
}
w, xerr := h.fetchWorker(c, workerID)
if xerr != nil {
return
}
resp := adminCreateWorkerResponse{
Worker: w,
SSHPublicKey: pub,
}
if enrollToken != "" {
resp.EnrollmentToken = enrollToken
resp.EnrollmentTokenTTL = int(2 * time.Hour / time.Second)
}
h.audit(c, models.AuditActionCreate, models.AuditEntityWorker, &workerID, map[string]string{
"name": req.Name,
"ssh_host": req.SSHHost,
"tier": req.WorkerType,
})
c.JSON(http.StatusCreated, resp)
}
// AdminListSSHWorkers lists all workers with their install state and last_seen.
//
// GET /admin/workers/managed
func (h *Handler) AdminListSSHWorkers(c *gin.Context) {
if h.WorkerRepo == nil {
errx.JSON(c, errx.New(errx.Internal, "worker repo not configured"))
return
}
workers, err := h.WorkerRepo.ListWorkersDetail(c.Request.Context())
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
// Hydrate tags in one round-trip so the UI gets them in the list response.
ptrs := make([]*models.Worker, len(workers))
for i := range workers {
ptrs[i] = &workers[i]
}
_ = h.WorkerRepo.HydrateWorkerTags(c.Request.Context(), ptrs)
c.JSON(http.StatusOK, gin.H{"data": workers})
}
// AdminGetSSHWorker returns full worker detail.
//
// GET /admin/workers/:id/managed
func (h *Handler) AdminGetSSHWorker(c *gin.Context) {
w, xerr := h.parseAndFetch(c)
if xerr != nil {
return
}
tags, err := h.WorkerRepo.GetWorkerTags(c.Request.Context(), w.ID)
if err == nil {
w.Tags = tags
}
c.JSON(http.StatusOK, w)
}
type setTagsBody struct {
Tags []string `json:"tags"`
}
// AdminSetWorkerTags replaces a worker's tag set.
//
// PUT /admin/workers/:id/tags
func (h *Handler) AdminSetWorkerTags(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
var body setTagsBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
// Normalize: trim, lowercase, dedupe, drop empties. The DB constraint
// rejects garbage tags too; this is just to avoid 500s on the happy path.
seen := map[string]struct{}{}
tags := make([]string, 0, len(body.Tags))
for _, t := range body.Tags {
t = strings.ToLower(strings.TrimSpace(t))
if t == "" {
continue
}
if _, ok := seen[t]; ok {
continue
}
seen[t] = struct{}{}
tags = append(tags, t)
}
if err := h.WorkerRepo.SetWorkerTags(c.Request.Context(), id, tags); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, err.Error()))
return
}
h.audit(c, models.AuditActionUpdate, models.AuditEntityWorker, &id, map[string]string{
"tags": strings.Join(tags, ","),
})
c.JSON(http.StatusOK, gin.H{"ok": true, "tags": tags})
}
// AdminListWorkerTags returns every distinct tag in use, for autocomplete.
//
// GET /admin/workers/tags
func (h *Handler) AdminListWorkerTags(c *gin.Context) {
tags, err := h.WorkerRepo.ListAllWorkerTags(c.Request.Context())
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{"data": tags})
}
// AdminTestWorker runs a no-op SSH command. Pins the host fingerprint on
// first success.
//
// POST /admin/workers/:id/test
func (h *Handler) AdminTestWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.TestConnection(c.Request.Context(), id); err != nil {
h.audit(c, models.AuditActionTest, models.AuditEntityWorker, &id, map[string]string{"ok": "false", "error": err.Error()})
c.JSON(http.StatusOK, gin.H{"ok": false, "error": err.Error()})
return
}
h.audit(c, models.AuditActionTest, models.AuditEntityWorker, &id, map[string]string{"ok": "true"})
c.JSON(http.StatusOK, gin.H{"ok": true})
}
// AdminInstallWorker uploads installer + env file and runs it.
//
// POST /admin/workers/:id/install
func (h *Handler) AdminInstallWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.Install(c.Request.Context(), id); err != nil {
h.audit(c, models.AuditActionInstall, models.AuditEntityWorker, &id, map[string]string{"ok": "false", "error": err.Error()})
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionInstall, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *Handler) AdminRestartWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.Restart(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionRestart, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *Handler) AdminUpdateWorkerImage(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.Update(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionUpgrade, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *Handler) AdminUninstallWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.Uninstall(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionUninstall, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *Handler) AdminWorkerStatusLive(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
st, err := h.WorkerOrchestrator.Status(c.Request.Context(), id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
c.JSON(http.StatusOK, st)
}
func (h *Handler) AdminWorkerLogs(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
lines, _ := strconv.Atoi(c.Query("lines"))
logs, err := h.WorkerOrchestrator.TailLogs(c.Request.Context(), id, lines)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{"logs": logs})
}
func (h *Handler) AdminRotateWorkerKeys(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
newPub, err := h.WorkerOrchestrator.RotateKeys(c.Request.Context(), id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionRotateKeys, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ssh_public_key": newPub})
}
func (h *Handler) AdminSystemUpdate(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
r, err := h.WorkerOrchestrator.SystemUpdate(c.Request.Context(), id)
if err != nil {
// Return the partial output to the client so they can see what failed
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
meta := map[string]string{}
if r != nil && r.RebootRequired {
meta["reboot_required"] = "true"
}
h.audit(c, models.AuditActionSystemUpdate, models.AuditEntityWorker, &id, meta)
c.JSON(http.StatusOK, r)
}
func (h *Handler) AdminRebootWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if err := h.WorkerOrchestrator.RebootWorker(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionReboot, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
// AdminConvertWorkerToDedicated drains a shared worker's accounts to a
// target worker, flips its worker_type to dedicated, and binds it to a
// user/org via dedicated_worker_assignments.
//
// Body:
//
// {
// "organization_id": "uuid", // the org that gets exclusive use
// "subscription_id": "uuid", // their active sub
// "drain_to_worker_id": "uuid|null" // optional: target for evicted accounts.
// // null = let assignment service pick
// // per-account
// }
//
// The whole operation is sequential, not transactional across services —
// if a step fails mid-flight the worker is left in a half-converted state
// and the admin needs to investigate. That's acceptable because the steps
// are individually idempotent: re-running the endpoint with the same
// inputs converges.
type convertToDedicatedBody struct {
OrganizationID string `json:"organization_id" binding:"required"`
SubscriptionID string `json:"subscription_id" binding:"required"`
DrainToWorkerID *string `json:"drain_to_worker_id"`
}
func (h *Handler) AdminConvertWorkerToDedicated(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
var body convertToDedicatedBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
orgID, err := uuid.Parse(body.OrganizationID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid organization_id"))
return
}
subID, err := uuid.Parse(body.SubscriptionID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid subscription_id"))
return
}
w, err := h.WorkerRepo.GetWorkerDetail(c.Request.Context(), id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
if w == nil {
errx.JSON(c, errx.New(errx.NotFound, "worker not found"))
return
}
if w.WorkerType == models.WorkerTypeDedicated {
errx.JSON(c, errx.New(errx.BadRequest, "worker is already dedicated"))
return
}
// Step 1: drain existing accounts to a target. If drain_to_worker_id is
// supplied, move them all there; otherwise we leave reassignment to the
// assignment service which picks per-account based on the source org.
accountIDs, err := h.WorkerRepo.GetEmailAccountsByWorkerID(c.Request.Context(), id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error()))
return
}
movedTo := ""
if body.DrainToWorkerID != nil && *body.DrainToWorkerID != "" {
targetID, perr := uuid.Parse(*body.DrainToWorkerID)
if perr != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid drain_to_worker_id"))
return
}
if targetID == id {
errx.JSON(c, errx.New(errx.BadRequest, "drain target must be a different worker"))
return
}
for _, aid := range accountIDs {
if err := h.WorkerRepo.UpdateEmailAccountWorker(c.Request.Context(), aid, targetID); err != nil {
errx.JSON(c, errx.New(errx.Internal, "drain "+aid.String()+": "+err.Error()))
return
}
_ = h.WorkerRepo.IncrementAccountCount(c.Request.Context(), targetID)
_ = h.WorkerRepo.DecrementAccountCount(c.Request.Context(), id)
}
movedTo = targetID.String()
} else if len(accountIDs) > 0 {
// We don't auto-pick targets here because the right per-account choice
// depends on each account's owning org. If the admin wants that, they
// should pick a single drain target.
errx.JSON(c, errx.New(errx.BadRequest, "worker has accounts; supply drain_to_worker_id to evict them first"))
return
}
// Step 2: flip worker_type to dedicated.
if err := h.WorkerRepo.SetWorkerType(c.Request.Context(), id, models.WorkerTypeDedicated); err != nil {
errx.JSON(c, errx.New(errx.Internal, "set type: "+err.Error()))
return
}
// Step 3: bind to user/org. Idempotent via CreateDedicatedAssignmentIfNotExists.
created, err := h.WorkerRepo.CreateDedicatedAssignmentIfNotExists(c.Request.Context(), &models.DedicatedWorkerAssignment{
ID: uuid.New(),
WorkerID: id,
OrganizationID: orgID,
SubscriptionID: subID,
AssignedAt: time.Now(),
})
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "create assignment: "+err.Error()))
return
}
h.audit(c, "convert_to_dedicated", models.AuditEntityWorker, &id, map[string]string{
"organization_id": orgID.String(),
"subscription_id": subID.String(),
"drained_to": movedTo,
"accounts_moved": itoa(len(accountIDs)),
"new_assignment": boolStr(created),
})
c.JSON(http.StatusOK, gin.H{
"ok": true,
"accounts_drained": len(accountIDs),
"new_assignment": created,
})
}
func itoa(n int) string { return strconv.Itoa(n) }
func boolStr(b bool) string {
if b {
return "true"
}
return "false"
}
type preflightBody struct {
Host string `json:"host" binding:"required"`
Port int `json:"port"`
}
type preflightResult struct {
OK bool `json:"ok"`
LatencyMS int64 `json:"latency_ms,omitempty"`
Error string `json:"error,omitempty"`
}
// AdminPreflightWorker checks TCP reachability of host:port before the
// admin commits to creating a worker row. Catches typos and firewall
// problems while the wizard is still open — much better UX than failing
// later at the SSH test step.
//
// Note: this does NOT attempt SSH handshake. We don't have credentials at
// this stage. A successful TCP dial just means "something is listening
// there"; the SSH test runs after the worker is created and the admin has
// pasted the generated pubkey.
func (h *Handler) AdminPreflightWorker(c *gin.Context) {
var body preflightBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
port := body.Port
if port == 0 {
port = 22
}
addr := net.JoinHostPort(body.Host, strconv.Itoa(port))
start := time.Now()
conn, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(c.Request.Context(), "tcp", addr)
if err != nil {
c.JSON(http.StatusOK, preflightResult{OK: false, Error: err.Error()})
return
}
_ = conn.Close()
c.JSON(http.StatusOK, preflightResult{OK: true, LatencyMS: time.Since(start).Milliseconds()})
}
type setRiskPoolBody struct {
RiskPool string `json:"risk_pool" binding:"required,oneof=clean risky quarantine"`
}
// AdminSetWorkerRiskPool moves a shared worker into a different risk pool.
// The rebalancer will redistribute mailboxes on the next tick — admins
// don't need to migrate accounts manually after this.
func (h *Handler) AdminSetWorkerRiskPool(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
var body setRiskPoolBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
if err := h.WorkerRepo.SetWorkerRiskPool(c.Request.Context(), id, models.WorkerRiskPool(body.RiskPool)); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionUpdate, models.AuditEntityWorker, &id, map[string]string{
"risk_pool": body.RiskPool,
})
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *Handler) AdminDeleteSSHWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
// Best-effort uninstall first. Ignore errors — the row deletion happens
// regardless so an unreachable worker doesn't leave orphan records.
_ = h.WorkerOrchestrator.Uninstall(c.Request.Context(), id)
if err := h.WorkerRepo.DeleteWorker(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionDelete, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{"ok": true})
}
// helpers
func (h *Handler) parseID(c *gin.Context) (uuid.UUID, bool) {
id, err := uuid.Parse(c.Param("id"))
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid worker ID"))
return uuid.Nil, false
}
return id, true
}
// audit fires an admin audit log entry. Writes to admin_audit_log (the
// table the /admin/audit-logs viewer queries) so every action on workers,
// credentials, and releases is browsable alongside ban/unban/etc.
//
// Fire-and-forget — AdminService spawns its own goroutine so the response
// is never blocked. Safe to call with nil entityID and/or nil metadata.
func (h *Handler) audit(c *gin.Context, action models.AuditAction, entity models.AuditEntityType, entityID *uuid.UUID, metadata map[string]string) {
if h.AdminService == nil {
return
}
adminID := middleware.GetAdminUserID(c)
if adminID == nil {
return
}
var details map[string]any
if len(metadata) > 0 {
details = make(map[string]any, len(metadata))
for k, v := range metadata {
details[k] = v
}
}
h.AdminService.LogAdminAction(
c.Request.Context(),
*adminID,
string(action),
string(entity),
entityID,
details,
c.ClientIP(),
c.GetHeader("User-Agent"),
)
}
func (h *Handler) parseAndFetch(c *gin.Context) (*models.Worker, error) {
id, ok := h.parseID(c)
if !ok {
return nil, errors.New("bad id")
}
return h.fetchWorker(c, id)
}
func (h *Handler) fetchWorker(c *gin.Context, id uuid.UUID) (*models.Worker, error) {
w, err := h.WorkerRepo.GetWorkerDetail(c.Request.Context(), id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return nil, err
}
if w == nil {
errx.JSON(c, errx.New(errx.NotFound, "worker not found"))
return nil, errors.New("not found")
}
return w, nil
}
func keypairGen() (pub, priv string, err error) {
return worker_orchestrator.GenerateKeypair()
}
+593
View File
@@ -0,0 +1,593 @@
package handler
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
"os"
"sort"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/fleetnode"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// The fleet is pull-based. A node joins with the instance token, gets the
// config it needs, then heartbeats forever; the reply to that heartbeat is the
// only channel by which the control plane tells it to do anything.
//
// POST /api/v1/fleet/join join token -> node id + env file
// POST /api/v1/internal/fleet/heartbeat internal -> desired version
//
// Nothing here reaches into a machine, which is why onboarding needs no
// keypair, no inbound port and no cloud account.
type fleetJoinRequest struct {
Token string `json:"token" binding:"required"`
Role string `json:"role" binding:"required"`
// NodeID lets a machine keep its identity across a rebuild. Omitted on a
// first join, in which case the control plane assigns one.
NodeID string `json:"node_id,omitempty"`
Name string `json:"name,omitempty"`
Region string `json:"region,omitempty"`
Address string `json:"address,omitempty"`
}
type fleetJoinResponse struct {
NodeID uuid.UUID `json:"node_id"`
Role string `json:"role"`
// EnvB64 is the complete environment file the node should write, base64
// encoded. Rendered from the backend's own configuration, so a node always
// gets exactly the infrastructure the control plane is using.
//
// Base64 rather than a raw JSON string because the join script is POSIX sh
// with no JSON parser: pulling a multi-line value containing quotes and
// backslashes back out with sed is guesswork, and got it wrong. One
// `base64 -d` is exact.
EnvB64 string `json:"env_b64"`
// DesiredVersion is what to run right now, so the first start is already
// on the right version instead of starting stale and updating a beat later.
DesiredVersion string `json:"desired_version,omitempty"`
// HeartbeatSeconds is how often to beat. Derived from the server's liveness
// window so the two can never drift apart.
HeartbeatSeconds int `json:"heartbeat_seconds"`
}
// FleetJoin enrols a node. It is the one endpoint reachable with the join
// token rather than an operator session, because the machine running it has no
// credentials yet — that is the whole point.
func (h *Handler) FleetJoin(c *gin.Context) {
if h.FleetNodes == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
var req fleetJoinRequest
if err := c.ShouldBindJSON(&req); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
ctx := c.Request.Context()
if err := h.FleetNodes.VerifyJoinToken(ctx, req.Token); err != nil {
switch err {
case fleetnode.ErrNoJoinToken:
// Distinct from a wrong token on purpose: this is a setup problem,
// and telling the operator so saves a long hunt.
errx.JSON(c, errx.New(errx.BadRequest, "this instance has no join token yet; issue one from Fleet settings or with `warmblyctl fleet join-token`"))
default:
errx.JSON(c, errx.New(errx.Unauthorized, "join token is not valid"))
}
return
}
role := models.NodeRole(strings.TrimSpace(req.Role))
if !role.Valid() {
errx.JSON(c, errx.New(errx.BadRequest, "role must be worker or consumer"))
return
}
nodeID := uuid.New()
if req.NodeID != "" {
parsed, err := uuid.Parse(req.NodeID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "node_id is not a valid uuid"))
return
}
nodeID = parsed
}
address := strings.TrimSpace(req.Address)
if address == "" {
address = c.ClientIP()
}
// Registering here rather than waiting for the first beat means the node
// shows up in the dashboard the moment it joins, even if it then fails to
// start. A join that silently produces nothing visible is the worst
// possible onboarding experience.
beat := models.NodeHeartbeat{
NodeID: nodeID,
Role: role,
Name: req.Name,
Region: req.Region,
Address: address,
}
reply, err := h.FleetNodes.Heartbeat(ctx, beat)
if err != nil {
if errors.Is(err, fleetnode.ErrRoleChanged) {
errx.JSON(c, errx.New(errx.BadRequest,
"that machine is already enrolled as the other role. Remove the node first, which releases anything assigned to it, then join again."))
return
}
errx.JSON(c, errx.New(errx.Internal, "enrol node: "+err.Error()))
return
}
c.JSON(http.StatusOK, fleetJoinResponse{
NodeID: nodeID,
Role: string(role),
EnvB64: base64.StdEncoding.EncodeToString([]byte(renderNodeEnv(nodeID, role, req.Region))),
DesiredVersion: reply.DesiredVersion,
HeartbeatSeconds: nodeHeartbeatSeconds(reply.LivenessSeconds),
})
}
// FleetHeartbeat records a beat and answers with the version the node should
// be running. Authenticated with INTERNAL_API_TOKEN, the same shared secret
// the node already needs to read encrypted keys.
func (h *Handler) FleetHeartbeat(c *gin.Context) {
if h.FleetNodes == nil {
c.Status(http.StatusNoContent)
return
}
var beat models.NodeHeartbeat
if err := c.ShouldBindJSON(&beat); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "decode body"})
return
}
reply, err := h.FleetNodes.Heartbeat(c.Request.Context(), beat)
if err != nil {
switch {
case errors.Is(err, fleetnode.ErrBadRole):
c.JSON(http.StatusBadRequest, gin.H{"error": "role must be worker or consumer"})
case errors.Is(err, fleetnode.ErrRoleChanged):
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
default:
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
}
// A worker that just booted holds no mailboxes: they live in memory only.
// Reload them now instead of leaving it to the reconciler's next pass,
// during which every send to it would fail with "not found".
if beat.Booted && beat.Role == models.NodeRoleWorker && h.EmailService != nil {
// Off the request: the reload publishes one ADD_EMAIL per mailbox.
go func(id uuid.UUID) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
h.EmailService.ReloadWorkerAccounts(ctx, id)
}(beat.NodeID)
}
c.JSON(http.StatusOK, reply)
}
// nodeHeartbeatSeconds derives the beat interval from the server's liveness
// window: beat three times per window, so two lost beats still do not look
// like a dead machine.
func nodeHeartbeatSeconds(livenessSeconds int) int {
if livenessSeconds <= 0 {
return 90
}
n := livenessSeconds / 3
if n < 15 {
return 15
}
return n
}
// nodeEnvKeys are the settings a node needs to do its job, in the order they
// are written. Every one is read from the backend's own environment, so a node
// is configured with exactly the infrastructure the control plane uses and
// there is nothing to keep in sync by hand.
//
// Deliberately excluded: PRIMARY_DB and anything else that would give a node
// direct database access. Workers reach relational data through the internal
// API and nothing else, and handing them a DSN here would quietly undo that.
var nodeEnvKeys = []string{
"APP_ENV",
"EVENTBUS_PROVIDER",
"NATS_URL",
"KAFKA_BOOTSTRAP_SERVERS",
"KAFKA_SASL_USERNAME",
"KAFKA_SASL_PASSWORD",
"SCHEMA_REGISTRY_URL",
"SCHEMA_REGISTRY_KEY",
"SCHEMA_REGISTRY_SECRET",
"CODEC_PROVIDER",
"REDIS",
"KMS_PROVIDER",
"KMS_LOCAL_MASTER_KEY",
"KMS_KEY_ID",
"CREDENTIALS_ENCRYPTION_KEY",
"BLOB_PROVIDER",
"BLOB_FS_ROOT",
"AWS_REGION",
"S3_BUCKET",
"BOX_GOOGLE_CLIENT_ID",
"BOX_GOOGLE_CLIENT_SECRET",
"BOX_OUTLOOK_CLIENT_ID",
"BOX_OUTLOOK_CLIENT_SECRET",
"MAIL_TLS_INSECURE",
"SENTRY_DSN",
}
// renderNodeEnv builds the env file a node writes to disk on join.
func renderNodeEnv(nodeID uuid.UUID, role models.NodeRole, region string) string {
var b strings.Builder
b.WriteString("# Written by `warmbly join`. Regenerate by joining again.\n")
fmt.Fprintf(&b, "WARMBLY_NODE_ID=%s\n", nodeID)
fmt.Fprintf(&b, "WARMBLY_NODE_ROLE=%s\n", role)
if region != "" {
fmt.Fprintf(&b, "WARMBLY_NODE_REGION=%s\n", region)
}
// A worker resolves its own identity from WORKER_ID; keeping the two equal
// means the placement row and the node row are the same machine.
if role == models.NodeRoleWorker {
fmt.Fprintf(&b, "WORKER_ID=%s\n", nodeID)
}
backend := strings.TrimRight(os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"), "/")
if backend == "" {
backend = strings.TrimRight(os.Getenv("APP_INTERNAL_URL"), "/")
}
fmt.Fprintf(&b, "WARMBLY_BACKEND_URL=%s\n", backend)
fmt.Fprintf(&b, "ENCRYPTED_KEYS_PROVIDER=%s\n", "http")
fmt.Fprintf(&b, "ENCRYPTED_KEYS_BACKEND_URL=%s\n", backend)
fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN"))
fmt.Fprintf(&b, "INTERNAL_API_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN"))
for _, k := range nodeEnvKeys {
if v := os.Getenv(k); v != "" {
fmt.Fprintf(&b, "%s=%s\n", k, v)
}
}
return b.String()
}
// ---- admin ----
// AdminFleetNodes lists the fleet: every worker and consumer, what version
// each is on, what it should be on, and what it is using.
func (h *Handler) AdminFleetNodes(c *gin.Context) {
if h.FleetNodes == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
role := models.NodeRole(c.Query("role"))
if role != "" && !role.Valid() {
errx.JSON(c, errx.New(errx.BadRequest, "role must be worker or consumer"))
return
}
nodes, err := h.FleetNodes.List(c.Request.Context(), role)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
sort.SliceStable(nodes, func(i, j int) bool { return nodes[i].Role < nodes[j].Role })
c.JSON(http.StatusOK, gin.H{"data": nodes})
}
// AdminFleetIssueJoinToken mints a new instance join token and returns it
// once. Issuing replaces the previous one, which is also how it is revoked.
func (h *Handler) AdminFleetIssueJoinToken(c *gin.Context) {
if h.FleetNodes == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
token, err := h.FleetNodes.IssueJoinToken(c.Request.Context())
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, "fleet_join_token_issued", models.AuditEntityWorker, nil, nil)
c.JSON(http.StatusOK, gin.H{
"token": token,
"note": "Shown once. Issuing a new token revokes this one; nodes already enrolled are unaffected.",
})
}
// ---- node tags ----
type setTagsBody struct {
Tags []string `json:"tags"`
}
// AdminListWorkerTags returns every tag in use, for the filter menu.
func (h *Handler) AdminListWorkerTags(c *gin.Context) {
tags, err := h.WorkerRepo.ListAllWorkerTags(c.Request.Context())
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{"data": tags})
}
// AdminSetWorkerTags replaces a node's operator-applied tags.
func (h *Handler) AdminSetWorkerTags(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
var body setTagsBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
// Normalise: trim, lowercase, dedupe, drop empties. The DB constraint
// rejects garbage too; this just avoids 500s on the happy path.
seen := map[string]struct{}{}
tags := make([]string, 0, len(body.Tags))
for _, t := range body.Tags {
t = strings.ToLower(strings.TrimSpace(t))
if t == "" {
continue
}
if _, dup := seen[t]; dup {
continue
}
seen[t] = struct{}{}
tags = append(tags, t)
}
if err := h.WorkerRepo.SetWorkerTags(c.Request.Context(), id, tags); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, err.Error()))
return
}
h.audit(c, models.AuditActionUpdate, models.AuditEntityWorker, &id, map[string]string{
"tags": strings.Join(tags, ","),
})
c.JSON(http.StatusOK, gin.H{"ok": true, "tags": tags})
}
// ---- isolated egress ----
type reserveWorkerBody struct {
OrganizationID string `json:"organization_id" binding:"required"`
SubscriptionID string `json:"subscription_id" binding:"required"`
}
// AdminFleetReserveWorker reserves a worker for one organization, so its
// mailboxes always sign in from an address no other tenant uses.
//
// It only writes the binding. Mailboxes already on the worker are not evicted
// here: the rotation loop moves other tenants off on its own schedule, and
// forcing them now would re-authenticate every one of them at once for no
// deliverability gain.
func (h *Handler) AdminFleetReserveWorker(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if h.WorkerRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "worker placement is not available on this instance"))
return
}
var body reserveWorkerBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
orgID, err := uuid.Parse(body.OrganizationID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid organization_id"))
return
}
subID, err := uuid.Parse(body.SubscriptionID)
if err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid subscription_id"))
return
}
ctx := c.Request.Context()
w, err := h.WorkerRepo.GetWorkerDetail(ctx, id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
if w == nil {
errx.JSON(c, errx.New(errx.NotFound, "worker not found"))
return
}
created, err := h.WorkerRepo.CreateDedicatedAssignmentIfNotExists(ctx, &models.DedicatedWorkerAssignment{
ID: uuid.New(),
WorkerID: id,
OrganizationID: orgID,
SubscriptionID: subID,
AssignedAt: time.Now(),
})
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "create reservation: "+err.Error()))
return
}
h.audit(c, "fleet_reserve_worker", models.AuditEntityWorker, &id, map[string]string{
"organization_id": orgID.String(),
"subscription_id": subID.String(),
"new_reservation": boolStr(created),
})
c.JSON(http.StatusOK, gin.H{"ok": true, "worker_id": id, "new_reservation": created})
}
// AdminFleetDeleteNode forgets a node.
//
// Its worker row cascades and its mailboxes are released, so they are re-placed
// on a live worker within a rotation pass rather than stranded. It does not
// stop anything on the machine: a node whose process is still running will
// re-join on its next heartbeat, which is deliberate — removing a row should
// not be a way to lose track of a machine that is still sending.
func (h *Handler) AdminFleetDeleteNode(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if h.FleetNodeRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
if err := h.FleetNodeRepo.Delete(c.Request.Context(), id); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, models.AuditActionDelete, models.AuditEntityWorker, &id, nil)
c.JSON(http.StatusOK, gin.H{
"ok": true,
"note": "Any mailboxes it carried will be re-placed within a few minutes. " +
"Stop the service on that machine too, or it will re-join on its next heartbeat.",
})
}
type patchNodeBody struct {
Name *string `json:"name,omitempty"`
Notes *string `json:"notes,omitempty"`
// PinnedVersion holds this node at a version. An empty string clears the
// pin and returns it to the fleet target.
PinnedVersion *string `json:"pinned_version,omitempty"`
}
// AdminFleetPatchNode applies the only things an operator sets on a node: what
// it is called, a note, and whether it is held at a version.
func (h *Handler) AdminFleetPatchNode(c *gin.Context) {
id, ok := h.parseID(c)
if !ok {
return
}
if h.FleetNodeRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
var body patchNodeBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
ctx := c.Request.Context()
changed := map[string]string{}
if body.Name != nil {
if err := h.FleetNodeRepo.SetName(ctx, id, *body.Name); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
changed["name"] = *body.Name
}
if body.Notes != nil {
if err := h.FleetNodeRepo.SetNotes(ctx, id, *body.Notes); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
changed["notes"] = *body.Notes
}
if body.PinnedVersion != nil {
if err := h.FleetNodeRepo.SetPinnedVersion(ctx, id, *body.PinnedVersion); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
changed["pinned_version"] = *body.PinnedVersion
}
if len(changed) == 0 {
errx.JSON(c, errx.New(errx.BadRequest, "nothing to change"))
return
}
h.audit(c, models.AuditActionUpdate, models.AuditEntityWorker, &id, changed)
node, err := h.FleetNodes.Get(ctx, id)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
c.JSON(http.StatusOK, node)
}
// AdminFleetRelease reports, and optionally sets, the version the fleet should
// converge on.
func (h *Handler) AdminFleetRelease(c *gin.Context) {
if h.FleetSettingsRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
state, err := h.FleetSettingsRepo.GetRelease(c.Request.Context())
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
if state == nil {
state = &models.FleetReleaseState{Channel: models.FleetChannelStable}
}
c.JSON(http.StatusOK, state)
}
type setReleaseBody struct {
Channel string `json:"channel,omitempty"`
Tag string `json:"tag,omitempty"`
}
// AdminFleetSetRelease moves the whole fleet. Setting a tag pins the channel
// too, so the next release check does not immediately undo a deliberate
// rollback.
func (h *Handler) AdminFleetSetRelease(c *gin.Context) {
if h.FleetSettingsRepo == nil {
errx.JSON(c, errx.New(errx.NotImplemented, "fleet enrolment is not available on this instance"))
return
}
var body setReleaseBody
if err := c.ShouldBindJSON(&body); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
ctx := c.Request.Context()
state, err := h.FleetSettingsRepo.GetRelease(ctx)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
if state == nil {
state = &models.FleetReleaseState{}
}
if body.Tag != "" {
state.Tag = strings.TrimSpace(body.Tag)
state.Channel = models.FleetChannelPinned
state.ResolvedAt = time.Now()
state.Source = "admin"
} else if body.Channel != "" {
switch body.Channel {
case models.FleetChannelStable, models.FleetChannelDev, models.FleetChannelPinned:
state.Channel = body.Channel
default:
errx.JSON(c, errx.New(errx.BadRequest, "channel must be stable, dev or pinned"))
return
}
} else {
errx.JSON(c, errx.New(errx.BadRequest, "supply a channel or a tag"))
return
}
if err := h.FleetSettingsRepo.SetRelease(ctx, state); err != nil {
errx.JSON(c, errx.New(errx.Internal, err.Error()))
return
}
h.audit(c, "fleet_release_set", models.AuditEntityWorker, nil, map[string]string{
"channel": state.Channel,
"tag": state.Tag,
})
c.JSON(http.StatusOK, state)
}
+12 -14
View File
@@ -26,6 +26,7 @@ import (
"github.com/warmbly/warmbly/internal/app/emailsend"
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
"github.com/warmbly/warmbly/internal/app/feature"
"github.com/warmbly/warmbly/internal/app/fleetnode"
"github.com/warmbly/warmbly/internal/app/form"
"github.com/warmbly/warmbly/internal/app/group"
"github.com/warmbly/warmbly/internal/app/instancecheck"
@@ -68,7 +69,6 @@ import (
"github.com/warmbly/warmbly/internal/app/webhook"
"github.com/warmbly/warmbly/internal/app/websitetracking"
"github.com/warmbly/warmbly/internal/app/worker"
"github.com/warmbly/warmbly/internal/app/worker_orchestrator"
"github.com/warmbly/warmbly/internal/pkg/generation"
"github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys"
@@ -158,10 +158,10 @@ type Handler struct {
AdminService admin.AdminService
AdminOutreachService adminoutreach.Service
// Worker orchestration (SSH-driven lifecycle for admin-managed workers)
WorkerOrchestrator *worker_orchestrator.Orchestrator
WorkerRepo repository.WorkerRepository
CredentialsRepo repository.CredentialsRepository
// Fleet. Nodes enrol with the join token and pull everything else; the
// control plane never reaches into a machine.
FleetNodes *fleetnode.Service
WorkerRepo repository.WorkerRepository
// UpdatesService backs the admin panel's update indicator and button.
UpdatesService *updates.Service
@@ -291,15 +291,13 @@ type Handler struct {
// Direct repositories used by handlers that don't yet have a
// service layer (avatars, etc.). Keep narrow and add a service
// only when business logic accumulates.
UserRepo repository.UserRepository
OrgRepo repository.OrganizationRepository
AttachmentRepo repository.AttachmentRepository
EmailImageRepo repository.EmailImageRepository
StorageBackendRepo repository.StorageBackendRepository
CloudCredentialRepo repository.CloudCredentialRepository
ProvisioningTemplateRepo repository.ProvisioningTemplateRepository
ProvisioningJobRepo repository.ProvisioningJobRepository
ProvisioningPolicyRepo repository.ProvisioningPolicyRepository
UserRepo repository.UserRepository
OrgRepo repository.OrganizationRepository
AttachmentRepo repository.AttachmentRepository
EmailImageRepo repository.EmailImageRepository
StorageBackendRepo repository.StorageBackendRepository
FleetNodeRepo repository.FleetNodeRepository
FleetSettingsRepo repository.FleetSettingsRepository
// Danger zone (delayed deletions for orgs & user accounts)
DangerZoneService dangerzone.Service
+8 -93
View File
@@ -1,35 +1,28 @@
package handler
import (
"context"
"encoding/json"
"io"
"net/http"
"os"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/infrastructure/kafka"
"github.com/warmbly/warmbly/internal/repository"
)
// Internal worker bootstrap + config endpoints. A worker process starts with
// a tiny envelope on disk and pulls everything else from here on boot:
// Internal worker config endpoint. A worker starts with the env file it wrote
// at join time and pulls the rest from here on boot:
//
// GET /api/v1/worker/config -> WorkerConfig JSON
// POST /api/v1/worker/heartbeat -> 204, auto-registers a new worker on
// first contact
// GET /api/v1/internal/worker/config -> WorkerConfig JSON
//
// Auth: shared bearer token (INTERNAL_API_TOKEN). Future upgrade is per-worker
// JWTs minted at registration time so tier comes from token claims rather
// than the heartbeat body.
// Heartbeats go to the role-agnostic /internal/fleet/heartbeat instead, which
// is what tells a node the version it should be running.
//
// Auth: shared bearer token (INTERNAL_API_TOKEN).
type WorkerEgressConfig struct {
ID uuid.UUID `json:"id"`
BindIP string `json:"bind_ip"`
Hostname string `json:"hostname"`
Tier string `json:"tier"`
Tags []string `json:"tags,omitempty"`
}
@@ -78,7 +71,6 @@ func (h *Handler) InternalWorkerConfig(c *gin.Context) {
ID: id,
BindIP: bindIP,
Hostname: tag,
Tier: "shared",
},
},
Kafka: WorkerKafkaConfig{
@@ -100,84 +92,7 @@ func (h *Handler) InternalWorkerConfig(c *gin.Context) {
c.JSON(http.StatusOK, cfg)
}
// HeartbeatPayload is what a worker sends on every heartbeat. The first
// heartbeat from an unknown WorkerID triggers auto-registration in the
// workers table; subsequent heartbeats just refresh last_seen.
type HeartbeatPayload struct {
WorkerID string `json:"worker_id"`
BindIP string `json:"bind_ip"`
Tier string `json:"tier,omitempty"` // shared_free | shared_premium (dedicated is rejected; allocated by the control plane)
EgressKind string `json:"egress_kind,omitempty"` // cold_smtp | oauth_api | warmup_only
// Stopping is set on the farewell beat a worker sends as it shuts down, so
// the row goes inactive at once instead of staying selectable until its
// heartbeat ages out. Placement would otherwise keep handing work to a
// process that is already gone.
Stopping bool `json:"stopping,omitempty"`
// Booted is set on the first beat of a fresh process. A worker holds its
// mailboxes in memory only, so the backend reloads every mailbox assigned
// to it right away; until then each send to it fails with "not found".
Booted bool `json:"booted,omitempty"`
}
func (h *Handler) InternalWorkerHeartbeat(c *gin.Context) {
body, err := io.ReadAll(c.Request.Body)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "read body"})
return
}
var p HeartbeatPayload
if err := json.Unmarshal(body, &p); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "decode body"})
return
}
id, err := uuid.Parse(p.WorkerID)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "valid worker_id required"})
return
}
if p.BindIP == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "bind_ip required"})
return
}
// A worker may only register as a shared tier. Dedicated capacity is
// allocated by the control plane (by promoting a spare shared worker), so
// a worker must never self-designate as dedicated. A blank tier is fine —
// it maps to the shared-premium default.
if !repository.IsClientRequestableTier(p.Tier) {
c.JSON(http.StatusBadRequest, gin.H{
"error": "dedicated tier cannot be requested by a worker; dedicated capacity is allocated automatically by the control plane",
"code": "tier_not_allowed",
})
return
}
if h.WorkerRepo == nil {
// Worker repository not wired (e.g. tests). Treat as 204 noop.
c.Status(http.StatusNoContent)
return
}
if p.Stopping {
if err := h.WorkerRepo.DeactivateWorker(c.Request.Context(), id); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.Status(http.StatusNoContent)
return
}
if err := h.WorkerRepo.UpsertOnHeartbeat(c.Request.Context(), id, p.BindIP, p.Tier, p.EgressKind); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if p.Booted && h.EmailService != nil {
// Off the request: the reload publishes one ADD_EMAIL per mailbox.
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
h.EmailService.ReloadWorkerAccounts(ctx, id)
}()
}
c.Status(http.StatusNoContent)
}
// envOr reads an environment variable with a fallback.
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
+28
View File
@@ -0,0 +1,28 @@
package handler
import (
_ "embed"
"net/http"
"github.com/gin-gonic/gin"
)
// The join script is served by the instance itself, not from warmbly.com, so a
// self-hosted fleet never depends on a vendor host being reachable and always
// gets a script that matches the backend it is joining.
//
//go:embed nodescript/join.sh
var joinScript string
// ServeJoinScript is the front door for adding a machine to the fleet:
//
// curl -fsSL https://<instance>/join.sh | sh -s -- --url ... --token ... --role worker
//
// Unauthenticated on purpose. The script itself is not a secret and does
// nothing without a valid join token; requiring credentials to read it would
// only mean pasting them twice.
func (h *Handler) ServeJoinScript(c *gin.Context) {
c.Header("Content-Type", "text/x-shellscript; charset=utf-8")
c.Header("Cache-Control", "no-store")
c.String(http.StatusOK, joinScript)
}
+510
View File
@@ -0,0 +1,510 @@
#!/bin/sh
# Join this machine to a Warmbly fleet.
#
# curl -fsSL https://<your-instance>/join.sh | sh -s -- \
# --url https://<your-instance> --token <join-token> --role worker
#
# It asks the control plane to enrol the machine, writes the config the control
# plane hands back, installs a systemd service for the role and a timer that
# keeps it on the version the control plane wants, and starts it.
#
# Nothing is pushed to this machine, before or after. No inbound port is
# opened, no SSH key is installed, and the only credential involved is the join
# token, which is used once and never stored.
#
# POSIX sh: this runs under whatever /bin/sh the host has, which on Debian and
# Ubuntu is dash.
set -eu
WARMBLY_URL=""
WARMBLY_TOKEN=""
WARMBLY_ROLE=""
WARMBLY_REGION=""
WARMBLY_NAME=""
WARMBLY_IMAGE_REPO="ghcr.io/warmbly/warmbly"
CONFIG_DIR="/etc/warmbly"
STATE_DIR="/var/lib/warmbly"
# What the container may write. Kept separate from STATE_DIR because STATE_DIR
# also holds image-ref, which systemd feeds to a root `docker run`: anything
# the node can rewrite there would choose the image root then executes.
AGENT_DIR="/var/lib/warmbly/node"
DRY_RUN="false"
PRINT_UNIT="false"
log() { printf '%s\n' "$*"; }
warn() { printf '%s\n' "$*" >&2; }
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
usage() {
cat <<'USAGE'
Join a machine to a Warmbly fleet.
--url <url> Your Warmbly instance, e.g. https://app.example.com (required)
--token <token> Fleet join token. Issue one in Fleet settings, or with
`warmblyctl fleet join-token`. (required)
--role <role> worker | consumer (required)
--region <label> Where this machine egresses from, e.g. eu-central.
Placement prefers a worker near where a mailbox's
provider expects sign-ins. Optional.
--name <name> Display name in the dashboard. Defaults to the hostname.
--image-repo <r> Container image repository. Defaults to
ghcr.io/warmbly/warmbly.
--config-dir <d> Where to write the env file. Default /etc/warmbly.
--dry-run Enrol and print what would be written, change nothing.
--print-unit Print the systemd unit that would be installed and exit.
Contacts nothing and writes nothing; used by
`make join-check`.
-h, --help This text.
A second run re-enrols the same machine: it keeps the existing node id, so the
node keeps its identity, history and mailbox placements.
USAGE
}
parse_args() {
while [ $# -gt 0 ]; do
case "$1" in
--url) WARMBLY_URL="${2:-}"; shift 2 ;;
--token) WARMBLY_TOKEN="${2:-}"; shift 2 ;;
--role) WARMBLY_ROLE="${2:-}"; shift 2 ;;
--region) WARMBLY_REGION="${2:-}"; shift 2 ;;
--name) WARMBLY_NAME="${2:-}"; shift 2 ;;
--image-repo) WARMBLY_IMAGE_REPO="${2:-}"; shift 2 ;;
--config-dir) CONFIG_DIR="${2:-}"; shift 2 ;;
--dry-run) DRY_RUN="true"; shift ;;
--print-unit) PRINT_UNIT="true"; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown option: $1 (try --help)" ;;
esac
done
}
require_args() {
[ -n "$WARMBLY_URL" ] || die "--url is required"
[ -n "$WARMBLY_TOKEN" ] || die "--token is required"
[ -n "$WARMBLY_ROLE" ] || die "--role is required (worker or consumer)"
case "$WARMBLY_ROLE" in
worker|consumer) ;;
*) die "--role must be worker or consumer, got '$WARMBLY_ROLE'" ;;
esac
[ -n "$WARMBLY_NAME" ] || WARMBLY_NAME="$(hostname 2>/dev/null || echo warmbly-node)"
# Trim a trailing slash so the URLs we build never double up.
WARMBLY_URL="${WARMBLY_URL%/}"
}
need_cmd() {
command -v "$1" >/dev/null 2>&1 || die "$1 is required but not installed"
}
check_deps() {
need_cmd curl
if [ "$DRY_RUN" = "false" ]; then
command -v docker >/dev/null 2>&1 || die "docker is required but not installed. Install it, then re-run."
command -v systemctl >/dev/null 2>&1 || die "systemd is required (this script installs a service and a timer)"
[ "$(id -u)" = "0" ] || die "run as root: this writes to $CONFIG_DIR and installs a systemd unit"
fi
}
# json_field extracts a top-level string field. The join response is generated
# by our own backend and is a flat object, so this stays honest without pulling
# in a JSON parser the host may not have.
json_field() {
# shellcheck disable=SC2016
sed -n 's/.*"'"$1"'"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1
}
# b64decode reads base64 on stdin. coreutils is the norm; openssl is the
# fallback for the images that ship without it.
b64decode() {
if command -v base64 >/dev/null 2>&1; then
base64 -d
else
openssl base64 -d -A
fi
}
existing_node_id() {
if [ -f "$CONFIG_DIR/node.env" ]; then
sed -n 's/^WARMBLY_NODE_ID=//p' "$CONFIG_DIR/node.env" | head -n 1
fi
}
enrol() {
prior="$(existing_node_id)"
if [ -n "$prior" ]; then
log "Re-joining as existing node $prior"
fi
body=$(printf '{"token":"%s","role":"%s","region":"%s","name":"%s","node_id":"%s"}' \
"$WARMBLY_TOKEN" "$WARMBLY_ROLE" "$WARMBLY_REGION" "$WARMBLY_NAME" "$prior")
tmp="$(mktemp)"
code=$(curl -sS -o "$tmp" -w '%{http_code}' \
-X POST "$WARMBLY_URL/api/v1/fleet/join" \
-H 'Content-Type: application/json' \
-d "$body" || echo 000)
if [ "$code" = "000" ]; then
rm -f "$tmp"
die "could not reach $WARMBLY_URL. Check the URL and that this machine can reach it."
fi
if [ "$code" != "200" ]; then
detail=$(json_field message < "$tmp")
[ -n "$detail" ] || detail=$(cat "$tmp")
rm -f "$tmp"
case "$code" in
401) die "the join token was rejected. Issue a fresh one and try again." ;;
*) die "enrolment failed (HTTP $code): $detail" ;;
esac
fi
NODE_ID=$(json_field node_id < "$tmp")
DESIRED_VERSION=$(json_field desired_version < "$tmp")
# The env file arrives base64 encoded, so a shell with no JSON parser can
# recover it exactly. Decoding is one command; picking a multi-line,
# quote-bearing value back out of JSON with sed is guesswork.
NODE_ENV=$(json_field env_b64 < "$tmp" | b64decode)
rm -f "$tmp"
[ -n "$NODE_ID" ] || die "the control plane did not return a node id"
[ -n "$NODE_ENV" ] || die "the control plane returned no configuration for this node"
[ -n "$DESIRED_VERSION" ] || DESIRED_VERSION="latest"
log "Enrolled as $WARMBLY_ROLE node $NODE_ID"
}
write_config() {
if [ "$DRY_RUN" = "true" ]; then
log ""
log "--dry-run: would write $CONFIG_DIR/node.env with:"
printf '%s\n' "$NODE_ENV" | sed 's/\(TOKEN=\|KEY=\|SECRET=\|PASSWORD=\).*/\1***/' | sed 's/^/ /'
log ""
log "--dry-run: would run image $WARMBLY_IMAGE_REPO/$WARMBLY_ROLE:$DESIRED_VERSION"
return 0
fi
mkdir -p "$CONFIG_DIR" "$STATE_DIR" "$AGENT_DIR"
# The node container runs as uid 1000 (deploy/docker/worker.Dockerfile), so
# the directory it writes into has to be owned by that uid, or its
# target-version write fails with EACCES, which it only logs, and auto-update
# silently never happens.
#
# Only this subdirectory, never STATE_DIR itself: a recursive chown there
# would re-own a bare-metal install's BLOB_FS_ROOT, and making STATE_DIR
# container-writable would let the node rewrite the image reference that
# systemd hands to a root `docker run --network host`.
chown 1000:1000 "$AGENT_DIR" 2>/dev/null || true
chmod 0700 "$AGENT_DIR"
# uid 1000 needs traverse on the parent to reach it. Readable and executable,
# never writable: image-ref lives here and systemd feeds it to a root
# `docker run`, so the node must not be able to replace it.
chmod 0755 "$STATE_DIR"
# Converge a machine joined by an earlier version of this script, which
# chowned the whole tree to uid 1000 and so left image-ref rewritable by the
# node. Re-owning is idempotent and cheap.
chown root:root "$STATE_DIR" 2>/dev/null || true
for f in image image-ref target-version; do
[ -e "$STATE_DIR/$f" ] || continue
chown root:root "$STATE_DIR/$f" 2>/dev/null || true
chmod 0644 "$STATE_DIR/$f"
done
umask 077
{
printf '%s\n' "$NODE_ENV"
printf 'WARMBLY_VERSION=%s\n' "$DESIRED_VERSION"
printf 'WARMBLY_TARGET_VERSION_PATH=%s/target-version\n' "$AGENT_DIR"
printf 'WARMBLY_NODE_NAME=%s\n' "$WARMBLY_NAME"
} > "$CONFIG_DIR/node.env"
chmod 600 "$CONFIG_DIR/node.env"
printf '%s\n' "$DESIRED_VERSION" > "$AGENT_DIR/target-version"
chown 1000:1000 "$AGENT_DIR/target-version" 2>/dev/null || true
printf '%s\n' "$WARMBLY_IMAGE_REPO/$WARMBLY_ROLE" > "$STATE_DIR/image"
# systemd performs no command substitution, so the image reference has to
# reach the unit as an environment variable it can expand itself.
printf 'WARMBLY_IMAGE_REF=%s/%s:%s\n' \
"$WARMBLY_IMAGE_REPO" "$WARMBLY_ROLE" "$DESIRED_VERSION" > "$STATE_DIR/image-ref"
log "Wrote $CONFIG_DIR/node.env"
}
# Blob handling. All of it reads the env in memory rather than the file on
# disk, so --dry-run reports the same thing a real join would do instead of
# reading a previous join's leftovers or failing on a file that is not there.
blob_provider() {
printf '%s\n' "$NODE_ENV" | sed -n 's/^BLOB_PROVIDER=//p' | head -n 1
}
# blobs_are_local matches what storage.NewFromEnv accepts, which is both
# "filesystem" and the "fs" alias. Testing only the long form left an fs
# instance with no mount and no warning.
blobs_are_local() {
case "$(blob_provider)" in
filesystem|fs) return 0 ;;
*) return 1 ;;
esac
}
blob_root() {
printf '%s\n' "$NODE_ENV" | sed -n 's/^BLOB_FS_ROOT=//p' | head -n 1
}
# ensure_blob_root prepares the directory the node's storage layer will open.
# It has to exist and be writable by uid 1000 before the container starts: the
# storage layer does MkdirAll and the process exits if that fails, and an
# unmounted path is created root-owned by docker, so the node restart-loops.
#
# Failures are reported, never swallowed: a silent skip here produces exactly
# that restart loop after the script has printed "Done".
ensure_blob_root() {
root="$1"
if [ ! -d "$root" ]; then
# 0022 for the duration: write_config sets umask 077, which would create
# every missing PARENT 0700 and leave a co-located backend unable to
# traverse in. The explicit chmod below only covers the leaf.
( umask 022 && mkdir -p "$root" ) || die "could not create BLOB_FS_ROOT '$root'"
chmod 0755 "$root" || die "could not set permissions on '$root'"
chown 1000:1000 "$root" 2>/dev/null || true
return 0
fi
# It already existed, so it belongs to something else - most likely a
# co-located backend. Re-owning it would break that backend, so this only
# reports: guessing at writability from the mode bits got both directions
# wrong (a 0666 root has no search bit, a root:1000 0775 root is fine), and a
# wrong guess is worse than saying plainly what to check.
owner=$(stat -c '%u' "$root" 2>/dev/null || echo "")
if [ -n "$owner" ] && [ "$owner" != "1000" ]; then
warn ""
warn "NOTE: $root already exists and is owned by uid $owner."
warn " The node runs as uid 1000. If it cannot write there, sends fail"
warn " when the worker tries to store a message body. Check with:"
warn ""
warn " sudo -u '#1000' test -w $root && echo writable || echo NOT writable"
warn ""
warn " Give uid 1000 access, or switch the instance to BLOB_PROVIDER=s3."
warn ""
fi
return 0
}
# validate_blob_root rejects a value docker could never mount, right after
# enrolment and before anything is written.
validate_blob_root() {
blobs_are_local || return 0
root=$(blob_root)
[ -n "$root" ] || return 0
case "$root" in
/*) ;;
*) die "BLOB_FS_ROOT is '$root', which is not an absolute path. Docker cannot mount a relative path; fix it on the backend and re-run." ;;
esac
if [ -e "$root" ] && [ ! -d "$root" ]; then
die "BLOB_FS_ROOT '$root' exists but is not a directory."
fi
return 0
}
# docker_mounts is every -v argument, on ONE line. Command substitution strips
# trailing newlines, so a multi-line value would collapse the unit's
# continuations and hand docker a stray token as the image name.
#
# Pure: it computes the list and creates nothing. Preparing the directories is
# ensure_blob_root's job, called from install_units, so the unit can be
# rendered and asserted on without touching the filesystem.
docker_mounts() {
mounts="-v $AGENT_DIR:$AGENT_DIR"
if blobs_are_local; then
root=$(blob_root)
if [ -n "$root" ]; then
mounts="$mounts -v $root:$root"
fi
fi
printf '%s' "$mounts"
}
# warn_shared_blobs is loud on purpose. A node on filesystem blobs either has
# its own copy, and cannot read the bodies the backend asked it to send, or
# shares a directory it may have no permission on. Both fail at send time, long
# after this script has printed "Done".
warn_shared_blobs() {
blobs_are_local || return 0
warn ""
warn "WARNING: this instance stores blobs on local disk (BLOB_PROVIDER=$(blob_provider),"
warn " BLOB_FS_ROOT=$(blob_root))."
warn ""
warn " A node needs the SAME storage the backend writes to, with"
warn " permissions it can read. That only holds when the node shares a"
warn " filesystem with the backend and the ids line up. Otherwise sends"
warn " fail when the worker cannot read the message body."
warn ""
warn " Set BLOB_PROVIDER=s3 on the backend before running nodes off-host,"
warn " then re-run this command."
warn ""
}
# render_unit prints the systemd service exactly as install_units writes it.
# Separate so it can be asserted on without root, Docker, or a real join:
# `make join-check` renders this and checks the result, because every defect
# this file has had parsed cleanly and only showed up in what it produced.
render_unit() {
service="warmbly-$WARMBLY_ROLE"
MOUNTS=$(docker_mounts)
cat <<UNIT
[Unit]
Description=Warmbly $WARMBLY_ROLE
After=docker.service network-online.target
Requires=docker.service
[Service]
Restart=always
RestartSec=5
# systemd does not run a shell, so the image reference comes from a file it
# reads as environment rather than from a command substitution. \${VAR} expands
# to exactly one argument, which is what an image:tag needs.
EnvironmentFile=$STATE_DIR/image-ref
# The container is replaced rather than reconfigured, so start always removes
# any previous one first: a name collision after an unclean stop would
# otherwise wedge the service in a restart loop.
ExecStartPre=-/usr/bin/docker rm -f $service
ExecStart=/usr/bin/docker run --rm --name $service --env-file $CONFIG_DIR/node.env --network host $MOUNTS \${WARMBLY_IMAGE_REF}
ExecStop=/usr/bin/docker stop $service
[Install]
WantedBy=multi-user.target
UNIT
}
install_units() {
[ "$DRY_RUN" = "false" ] || return 0
service="warmbly-$WARMBLY_ROLE"
if blobs_are_local; then
root=$(blob_root)
if [ -n "$root" ]; then
# Keep this a standalone statement: make join-check asserts on it by
# first field, because a looser match was satisfied by the name
# appearing inside a warn string.
ensure_blob_root "$root"
fi
fi
render_unit > "/etc/systemd/system/$service.service"
# The updater is what makes auto-update work without anything reaching into
# this machine. The node writes the version the control plane wants into
# $STATE_DIR/target-version on each heartbeat; this notices the file changed,
# pulls, and restarts. Keeping it outside the service means the process being
# replaced is never the process doing the replacing.
cat > "/usr/local/bin/warmbly-node-update" <<'UPDATER'
#!/bin/sh
set -eu
STATE_DIR="/var/lib/warmbly"
AGENT_DIR="/var/lib/warmbly/node"
CONFIG_DIR="/etc/warmbly"
[ -f "$AGENT_DIR/target-version" ] || exit 0
[ -f "$STATE_DIR/image" ] || exit 0
target="$(cat "$AGENT_DIR/target-version")"
image="$(cat "$STATE_DIR/image")"
current="$(sed -n 's/^WARMBLY_VERSION=//p' "$CONFIG_DIR/node.env" | head -n 1)"
[ -n "$target" ] || exit 0
[ "$target" != "$current" ] || exit 0
# The node writes this file, and root runs whatever image it names, so the
# value is validated rather than trusted: tag characters only, no registry or
# path separators that could redirect the pull somewhere else.
case "$target" in
*[!A-Za-z0-9._-]*) echo "warmbly-node-update: refusing malformed target '$target'"; exit 0 ;;
esac
role="$(sed -n 's/^WARMBLY_NODE_ROLE=//p' "$CONFIG_DIR/node.env" | head -n 1)"
[ -n "$role" ] || exit 0
# Pull first. If the image is not there yet, leave the node on the version it
# is running rather than restarting it into a pull failure.
if ! docker pull "$image:$target" >/dev/null 2>&1; then
echo "warmbly-node-update: $image:$target is not pullable yet; staying on $current"
exit 0
fi
sed -i "s|^WARMBLY_VERSION=.*|WARMBLY_VERSION=$target|" "$CONFIG_DIR/node.env"
printf 'WARMBLY_IMAGE_REF=%s:%s\n' "$image" "$target" > "$STATE_DIR/image-ref"
echo "warmbly-node-update: $current -> $target"
systemctl restart "warmbly-$role"
UPDATER
chmod 755 /usr/local/bin/warmbly-node-update
cat > /etc/systemd/system/warmbly-node-update.service <<'UNIT'
[Unit]
Description=Apply the Warmbly version the control plane asked for
[Service]
Type=oneshot
ExecStart=/usr/local/bin/warmbly-node-update
UNIT
cat > /etc/systemd/system/warmbly-node-update.timer <<'UNIT'
[Unit]
Description=Check for a new Warmbly version
[Timer]
OnBootSec=2min
OnUnitActiveSec=2min
[Install]
WantedBy=timers.target
UNIT
systemctl daemon-reload
log "Installed $service.service and warmbly-node-update.timer"
}
start_node() {
[ "$DRY_RUN" = "false" ] || return 0
service="warmbly-$WARMBLY_ROLE"
log "Pulling $WARMBLY_IMAGE_REPO/$WARMBLY_ROLE:$DESIRED_VERSION"
docker pull "$WARMBLY_IMAGE_REPO/$WARMBLY_ROLE:$DESIRED_VERSION" >/dev/null
systemctl enable --now warmbly-node-update.timer >/dev/null 2>&1 || true
systemctl enable "$service" >/dev/null 2>&1 || true
systemctl restart "$service"
log ""
log "Done. This machine is now a Warmbly $WARMBLY_ROLE."
log ""
log " Node id $NODE_ID"
log " Version $DESIRED_VERSION"
log " Logs journalctl -u $service -f"
log " Status systemctl status $service"
log ""
log "It will appear in Fleet within a minute or two, and will keep itself on"
log "whatever version you set there. Nothing else to do."
}
main() {
# The calls below are asserted on by `make join-check`, matched on their
# first field, so each stays a standalone statement. Reformatting one into
# `if ! x; then` fails the build with a message about ordering.
parse_args "$@"
if [ "$PRINT_UNIT" = "true" ]; then
# No enrolment, no network, no files. NODE_ENV is whatever the caller
# supplied, so the blob branches can be exercised from a test.
WARMBLY_ROLE="${WARMBLY_ROLE:-worker}"
NODE_ENV="${NODE_ENV:-}"
validate_blob_root
render_unit
return 0
fi
require_args
check_deps
enrol
# Validate the config we were handed before writing any of it: failing later
# leaves an enrolled node with files on disk and no service.
validate_blob_root
write_config
install_units
start_node
warn_shared_blobs
return 0
}
main "$@"
-102
View File
@@ -1,102 +0,0 @@
package handler
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"strings"
"github.com/gin-gonic/gin"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
func (h *Handler) ServeWorkerInstaller(c *gin.Context) {
if h.WorkerOrchestrator == nil {
errx.JSON(c, errx.New(errx.ServiceUnavailable, "worker installer is not configured"))
return
}
script, err := h.WorkerOrchestrator.InstallerScript()
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to load worker installer"))
return
}
c.Header("Content-Type", "text/x-shellscript; charset=utf-8")
c.Header("Cache-Control", "no-cache")
c.Data(http.StatusOK, "text/x-shellscript; charset=utf-8", script)
}
type workerEnrollmentRequest struct {
Token string `json:"token"`
PublicIP string `json:"public_ip,omitempty"`
}
// EnrollWorker exchanges a one-time enrollment token for a complete worker
// dotenv file. It is intentionally public: the high-entropy one-time token is
// the credential, and it is consumed atomically before secrets are returned.
func (h *Handler) EnrollWorker(c *gin.Context) {
if h.WorkerRepo == nil || h.WorkerOrchestrator == nil {
errx.JSON(c, errx.New(errx.ServiceUnavailable, "worker enrollment is not configured"))
return
}
var req workerEnrollmentRequest
if err := c.ShouldBindJSON(&req); err != nil {
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
return
}
req.Token = strings.TrimSpace(req.Token)
if req.Token == "" {
errx.JSON(c, errx.New(errx.BadRequest, "enrollment token is required"))
return
}
sum := sha256.Sum256([]byte(req.Token))
worker, err := h.WorkerRepo.ConsumeEnrollmentToken(c.Request.Context(), hex.EncodeToString(sum[:]))
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to consume enrollment token"))
return
}
if worker == nil {
errx.JSON(c, errx.New(errx.Unauthorized, "enrollment token is invalid or expired"))
return
}
ip := strings.TrimSpace(req.PublicIP)
if ip == "" {
ip = c.ClientIP()
}
if ip != "" {
_ = h.WorkerRepo.RecordEnrolledIP(c.Request.Context(), worker.ID, ip)
}
envFile, _, err := h.WorkerOrchestrator.RenderEnrollmentEnv(c.Request.Context(), worker.ID)
if err != nil {
errx.JSON(c, errx.New(errx.Internal, "failed to render worker config"))
return
}
envFile += "WORKER_TIER=" + workerTierLabel(worker) + "\n"
if ip != "" {
envFile += "WORKER_PUBLIC_IP=" + ip + "\n"
}
if worker.EgressKind != "" {
envFile += "WORKER_EGRESS_KIND=" + string(worker.EgressKind) + "\n"
}
c.Header("Content-Type", "text/plain; charset=utf-8")
c.Header("Cache-Control", "no-store")
c.String(http.StatusOK, envFile)
}
func workerTierLabel(w *models.Worker) string {
if w == nil {
return "shared_premium"
}
if w.WorkerType == models.WorkerTypeDedicated {
return "dedicated"
}
if w.FreeTier {
return "shared_free"
}
return "shared_premium"
}
+19 -28
View File
@@ -83,8 +83,12 @@ func Run(
// Public worker enrollment. The one-time enrollment token is the
// credential; successful exchange returns a dotenv file for the installer
// and consumes the token.
r.GET("/worker-install.sh", h.ServeWorkerInstaller)
r.POST("/api/v1/workers/enroll", h.EnrollWorker)
// Joining the fleet. The script is public (it does nothing without a
// token); the enrolment endpoint is the only one reachable with the join
// token rather than an operator session, because the machine running it
// has no credentials yet.
r.GET("/join.sh", h.ServeJoinScript)
r.POST("/api/v1/fleet/join", h.FleetJoin)
// Public OAuth-bouncer pages used by the mailbox onboarding popup.
// The provider redirects here; the page postMessages the code/state
@@ -144,7 +148,7 @@ func Run(
// on boot (worker_id + bind_ip + tag) and pull their runtime config
// instead of carrying it all in the install-time env file.
internal.GET("/worker/config", h.InternalWorkerConfig)
internal.POST("/worker/heartbeat", h.InternalWorkerHeartbeat)
internal.POST("/fleet/heartbeat", h.FleetHeartbeat)
// Hosted forms: the forms service (cmd/forms) resolves published
// forms, forwards deduped funnel events and visitor submissions
@@ -1418,30 +1422,11 @@ func Run(
adminRoutes.POST("/workers/:id/reassign", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminReassignEmails)
// SSH-managed worker lifecycle (admin-driven add / install / restart / logs)
adminRoutes.GET("/workers/managed", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminListSSHWorkers)
adminRoutes.POST("/workers", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminCreateWorker)
adminRoutes.GET("/workers/:id/managed", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminGetSSHWorker)
adminRoutes.POST("/workers/:id/test", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminTestWorker)
adminRoutes.POST("/workers/:id/install", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminInstallWorker)
adminRoutes.POST("/workers/:id/restart", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminRestartWorker)
adminRoutes.POST("/workers/:id/upgrade", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminUpdateWorkerImage)
adminRoutes.POST("/workers/:id/uninstall", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminUninstallWorker)
adminRoutes.POST("/workers/:id/rotate-keys", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminRotateWorkerKeys)
adminRoutes.GET("/workers/:id/live-status", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminWorkerStatusLive)
adminRoutes.GET("/workers/:id/logs", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminWorkerLogs)
adminRoutes.DELETE("/workers/:id", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminDeleteSSHWorker)
adminRoutes.PUT("/workers/:id/profile", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminAssignWorkerProfile)
adminRoutes.POST("/workers/:id/apply", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminApplyWorkerConfig)
adminRoutes.POST("/workers/:id/system-update", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSystemUpdate)
adminRoutes.POST("/workers/:id/reboot", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminRebootWorker)
adminRoutes.POST("/workers/preflight", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminPreflightWorker)
adminRoutes.GET("/workers/tags", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminListWorkerTags)
adminRoutes.PUT("/workers/:id/tags", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSetWorkerTags)
// Removed for self-host: worker convert-to-dedicated + risk-pool (multi-tenant
// IP-reputation fleet constructs), reusable AWS credentials + worker profiles
// (cloud-fleet env templating), and GitHub release auto-roll. Attach and manage
// machines you own via the SSH worker lifecycle above.
// Removed for self-host: reusable AWS credentials + worker profiles
// (cloud-fleet env templating) and GitHub release auto-roll. Attach and
// manage machines you own via the SSH worker lifecycle above.
// Warmup Management
adminRoutes.GET("/warmup/pools", middleware.RequireAdminPermission(models.AdminPermViewWarmupPool), h.AdminListWarmupPools)
@@ -1543,12 +1528,18 @@ func Run(
adminRoutes.POST("/jobs/:name/run", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminRunJob)
// Fleet placement: capacity per worker, the control loops' decision
// log, and dedicated worker bindings.
// log, and isolated-egress reservations.
adminRoutes.GET("/fleet/nodes", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetNodes)
adminRoutes.POST("/fleet/join-token", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetIssueJoinToken)
adminRoutes.PATCH("/fleet/nodes/:id", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetPatchNode)
adminRoutes.DELETE("/fleet/nodes/:id", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetDeleteNode)
adminRoutes.GET("/fleet/release", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetRelease)
adminRoutes.PUT("/fleet/release", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetSetRelease)
adminRoutes.GET("/fleet/capacity", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetCapacity)
adminRoutes.GET("/fleet/decisions", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDecisions)
adminRoutes.GET("/fleet/dedicated", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDedicated)
adminRoutes.POST("/fleet/dedicated/:orgId/release", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetReleaseDedicated)
adminRoutes.POST("/workers/:id/convert-dedicated", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminConvertWorkerToDedicated)
adminRoutes.POST("/fleet/dedicated/:orgId/release", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetReleaseIsolatedEgress)
adminRoutes.POST("/workers/:id/reserve", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetReserveWorker)
// Workspace transfers: the same archive service the owner uses from
// Settings > Data, driven by the operator for any workspace.
+2 -4
View File
@@ -243,7 +243,7 @@ func (s *JobsService) deactivateIfLongDead(ctx context.Context, w models.Worker)
if n, herr := s.Cache.Exists(ctx, "worker:heartbeat:"+w.ID.String()).Result(); herr != nil || n > 0 {
return
}
if err := s.WorkerRepo.DeactivateWorker(ctx, w.ID); err != nil {
if err := s.FleetNodeRepo.Deactivate(ctx, w.ID); err != nil {
log.Warn().Err(err).Str("worker_id", w.ID.String()).Msg("failed to deactivate dead worker")
return
}
@@ -262,9 +262,7 @@ func (s *JobsService) accountOrgs(ctx context.Context, accountIDs []uuid.UUID) m
}
func (s *JobsService) findHealthyWorker(ctx context.Context, deadWorker models.Worker) (*models.Worker, error) {
// Get workers of the same tier that are alive
freeTier := deadWorker.FreeTier
workers, err := s.WorkerRepo.GetSharedWorkersByTier(ctx, freeTier)
workers, err := s.WorkerRepo.ListPlaceableWorkers(ctx)
if err != nil {
return nil, err
}
+18 -83
View File
@@ -4,33 +4,27 @@ import (
"context"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/jobrun"
"github.com/warmbly/warmbly/internal/models"
)
// StartRiskRebalancer recomputes per-mailbox risk bands from warmup health
// state and, when a mailbox's band no longer matches its worker's risk
// pool, migrates it to a worker in the right pool.
// StartRiskRebalancer recomputes each mailbox's risk band from its warmup
// health state.
//
// Why a periodic batch instead of event-driven: warmup health state
// changes on a slow rolling-window basis (warmup_health_sweep job runs
// hourly), so reacting in real time doesn't buy much. A 1h batch keeps
// the model simple and avoids thundering-herd migrations after the warmup
// sweep finishes.
// It used to also migrate mailboxes so a "risky" one never shared a worker
// with a clean one. That segregation protected nothing: the worker is not the
// sending identity, so a mailbox landing in spam cannot drag down a neighbour
// whose mail leaves through an entirely different provider. The band survives
// because warmup partner selection and per-mailbox pacing genuinely use it.
//
// Dedicated workers are exempt — their tenant boundary is the customer,
// not the risk band.
// Why a periodic batch instead of event-driven: warmup health state changes on
// a slow rolling-window basis (the warmup_health_sweep job runs hourly), so
// reacting in real time doesn't buy much.
func (s *JobsService) StartRiskRebalancer(ctx context.Context, interval time.Duration) {
if s.WorkerRepo == nil {
return
}
if s.AssignmentService == nil {
log.Info().Msg("risk rebalancer disabled: AssignmentService not configured")
return
}
// The boot pass (so a fresh deploy converges quickly) keeps its shorter budget.
first := true
jobrun.Loop(ctx, "risk_rebalancer", interval, true, func(ctx context.Context) error {
@@ -49,87 +43,28 @@ func (s *JobsService) StartRiskRebalancer(ctx context.Context, interval time.Dur
func (s *JobsService) rebalanceRisk(ctx context.Context) {
candidates, err := s.WorkerRepo.ListRiskCandidates(ctx, 1000)
if err != nil {
log.Warn().Err(err).Msg("risk rebalancer: list candidates failed")
log.Warn().Err(err).Msg("risk band sweep: list candidates failed")
return
}
var (
updatedBand int
migrated int
skipped int
)
var updatedBand int
for _, c := range candidates {
newBand := models.RiskBandFromHealth(c.HealthState)
// 1. Update the stored band if it changed.
if newBand != c.CurrentBand {
if err := s.WorkerRepo.SetEmailAccountRiskBand(ctx, c.EmailAccountID, newBand); err != nil {
log.Warn().Err(err).Str("account_id", c.EmailAccountID.String()).Msg("risk rebalancer: set band failed")
continue
}
updatedBand++
}
// 2. Migrate if the new band's matching pool isn't where the mailbox
// currently sits. Dedicated workers were already excluded by the
// candidate query, so c.WorkerType is always shared (or nil/unset).
if c.WorkerID == nil {
skipped++
if newBand == c.CurrentBand {
continue
}
wantPool := newBand.MatchingRiskPool()
if c.WorkerRiskPool == wantPool {
if err := s.WorkerRepo.SetEmailAccountRiskBand(ctx, c.EmailAccountID, newBand); err != nil {
log.Warn().Err(err).Str("account_id", c.EmailAccountID.String()).Msg("risk band sweep: set band failed")
continue
}
target, err := s.AssignmentService.SelectSharedWorkerForBand(ctx, c.WorkerFreeTier, newBand)
if err != nil {
log.Warn().Err(err).Str("account_id", c.EmailAccountID.String()).Msg("risk rebalancer: pick target failed")
skipped++
continue
}
if target == nil || target.ID == *c.WorkerID {
skipped++
continue
}
if err := s.WorkerRepo.UpdateEmailAccountWorker(ctx, c.EmailAccountID, target.ID); err != nil {
log.Warn().Err(err).Str("account_id", c.EmailAccountID.String()).Msg("risk rebalancer: migrate failed")
skipped++
continue
}
_ = s.WorkerRepo.DecrementAccountCount(ctx, *c.WorkerID)
_ = s.WorkerRepo.IncrementAccountCount(ctx, target.ID)
migrated++
// Audit each migration so admins can see the rebalancer's decisions.
if s.AdminRepo != nil {
_ = s.AdminRepo.CreateAuditLog(ctx, &models.AdminAuditLog{
ID: uuid.New(),
AdminUserID: uuid.Nil,
Action: "risk_rebalance_migrate",
TargetType: "email_account",
TargetID: c.EmailAccountID,
Details: map[string]any{
"from_worker": c.WorkerID.String(),
"to_worker": target.ID.String(),
"from_band": string(c.CurrentBand),
"to_band": string(newBand),
"health": string(c.HealthState),
},
UserAgent: "system",
CreatedAt: time.Now(),
})
}
updatedBand++
}
if updatedBand > 0 || migrated > 0 {
if updatedBand > 0 {
log.Info().
Int("updated_bands", updatedBand).
Int("migrated", migrated).
Int("skipped", skipped).
Int("scanned", len(candidates)).
Msg("risk rebalancer pass complete")
Msg("risk band sweep complete")
}
}
+1
View File
@@ -47,6 +47,7 @@ type JobsService struct {
WarmupEngagementRepo repository.WarmupEngagementRepository
WarmupService warmupapp.Service
WorkerRepo repository.WorkerRepository
FleetNodeRepo repository.FleetNodeRepository
// LifecycleRepo moves mailboxes in and out of cold rotation. Nil disables
// the lifecycle rebalancer entirely.
LifecycleRepo repository.SendLifecycleRepository
+1 -1
View File
@@ -47,7 +47,7 @@ func (s *JobsService) syncHeartbeats(ctx context.Context) error {
if err != nil {
continue
}
if err := s.WorkerRepo.UpdateLastSeen(ctx, w.ID, t); err != nil {
if err := s.FleetNodeRepo.TouchLastSeen(ctx, w.ID, t); err != nil {
log.Warn().Err(err).Str("worker_id", w.ID.String()).Msg("heartbeat sync: update failed")
}
}
+5 -12
View File
@@ -198,16 +198,9 @@ func (s *emailService) OnboardSMTPIMAP(ctx context.Context, userID string, orgID
return nil, errx.ErrEmailOnboardNoWorker
}
// Pick any healthy worker for the one-shot validation handshake. Tier is
// irrelevant here (nothing is placed yet, the worker just dials the
// credentials once), so fall back to the other tier rather than failing:
// asking only for free-tier workers made onboarding impossible on any
// deployment whose workers all register as premium, which includes a stock
// self-host install.
w, werr := s.workerAssignment.SelectSharedWorker(ctx, false)
if werr != nil || w == nil {
w, werr = s.workerAssignment.SelectSharedWorker(ctx, true)
}
// Any live worker can run the one-shot validation handshake: nothing is
// placed yet, the worker just dials the credentials once and reports back.
w, werr := s.workerAssignment.SelectValidationWorker(ctx)
if werr != nil || w == nil {
return nil, errx.ErrEmailOnboardNoWorker
}
@@ -225,8 +218,8 @@ func (s *emailService) OnboardSMTPIMAP(ctx context.Context, userID string, orgID
return nil, xerr
}
// Assign the long-term worker (free vs paid tier). Failure here is non-fatal:
// the scheduler will pick the account up on its next pass.
// Place the mailbox for real. Failure here is non-fatal: the scheduler
// picks the account up on its next pass.
if orgID != nil {
if _, err := s.workerAssignment.AssignWorkerToEmail(ctx, acc.ID, *orgID); err != nil {
errs.CaptureException(err)
+3 -6
View File
@@ -149,12 +149,9 @@ func (s *emailService) UpdateSMTPIMAPCredentials(ctx context.Context, orgID *uui
if s.workerAssignment == nil {
return nil, errx.ErrEmailOnboardNoWorker
}
// Any healthy worker can run the one-shot validation handshake, same as at
// connect time; tier only matters for placement.
w, werr := s.workerAssignment.SelectSharedWorker(ctx, false)
if werr != nil || w == nil {
w, werr = s.workerAssignment.SelectSharedWorker(ctx, true)
}
// Any live worker can run the one-shot validation handshake, same as at
// connect time.
w, werr := s.workerAssignment.SelectValidationWorker(ctx)
if werr != nil || w == nil {
return nil, errx.ErrEmailOnboardNoWorker
}
@@ -62,8 +62,11 @@ func newRemovalLiveFixture(t *testing.T) *removalLiveFixture {
f.org, "drop-"+f.org.String()[:8], f.user)
// One mailbox's worth of load: an smtp_imap mailbox that is not warming
// weighs 1.0, which is what the delete has to refund.
exec(`INSERT INTO workers (id, name, ip_addr, active, account_count, load_score)
VALUES ($1, 'drop-test', '127.0.0.1', true, 1, 1)`, f.worker)
// A worker is a node (the machine) plus a placement row (the mail on it).
exec(`INSERT INTO fleet_nodes (id, role, name, address, active, last_seen_at)
VALUES ($1, 'worker', 'drop-test', '127.0.0.1', true, now())`, f.worker)
exec(`INSERT INTO workers (id, account_count, load_score)
VALUES ($1, 1, 1)`, f.worker)
exec(`INSERT INTO email_accounts (id, user_id, organization_id, worker_id, email, name,
signature_plain, signature_html, provider, status, campaign_limit, min_wait_time)
VALUES ($1, $2, $3, $4, $5, 'Drop', '', '', 'smtp_imap', 'active', 50, 600)`,
@@ -76,7 +79,7 @@ func newRemovalLiveFixture(t *testing.T) *removalLiveFixture {
arg any
}{
{`DELETE FROM email_accounts WHERE id = $1`, f.mailbox},
{`DELETE FROM workers WHERE id = $1`, f.worker},
{`DELETE FROM fleet_nodes WHERE id = $1`, f.worker},
{`DELETE FROM organizations WHERE id = $1`, f.org},
{`DELETE FROM users WHERE id = $1`, f.user},
} {
+2 -2
View File
@@ -354,13 +354,13 @@ func TestDeleteGivesTheWorkerItsCapacityBack(t *testing.T) {
func TestDeleteRefundsTheWeightTheMailboxWasChargedAt(t *testing.T) {
f := newRemovalFixture(t)
warming := time.Now()
f.repo.account.Provider = "gmail-api"
f.repo.account.Provider = "gmail"
f.repo.account.Warmup = &warming
if xerr := f.svc.Delete(context.Background(), f.user.String(), f.mailbox.String()); xerr != nil {
t.Fatalf("delete: %v", xerr)
}
if len(f.repo.refunded) != 1 || f.repo.refunded[0] != worker.MailboxWeight("gmail-api", true) {
if len(f.repo.refunded) != 1 || f.repo.refunded[0] != worker.MailboxWeight("gmail", true) {
t.Errorf("refund = %v, want the warmup weight", f.repo.refunded)
}
}
+23 -25
View File
@@ -57,33 +57,31 @@ func (q *QuarantineEvaluator) tick(ctx context.Context) error {
models.WorkerHealthQuarantined,
}
for _, freeTier := range []bool{true, false} {
rows, err := q.WorkerRepo.ListCapacityCandidates(ctx, freeTier, allStates)
if err != nil {
return err
rows, err := q.WorkerRepo.ListCapacityCandidates(ctx, allStates)
if err != nil {
return err
}
for _, row := range rows {
newState := q.classify(row)
if newState == row.HealthState {
continue
}
for _, row := range rows {
newState := q.classify(row)
if newState == row.HealthState {
continue
}
if err := q.WorkerRepo.SetWorkerHealthState(ctx, row.WorkerID, newState); err != nil {
log.Warn().Err(err).Str("worker", row.WorkerID.String()).Msg("set worker health state failed")
continue
}
wid := row.WorkerID
_ = q.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "quarantine",
WorkerID: &wid,
Reason: fmt.Sprintf("%s -> %s (bounces=%d complaints=%d sends=%d)", row.HealthState, newState, row.BouncesHard1h, row.Complaints1h, row.SendsAttempted1h),
TriggeredBy: "auto:quarantine",
})
log.Info().
Str("worker", row.WorkerID.String()).
Str("from", string(row.HealthState)).
Str("to", string(newState)).
Msg("worker health state transition")
if err := q.WorkerRepo.SetWorkerHealthState(ctx, row.WorkerID, newState); err != nil {
log.Warn().Err(err).Str("worker", row.WorkerID.String()).Msg("set worker health state failed")
continue
}
wid := row.WorkerID
_ = q.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "quarantine",
WorkerID: &wid,
Reason: fmt.Sprintf("%s -> %s (bounces=%d complaints=%d sends=%d)", row.HealthState, newState, row.BouncesHard1h, row.Complaints1h, row.SendsAttempted1h),
TriggeredBy: "auto:quarantine",
})
log.Info().
Str("worker", row.WorkerID.String()).
Str("from", string(row.HealthState)).
Str("to", string(newState)).
Msg("worker health state transition")
}
return nil
}
+134 -133
View File
@@ -1,57 +1,53 @@
// Package fleet runs the autonomous control loops that manage the worker
// fleet without operator clicks: rebalancing mailboxes off hot workers,
// scaling the fleet up when capacity runs out, draining quarantined
// workers, and rotating IPs when reputation tanks.
// Package fleet runs the autonomous control loops that manage the worker fleet
// without operator clicks: rotating mailboxes off workers that can no longer
// carry them, scaling up when capacity runs out, and draining quarantined
// machines.
//
// Every action is recorded in decision_log so admins can audit what the
// system did and why.
// Every action is recorded in decision_log so admins can audit what the system
// did and why.
package fleet
import (
"context"
"fmt"
"sort"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
workerapp "github.com/warmbly/warmbly/internal/app/worker"
"github.com/warmbly/warmbly/internal/jobrun"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// Rebalancer drains over-utilised workers onto under-utilised peers within
// the same tier. Runs on an interval; idempotent so running twice doesn't
// double-migrate.
// Rotator moves mailboxes off workers that should not be carrying them.
//
// Safety rails:
// - 24h cooldown per mailbox (prevents thrashing)
// - Max 10% of fleet mailboxes in-flight at any time
// - Only migrate when destination has health_state in (healthy, watch)
// - Skip during the mailbox's local peak hours (8am-6pm) to avoid
// disrupting active campaigns
type Rebalancer struct {
WorkerRepo repository.WorkerRepository
Decisions repository.DecisionLogRepository
HotThresh float64 // utilization above which a worker is "hot" (default 0.80)
ColdThresh float64 // utilization below which a worker is "cold" (default 0.50)
Cooldown time.Duration // per-mailbox migration cooldown (default 24h)
MaxInflight int // max mailboxes migrating concurrently (default 200)
Interval time.Duration // tick interval (default 5min)
// It is deliberately reluctant. Every move changes the client IP a mailbox's
// provider sees, and providers read a moving sign-in location as risk: Google
// challenges the login, and both Google and Microsoft throttle authentication
// per address. So the loop only considers mailboxes whose worker is dead,
// degraded or over capacity, and even then a mailbox has to clear a residency
// floor and the destination has to score materially better before anything
// moves. A fleet where nothing rotates is a healthy fleet, not a broken loop.
type Rotator struct {
WorkerRepo repository.WorkerRepository
Assignment workerapp.WorkerAssignmentService
Decisions repository.DecisionLogRepository
// MaxMovesPerTick bounds how much churn one pass can create, so a fleet-wide
// event (a bad deploy, a provider outage) degrades gracefully instead of
// re-placing everything at once.
MaxMovesPerTick int
// ScanLimit bounds how many candidate mailboxes one pass examines.
ScanLimit int
Interval time.Duration
}
func (r *Rebalancer) defaults() {
if r.HotThresh == 0 {
r.HotThresh = 0.80
func (r *Rotator) defaults() {
if r.MaxMovesPerTick == 0 {
r.MaxMovesPerTick = 50
}
if r.ColdThresh == 0 {
r.ColdThresh = 0.50
}
if r.Cooldown == 0 {
r.Cooldown = 24 * time.Hour
}
if r.MaxInflight == 0 {
r.MaxInflight = 200
if r.ScanLimit == 0 {
r.ScanLimit = 500
}
if r.Interval == 0 {
r.Interval = 5 * time.Minute
@@ -59,122 +55,127 @@ func (r *Rebalancer) defaults() {
}
// Run blocks until ctx is cancelled, ticking every Interval.
func (r *Rebalancer) Run(ctx context.Context) {
func (r *Rotator) Run(ctx context.Context) {
r.defaults()
jobrun.Loop(ctx, "fleet_rebalance", r.Interval, false, r.tick)
jobrun.Loop(ctx, "fleet_rotate", r.Interval, false, r.tick)
}
func (r *Rebalancer) tick(ctx context.Context) error {
// Evaluate each tier independently — never migrate across tiers (would
// violate the free/premium/dedicated isolation contract).
for _, freeTier := range []bool{true, false} {
if err := r.tickTier(ctx, freeTier); err != nil {
log.Warn().Err(err).Bool("free_tier", freeTier).Msg("rebalance tier failed")
}
}
return nil
}
func (r *Rebalancer) tickTier(ctx context.Context, freeTier bool) error {
// Only workers that are eligible to receive load.
rows, err := r.WorkerRepo.ListCapacityCandidates(ctx, freeTier, []models.WorkerHealthState{
models.WorkerHealthHealthy,
models.WorkerHealthWatch,
})
if err != nil {
return err
}
if len(rows) < 2 {
// Nothing to balance against — fleet is one worker (or zero).
func (r *Rotator) tick(ctx context.Context) error {
if r.Assignment == nil {
return nil
}
// Build utilisation list.
type candidate struct {
WorkerID uuid.UUID
Utilization float64
Effective float64
Load float64
}
cands := make([]candidate, 0, len(rows))
for _, row := range rows {
eff := row.BaseCapacity * row.HealthMultiplier * row.AgeMultiplier
if eff <= 0 {
eff = 1
}
util := row.LoadScore / eff
cands = append(cands, candidate{
WorkerID: row.WorkerID,
Utilization: util,
Effective: eff,
Load: row.LoadScore,
})
}
// Sort hottest to coldest.
sort.Slice(cands, func(i, j int) bool { return cands[i].Utilization > cands[j].Utilization })
hot := cands[0]
cold := cands[len(cands)-1]
if hot.Utilization <= r.HotThresh {
return nil // nothing hot enough to bother
}
if cold.Utilization >= r.ColdThresh {
// Fleet is uniformly busy — can't rebalance, must scale.
_ = r.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "rebalance",
Reason: fmt.Sprintf("fleet uniformly hot in tier free=%v (min util=%.0f%%, hot util=%.0f%%) — scale loop should trigger", freeTier, cold.Utilization*100, hot.Utilization*100),
TriggeredBy: "auto:rebalance",
})
return nil
}
// Compute how much load to shift to bring hot down to 70%.
target := hot.Effective * 0.70
excess := hot.Load - target
if excess <= 0 {
return nil
}
// Pick mailboxes off the hot worker, freshest-migrated-last so the
// 24h cooldown filters do the right thing.
mailboxes, err := r.WorkerRepo.GetEmailAccountsByWorkerID(ctx, hot.WorkerID)
candidates, err := r.WorkerRepo.ListRotationCandidates(ctx, workerapp.RotationHotUtilization, r.ScanLimit)
if err != nil {
return err
}
now := time.Now()
moved := 0
for _, mbID := range mailboxes {
if excess <= 0 || moved >= r.MaxInflight {
for _, state := range candidates {
if moved >= r.MaxMovesPerTick {
break
}
// Move it.
if err := r.WorkerRepo.UpdateEmailAccountWorker(ctx, mbID, cold.WorkerID); err != nil {
log.Warn().Err(err).Str("mailbox", mbID.String()).Msg("rebalance migration failed")
if state.WorkerID == nil || state.OrganizationID == nil {
continue
}
_ = r.WorkerRepo.DecrementAccountCount(ctx, hot.WorkerID)
_ = r.WorkerRepo.IncrementAccountCount(ctx, cold.WorkerID)
_ = r.WorkerRepo.AddLoadScore(ctx, hot.WorkerID, -1.0)
_ = r.WorkerRepo.AddLoadScore(ctx, cold.WorkerID, 1.0)
_ = r.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "rebalance",
WorkerID: &hot.WorkerID,
MailboxID: &mbID,
Reason: fmt.Sprintf("hot %.0f%% -> cold %.0f%% (tier free=%v)", hot.Utilization*100, cold.Utilization*100, freeTier),
TriggeredBy: "auto:rebalance",
awayFromOwn := state.ReservedWorkerID != nil && *state.ReservedWorkerID != *state.WorkerID
urgency, reason := workerapp.EvaluateRotation(workerapp.RotationInput{
WorkerActive: state.WorkerActive,
WorkerLive: state.WorkerLive,
WorkerHealth: state.WorkerHealth,
WorkerUtilization: state.WorkerUtilization,
Residency: state.Residency(now),
OnSomeoneElsesReservedWorker: state.WorkerReservedForOtherOrg,
AwayFromOwnReservedWorker: awayFromOwn,
})
if urgency == workerapp.RotationStay {
continue
}
if !workerapp.MayMove(urgency, state.Residency(now)) {
continue
}
lookup := workerapp.PlacementLookup{
EmailAccountID: state.EmailAccountID,
OrgID: *state.OrganizationID,
CurrentWorkerID: state.WorkerID,
Region: state.WorkerRegion,
}
// When the mailbox has to LEAVE where it is, the current worker must be
// off the table: it is still the incumbent, still carries the
// stickiness bonus, and would win its own scoring, so the loop would
// bail on "target == current" and the mailbox would never go anywhere.
// Both fields are cleared together: with the incumbent excluded from
// the candidate list there is nothing left for the stickiness bonus or
// the incumbent score to apply to. The decision log still names what
// the mailbox left, because that comes from state.WorkerID.
leaving := mustLeave(urgency, state)
if leaving {
lookup.CurrentWorkerID = nil
lookup.ExcludeWorkerID = state.WorkerID
}
res, err := r.Assignment.SelectWorkerFor(ctx, lookup)
if err != nil || res == nil || res.Worker == nil {
continue
}
if res.Worker.ID == *state.WorkerID {
continue
}
if !workerapp.WorthMoving(urgency, res.IncumbentScore, res.Score, res.Mandated) {
continue
}
from := *state.WorkerID
if err := r.Assignment.MoveMailbox(ctx, state.EmailAccountID, &from, res.Worker.ID); err != nil {
log.Warn().Err(err).Str("mailbox", state.EmailAccountID.String()).Msg("rotation move failed")
continue
}
moved++
excess -= 1.0
mailboxID := state.EmailAccountID
_ = r.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "rotate",
WorkerID: &from,
MailboxID: &mailboxID,
Reason: rotationReason(reason, res, leaving),
TriggeredBy: "auto:rotate",
})
}
if moved > 0 {
log.Info().
Bool("free_tier", freeTier).
Int("moved", moved).
Str("from", hot.WorkerID.String()).
Str("to", cold.WorkerID.String()).
Msg("rebalance migrated mailboxes")
log.Info().Int("moved", moved).Int("scanned", len(candidates)).Msg("rotation pass complete")
}
return nil
}
// rotationReason is the decision-log line. An urgent move is not a comparison
// - the mailbox had to go - so it does not pretend to be one; printing a score
// against an incumbent that was never a candidate reads as "the incumbent
// scored zero" rather than "the incumbent was not scored".
func rotationReason(reason string, res *workerapp.PlacementResult, leaving bool) string {
if leaving {
return fmt.Sprintf("%s; moved to %s", reason, res.Worker.ID)
}
return fmt.Sprintf("%s; moved to %s (score %.2f vs %.2f)",
reason, res.Worker.ID, res.Score, res.IncumbentScore)
}
// mustLeave reports whether staying put is not an option, as opposed to merely
// being improvable. A dead or degraded worker cannot do the work, and a worker
// reserved for someone else must not keep a stranger's mail.
func mustLeave(urgency workerapp.RotationUrgency, state repository.MailboxPlacementState) bool {
return urgency == workerapp.RotationImmediate ||
urgency == workerapp.RotationElevated ||
state.WorkerReservedForOtherOrg
}
// DrainWorker moves every mailbox off one worker, ignoring residency. Used by
// the admin drain action and by the quarantine loop when a worker is blocked.
func (r *Rotator) DrainWorker(ctx context.Context, workerID uuid.UUID) error {
if r.Assignment == nil {
return nil
}
return r.Assignment.MigrateEmailsFromWorker(ctx, workerID)
}
+224
View File
@@ -0,0 +1,224 @@
package fleet
import (
"context"
"os"
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
workerapp "github.com/warmbly/warmbly/internal/app/worker"
"github.com/warmbly/warmbly/internal/infrastructure/db"
"github.com/warmbly/warmbly/internal/repository"
)
// Live end-to-end cover for placement and rotation against a real database.
// The unit tests pin the scoring and the residency gates; this pins the part
// that only breaks against Postgres: the queries, the counter bookkeeping and
// the loop wiring. Skipped unless WARMBLY_TEST_DB is set, and it wants a
// database this branch's migrations have been applied to:
//
// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \
// go test ./internal/app/fleet/ -run Live -v
//
// Self-contained: it creates its own org, workers and mailboxes, and removes
// them again, so it can run against a database that already has data.
func TestLivePlacementAndRotation(t *testing.T) {
dsn := os.Getenv("WARMBLY_TEST_DB")
if dsn == "" {
t.Skip("WARMBLY_TEST_DB unset")
}
// Billing off is the self-host default and makes the pool assertion below
// distinguishable from the column default.
t.Setenv("BILLING_PROVIDER", "none")
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
repo := repository.NewWorkerRepository(pool)
svc := workerapp.NewAssignmentService(repo, nil, nil)
// Two workers, both heartbeating. A worker is a node plus a placement row,
// so both halves are created the way an enrolling node creates them.
first, second := uuid.New(), uuid.New()
for _, id := range []uuid.UUID{first, second} {
if _, err := pool.Exec(ctx, `
INSERT INTO fleet_nodes (id, role, name, address, active, region, last_seen_at)
VALUES ($1, 'worker', $2, '10.77.0.1', true, 'eu-central', now())`,
id, "live-test-"+id.String()[:8]); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO workers (id, health_state, load_score) VALUES ($1, 'healthy', 0)`, id); err != nil {
t.Fatal(err)
}
}
userID, orgID := uuid.New(), uuid.New()
if _, err := pool.Exec(ctx,
`INSERT INTO users (id, first_name, last_name, email) VALUES ($1,'Live','Test',$2)`,
userID, "live-"+userID.String()[:8]+"@example.test"); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx,
`INSERT INTO organizations (id, name, owner_user_id) VALUES ($1,'Live Test',$2)`,
orgID, userID); err != nil {
t.Fatal(err)
}
mailboxes := make([]uuid.UUID, 0, 3)
for i, provider := range []string{"smtp_imap", "gmail", "smtp_imap"} {
id := uuid.New()
if _, err := pool.Exec(ctx, `
INSERT INTO email_accounts (id, user_id, organization_id, email, name,
signature_plain, signature_html, provider, status)
VALUES ($1,$2,$3,$4,'Live','','',$5::email_provider,'active')`,
id, userID, orgID, id.String()[:8]+"@example.test", provider); err != nil {
t.Fatalf("mailbox %d: %v", i, err)
}
mailboxes = append(mailboxes, id)
}
t.Cleanup(func() {
c := context.Background()
_, _ = pool.Exec(c, `DELETE FROM email_accounts WHERE organization_id = $1`, orgID)
_, _ = pool.Exec(c, `DELETE FROM organizations WHERE id = $1`, orgID)
_, _ = pool.Exec(c, `DELETE FROM users WHERE id = $1`, userID)
_, _ = pool.Exec(c, `DELETE FROM fleet_nodes WHERE id = ANY($1)`, []uuid.UUID{first, second})
})
if err := repo.RefreshWorkerCapacityView(ctx); err != nil {
t.Fatal(err)
}
// 1. Every mailbox places, and is stamped so rotation can enforce residency.
//
// Which worker it lands on is deliberately NOT asserted. The database
// may already hold a fleet, any live worker is a legitimate answer, and
// live tests in other packages create and delete their own workers
// concurrently - re-reading the chosen worker here raced with one of
// those deletes. That placement only ever picks a live worker is
// covered by the placement unit tests, which need no database.
for i, mb := range mailboxes {
got, err := svc.AssignWorkerToEmail(ctx, mb, orgID)
if err != nil || got == nil {
t.Fatalf("mailbox %d did not place: %v", i, err)
}
var assigned *string
if err := pool.QueryRow(ctx,
`SELECT worker_assigned_at::text FROM email_accounts WHERE id=$1`, mb).Scan(&assigned); err != nil {
t.Fatal(err)
}
if assigned == nil {
t.Fatalf("mailbox %d: worker_assigned_at not stamped, so rotation cannot enforce residency", i)
}
}
// 1b. Placement also settles warmup pool membership. It used to fall out of
// tier placement; with tiers gone it has to be set explicitly, and
// leaving it unset silently warms paying customers in the free pool.
//
// The assertion has to distinguish "set" from "left at the default",
// so it runs with billing disabled, where every org resolves to the
// premium pool and the column default ('free') is a visible failure.
for i, mb := range mailboxes {
var poolType *string
if err := pool.QueryRow(ctx,
`SELECT warmup_pool_type FROM email_accounts WHERE id = $1`, mb).Scan(&poolType); err != nil {
t.Fatalf("mailbox %d: read warmup pool: %v", i, err)
}
if poolType == nil || *poolType != "premium" {
got := "<null>"
if poolType != nil {
got = *poolType
}
t.Fatalf("mailbox %d: warmup_pool_type is %q, want \"premium\"; placement is not assigning pool membership", i, got)
}
}
// 2. Gather them onto this test's own worker, so the rotation assertions
// below are about this test's fleet and not whatever else is running.
placed := map[uuid.UUID]uuid.UUID{}
for _, mb := range mailboxes {
var from *uuid.UUID
if err := pool.QueryRow(ctx, `SELECT worker_id FROM email_accounts WHERE id=$1`, mb).Scan(&from); err != nil {
t.Fatal(err)
}
if err := svc.MoveMailbox(ctx, mb, from, first); err != nil {
t.Fatalf("gather onto the test worker: %v", err)
}
placed[mb] = first
}
// 3. Load accounting follows MailboxWeight: 1.0 + 0.05 + 1.0.
var testLoad float64
if err := pool.QueryRow(ctx,
`SELECT COALESCE(sum(load_score),0) FROM workers WHERE id = ANY($1)`,
[]uuid.UUID{first, second}).Scan(&testLoad); err != nil {
t.Fatal(err)
}
if testLoad < 2.04 || testLoad > 2.06 {
t.Fatalf("load_score on the test workers = %.2f, want 2.05 (smtp 1.0 + gmail 0.05 + smtp 1.0)", testLoad)
}
rot := &Rotator{
WorkerRepo: repo,
Assignment: svc,
Decisions: repository.NewDecisionLogRepository(&db.DB{Pool: pool}),
}
rot.defaults()
// 4. A healthy fleet must not churn. Moving a mailbox changes the client IP
// its provider sees, so an idle tick doing nothing is the correct result.
if err := rot.tick(ctx); err != nil {
t.Fatalf("tick on a healthy fleet: %v", err)
}
for mb, want := range placed {
var got *uuid.UUID
if err := pool.QueryRow(ctx, `SELECT worker_id FROM email_accounts WHERE id=$1`, mb).Scan(&got); err != nil {
t.Fatal(err)
}
if got == nil || *got != want {
t.Fatalf("rotation moved a mailbox off a healthy worker (%v -> %v); it must not churn", want, got)
}
}
// 5. A worker stops heartbeating: everything on it has to leave, because a
// command queued for a dead worker is never executed and never answered.
if _, err := pool.Exec(ctx,
`UPDATE fleet_nodes SET last_seen_at = now() - interval '1 hour' WHERE id = $1`, first); err != nil {
t.Fatal(err)
}
if err := repo.RefreshWorkerCapacityView(ctx); err != nil {
t.Fatal(err)
}
if err := rot.tick(ctx); err != nil {
t.Fatalf("tick with a dead worker: %v", err)
}
var stranded int
if err := pool.QueryRow(ctx,
`SELECT count(*) FROM email_accounts WHERE organization_id=$1 AND worker_id=$2`,
orgID, first).Scan(&stranded); err != nil {
t.Fatal(err)
}
if stranded != 0 {
t.Fatalf("%d mailboxes stranded on a worker that stopped heartbeating", stranded)
}
// 6. The move has to be auditable.
var decisions int
if err := pool.QueryRow(ctx,
`SELECT count(*) FROM decision_log WHERE kind='rotate' AND worker_id=$1`, first).Scan(&decisions); err != nil {
t.Fatal(err)
}
if decisions == 0 {
t.Fatal("rotation moved mailboxes without writing decision_log entries")
}
}
+10 -79
View File
@@ -2,7 +2,6 @@ package fleet
import (
"context"
"encoding/json"
"fmt"
"time"
@@ -12,21 +11,13 @@ import (
"github.com/warmbly/warmbly/internal/repository"
)
// Scaler evaluates fleet utilization on a slow tick (default 1h) and
// either emits a "needs more workers" alert or, when AutoProvision is
// allowed by the provisioning policy, enqueues a provisioning job from
// the active auto-template for the relevant tier.
//
// Thresholds:
//
// info -> fleet < 50% utilization (nothing to do)
// warning -> fleet >= 70% sustained, alert
// critical -> fleet >= 85% sustained, alert + (if AUTO_PROVISION) provision
// Scaler watches how full the fleet is and says so. It no longer buys
// machines: there is no cloud account to buy them with, and a node joins by
// running one command on a box you already have. What it still does is notice
// that capacity is running out before sending starts backing up, and write
// that to decision_log where an operator will see it.
type Scaler struct {
WorkerRepo repository.WorkerRepository
PolicyRepo repository.ProvisioningPolicyRepository
TemplateRepo repository.ProvisioningTemplateRepository
JobRepo repository.ProvisioningJobRepository
Decisions repository.DecisionLogRepository
Interval time.Duration // default 1h
CriticalThresh float64 // default 0.85
@@ -47,21 +38,12 @@ func (s *Scaler) defaults() {
func (s *Scaler) Run(ctx context.Context) {
s.defaults()
// Runs once on boot so an admin doesn't wait an hour for the first signal.
// Runs once on boot so an admin does not wait an hour for the first signal.
jobrun.Loop(ctx, "fleet_scale", s.Interval, true, s.tick)
}
func (s *Scaler) tick(ctx context.Context) error {
for _, freeTier := range []bool{true, false} {
if err := s.tickTier(ctx, freeTier); err != nil {
log.Warn().Err(err).Bool("free_tier", freeTier).Msg("scale tier failed")
}
}
return nil
}
func (s *Scaler) tickTier(ctx context.Context, freeTier bool) error {
rows, err := s.WorkerRepo.ListCapacityCandidates(ctx, freeTier, []models.WorkerHealthState{
rows, err := s.WorkerRepo.ListCapacityCandidates(ctx, []models.WorkerHealthState{
models.WorkerHealthHealthy,
models.WorkerHealthWatch,
})
@@ -84,11 +66,6 @@ func (s *Scaler) tickTier(ctx context.Context, freeTier bool) error {
util = totalLoad / totalCap
}
tierName := "shared_premium"
if freeTier {
tierName = "shared_free"
}
severity := ""
switch {
case util >= s.CriticalThresh:
@@ -101,7 +78,7 @@ func (s *Scaler) tickTier(ctx context.Context, freeTier bool) error {
return nil
}
alertReason := fmt.Sprintf("%s utilization %.0f%% (load=%.1f cap=%.1f)", tierName, util*100, totalLoad, totalCap)
alertReason := fmt.Sprintf("fleet utilization %.0f%% (load=%.1f cap=%.1f)", util*100, totalLoad, totalCap)
_ = s.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "scale_alert",
Reason: alertReason,
@@ -109,62 +86,16 @@ func (s *Scaler) tickTier(ctx context.Context, freeTier bool) error {
})
log.Warn().
Str("severity", severity).
Bool("free_tier", freeTier).
Float64("utilization", util).
Msg(alertReason)
if severity != "critical" {
return nil
}
// Critical — try to auto-provision if policy allows.
pol, err := s.PolicyRepo.Get(ctx, "hetzner")
if err != nil {
return err
}
if pol == nil || !pol.AutoProvision {
if severity == "critical" {
_ = s.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "scale_alert",
Reason: "auto_provision=false; admin approval required",
Reason: "fleet is nearly full; add a node with `warmbly join` on any machine you own",
TriggeredBy: "auto:scale",
})
return nil
}
tpl, err := s.TemplateRepo.GetAutoForTier(ctx, tierName)
if err != nil {
return err
}
if tpl == nil {
_ = s.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "scale_alert",
Reason: fmt.Sprintf("no auto-template configured for tier %s", tierName),
TriggeredBy: "auto:scale",
})
return nil
}
// Snapshot the template into a new provisioning_jobs row.
cfgBytes, _ := json.Marshal(tpl)
job := &repository.ProvisioningJob{
State: models.ProvJobPending,
TriggeredBy: "auto:scale",
Provider: tpl.Provider,
TemplateID: &tpl.ID,
Config: cfgBytes,
}
if err := s.JobRepo.Create(ctx, job); err != nil {
return err
}
_ = s.Decisions.Insert(ctx, &repository.DecisionLog{
Kind: "provision",
Reason: fmt.Sprintf("auto-provisioning from template %q (util %.0f%%)", tpl.Name, util*100),
TriggeredBy: "auto:scale",
})
log.Info().
Str("template", tpl.Name).
Str("job", job.ID.String()).
Msg("auto-provisioned new worker(s)")
return nil
}
+185
View File
@@ -0,0 +1,185 @@
// Package fleetnode is the control-plane half of the pull-based fleet.
//
// A node joins by running one command with the instance join token, then
// heartbeats forever. It is never reached into: everything the control plane
// wants a node to do comes back in the heartbeat reply, which today is exactly
// one instruction — what version to be running.
package fleetnode
import (
"context"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"errors"
"strings"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/crypt"
"github.com/warmbly/warmbly/internal/repository"
)
var (
// ErrNoJoinToken means no token has ever been issued, so nothing may join.
// Deliberately distinct from a wrong token: an instance with no token is a
// setup problem, not an attack.
ErrNoJoinToken = errors.New("no join token has been issued for this instance")
ErrBadToken = errors.New("join token is not valid")
ErrBadRole = errors.New("unknown node role")
// ErrRoleChanged is a node claiming an id that is already registered under
// the other role. Silently accepting it would leave a worker's mailboxes
// assigned to a machine that has stopped doing worker work.
ErrRoleChanged = errors.New("that node id is already registered under a different role")
)
type Service struct {
nodes repository.FleetNodeRepository
workers repository.WorkerRepository
settings repository.FleetSettingsRepository
}
func New(
nodes repository.FleetNodeRepository,
workers repository.WorkerRepository,
settings repository.FleetSettingsRepository,
) *Service {
return &Service{nodes: nodes, workers: workers, settings: settings}
}
// IssueJoinToken mints a new instance join token, stores only its hash, and
// returns the plaintext. The caller must show it once: it cannot be recovered.
//
// Issuing replaces any previous token, which is also how you revoke one.
// Existing nodes are unaffected — they are already enrolled, and the token
// only gates joining.
func (s *Service) IssueJoinToken(ctx context.Context) (string, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", err
}
token := base64.RawURLEncoding.EncodeToString(raw)
if err := s.settings.SetJoinTokenHash(ctx, crypt.SHA256(token)); err != nil {
return "", err
}
return token, nil
}
// VerifyJoinToken checks a presented token in constant time.
func (s *Service) VerifyJoinToken(ctx context.Context, token string) error {
want, err := s.settings.GetJoinTokenHash(ctx)
if err != nil {
return err
}
if want == "" {
return ErrNoJoinToken
}
got := crypt.SHA256(strings.TrimSpace(token))
if subtle.ConstantTimeCompare([]byte(got), []byte(want)) != 1 {
return ErrBadToken
}
return nil
}
// Heartbeat records a beat and answers with what the node should be running.
//
// Registration is the first heartbeat: there is no separate create step, so a
// node rebuilt from scratch simply reappears under the same id. A node that
// declares itself a worker also gets its placement row, because placement
// reads `workers` and a node with no row there would be invisible to it.
func (s *Service) Heartbeat(ctx context.Context, beat models.NodeHeartbeat) (*models.NodeHeartbeatReply, error) {
if !beat.Role.Valid() {
return nil, ErrBadRole
}
if beat.NodeID == uuid.Nil {
return nil, errors.New("node_id required")
}
// A node may not change what it does under the same id. Checked before the
// upsert so the row is never half-migrated between roles.
if existing, err := s.nodes.Get(ctx, beat.NodeID); err == nil && existing != nil && existing.Role != beat.Role {
return nil, ErrRoleChanged
}
if beat.Stopping {
// The farewell beat. Go inactive at once rather than staying selectable
// until the beat ages out; placement must not hand work to a process
// that has already gone.
if err := s.nodes.Deactivate(ctx, beat.NodeID); err != nil {
return nil, err
}
return &models.NodeHeartbeatReply{LivenessSeconds: int(models.NodeLivenessWindow.Seconds())}, nil
}
if err := s.nodes.UpsertOnHeartbeat(ctx, beat); err != nil {
return nil, err
}
if beat.Role == models.NodeRoleWorker && s.workers != nil {
if err := s.workers.EnsureWorkerRow(ctx, beat.NodeID); err != nil {
return nil, err
}
}
reply := &models.NodeHeartbeatReply{
LivenessSeconds: int(models.NodeLivenessWindow.Seconds()),
}
reply.DesiredVersion = s.desiredVersion(ctx, beat.NodeID)
return reply, nil
}
// desiredVersion resolves what this node should run: its own pin if it has
// one, otherwise the fleet-wide resolved release.
//
// Any failure answers "" — no opinion. A node that cannot be told what to run
// must keep running what it has, because the alternative is a control-plane
// hiccup rolling the whole fleet.
func (s *Service) desiredVersion(ctx context.Context, nodeID uuid.UUID) string {
if node, err := s.nodes.Get(ctx, nodeID); err == nil && node != nil && node.PinnedVersion != "" {
return node.PinnedVersion
}
state, err := s.settings.GetRelease(ctx)
if err != nil {
return ""
}
return state.DesiredVersion()
}
// List returns the fleet, with each node's resolved target attached so a
// caller can see at a glance which machines are behind.
func (s *Service) List(ctx context.Context, role models.NodeRole) ([]models.FleetNode, error) {
nodes, err := s.nodes.List(ctx, role)
if err != nil {
return nil, err
}
state, err := s.settings.GetRelease(ctx)
if err != nil {
return nil, err
}
fleetTarget := state.DesiredVersion()
for i := range nodes {
if nodes[i].PinnedVersion != "" {
nodes[i].DesiredVersion = nodes[i].PinnedVersion
continue
}
nodes[i].DesiredVersion = fleetTarget
}
return nodes, nil
}
// Get returns one node with its resolved target attached.
func (s *Service) Get(ctx context.Context, id uuid.UUID) (*models.FleetNode, error) {
node, err := s.nodes.Get(ctx, id)
if err != nil || node == nil {
return nil, err
}
if node.PinnedVersion != "" {
node.DesiredVersion = node.PinnedVersion
return node, nil
}
state, err := s.settings.GetRelease(ctx)
if err != nil {
return nil, err
}
node.DesiredVersion = state.DesiredVersion()
return node, nil
}
+2 -2
View File
@@ -39,7 +39,7 @@ func checkNoWorkerHeartbeat(ctx context.Context, d Deps, in Input) *Finding {
var lastSeen *time.Time
var assigned int
err := d.DB.QueryRow(ctx, `
SELECT (SELECT max(last_seen_at) FROM workers),
SELECT (SELECT max(last_seen_at) FROM fleet_nodes WHERE role = 'worker'),
(SELECT count(*) FROM email_accounts WHERE worker_id IS NOT NULL)
`).Scan(&lastSeen, &assigned)
if err != nil || assigned == 0 {
@@ -71,7 +71,7 @@ func checkCodecNotJSON(ctx context.Context, d Deps, in Input) *Finding {
return nil
}
var workers int
if err := d.DB.QueryRow(ctx, `SELECT count(*) FROM workers`).Scan(&workers); err != nil || workers == 0 {
if err := d.DB.QueryRow(ctx, `SELECT count(*) FROM fleet_nodes WHERE role = 'worker'`).Scan(&workers); err != nil || workers == 0 {
return nil
}
return result(CategoryWorkers, SeverityError, "Codec is not JSON",
+3 -3
View File
@@ -702,10 +702,10 @@ var table = []Entry{
Resolve: envValue("WORKER_ID"),
},
{
Key: "WORKER_TIER", Group: GroupWorkers, RuntimeChangeable: ChangeBootOnly,
Effect: "Which placement tier a worker accepts. Free-trial organizations place onto free workers, paid ones onto premium.",
Key: "WARMBLY_NODE_REGION", Group: GroupWorkers, RuntimeChangeable: ChangeBootOnly,
Effect: "Where this node egresses from, as a free-form label. Placement prefers a worker near where a mailbox's provider expects sign-ins; unset scores neutral.",
DocsAnchor: docsWorkers,
Resolve: envValue("WORKER_TIER"),
Resolve: envValue("WARMBLY_NODE_REGION"),
},
{
Key: "MAIL_TLS_INSECURE", Group: GroupWorkers, RuntimeChangeable: ChangeBootOnly,
-55
View File
@@ -1,55 +0,0 @@
package provisioning
import (
"context"
"github.com/google/uuid"
)
// Installer is the SSH-driven side of provisioning: given a freshly-created
// server with a known SSH key, copy the worker binary + env, configure the
// OS to bind each Primary IP, and start one systemd unit per IP.
//
// The real implementation lives in internal/app/worker_orchestrator (which
// already does single-IP installs over SSH). The provisioning state machine
// depends only on this small interface so it stays testable and so we can
// later swap to cloud-init / Ansible / whatever without touching it.
type Installer interface {
// Install runs scripts/install-worker.sh on the target server. ips
// is the comma-separated list of attached Primary IPs. workerEnv is
// the rendered content of /etc/warmbly/worker.env.
Install(ctx context.Context, req InstallRequest) (*InstallResult, error)
}
// InstallRequest is what Install takes.
type InstallRequest struct {
Host string // public IP or hostname the SSH session connects to
SSHPort int // default 22
SSHUser string // default root
SSHKeyPEM []byte // private key bytes (PEM-encoded ed25519)
IPs []string // dotted-quad strings, will be passed to --ips
WorkerEnv string // rendered /etc/warmbly/worker.env contents
ImageTag string // e.g. ghcr.io/warmbly/worker:v1.2.3
ExpectedIDs []uuid.UUID // workers that should heartbeat after install
}
// InstallResult tells the state machine what's running on the box.
type InstallResult struct {
InstalledWorkerIDs []uuid.UUID // typically the same as ExpectedIDs
Logs string // captured installer output, optional
}
// StubInstaller is a no-op that records its inputs. Useful for tests and
// for environments without SSH access.
type StubInstaller struct {
Calls []InstallRequest
Err error
}
func (s *StubInstaller) Install(_ context.Context, req InstallRequest) (*InstallResult, error) {
s.Calls = append(s.Calls, req)
if s.Err != nil {
return nil, s.Err
}
return &InstallResult{InstalledWorkerIDs: req.ExpectedIDs}, nil
}
-47
View File
@@ -1,47 +0,0 @@
package provisioning
import (
"context"
"net"
"strings"
"time"
)
const rdnsLookupTimeout = 5 * time.Second
// VerifyReverseDNS confirms a sending IP has a PTR record AND that the PTR name
// forward-resolves back to the same IP (forward-confirmed reverse DNS / FCrDNS).
// A missing or mismatched PTR is a classic mailbox-provider rejection cause, so
// this complements the set-side SetReverseDNS by verifying the record actually
// took and is self-consistent.
//
// Returns the PTR hostname (if any), whether FCrDNS holds, and any lookup error.
func VerifyReverseDNS(ctx context.Context, ip string) (ptr string, fcrdnsOK bool, err error) {
ip = strings.TrimSpace(ip)
if ip == "" {
return "", false, nil
}
resolver := &net.Resolver{}
c1, cancel1 := context.WithTimeout(ctx, rdnsLookupTimeout)
defer cancel1()
names, err := resolver.LookupAddr(c1, ip)
if err != nil || len(names) == 0 {
return "", false, err
}
ptr = strings.TrimSuffix(names[0], ".")
c2, cancel2 := context.WithTimeout(ctx, rdnsLookupTimeout)
defer cancel2()
addrs, ferr := resolver.LookupHost(c2, ptr)
if ferr != nil {
return ptr, false, ferr
}
for _, a := range addrs {
if a == ip {
return ptr, true, nil
}
}
return ptr, false, nil
}
-133
View File
@@ -1,133 +0,0 @@
package provisioning
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// Runner is the background loop that actually drives provisioning jobs.
//
// Previously nothing called Service.Run, so a job created from the admin UI sat
// in "pending" forever. The Runner polls for in-flight jobs and drives each one
// to a terminal state (completed/failed). Run is idempotent and resumes from a
// job's current state, so a backend restart mid-provision is recoverable.
type Runner struct {
Jobs repository.ProvisioningJobRepository
Svc *Service
Interval time.Duration // default 15s
// DryRun is informational — it only affects the boot log line. The actual
// dry-run behaviour comes from the Service's provider/installer wiring.
DryRun bool
}
// Run blocks until ctx is cancelled, processing in-flight jobs on each tick.
func (r *Runner) Run(ctx context.Context) {
interval := r.Interval
if interval == 0 {
interval = 15 * time.Second
}
log.Info().
Bool("dry_run", r.DryRun).
Dur("interval", interval).
Msg("provisioning runner started")
tick := time.NewTicker(interval)
defer tick.Stop()
r.processOnce(ctx)
for {
select {
case <-ctx.Done():
return
case <-tick.C:
r.processOnce(ctx)
}
}
}
func (r *Runner) processOnce(ctx context.Context) {
if r.Svc == nil || r.Jobs == nil {
return
}
jobs, err := r.Jobs.ListInFlight(ctx)
if err != nil {
log.Warn().Err(err).Msg("provisioning runner: list in-flight failed")
return
}
for i := range jobs {
job := jobs[i]
// rolling_back is a transient state the Service sets just before it
// fails a job; skip it so we don't log a spurious "already terminal".
if job.State == models.ProvJobRollingBack {
continue
}
if err := r.Svc.Run(ctx, job.ID); err != nil {
log.Warn().Err(err).Str("job", job.ID.String()).Msg("provisioning job failed")
}
}
}
// ---------------------------------------------------------------------------
// Dry-run provider
// ---------------------------------------------------------------------------
var dryRunSeq atomic.Uint64
// DryRunProvider implements cloudprovider.Provider without touching any real
// cloud API. It returns plausible fake server/IP identifiers so the state
// machine can run end-to-end in local dev (and in any environment where
// PROVISIONING_DRY_RUN is on) without creating — or billing — real machines.
type DryRunProvider struct{}
func (DryRunProvider) Name() string { return "dry-run" }
func (DryRunProvider) Locations(context.Context) ([]cloudprovider.Location, error) {
return []cloudprovider.Location{}, nil
}
func (DryRunProvider) ServerTypes(context.Context) ([]cloudprovider.ServerType, error) {
return []cloudprovider.ServerType{}, nil
}
func (DryRunProvider) Images(context.Context) ([]cloudprovider.Image, error) {
return []cloudprovider.Image{}, nil
}
func (DryRunProvider) Verify(context.Context) error { return nil }
func (DryRunProvider) CreateServer(_ context.Context, req cloudprovider.CreateServerRequest) (*cloudprovider.Server, error) {
n := dryRunSeq.Add(1)
return &cloudprovider.Server{
ID: fmt.Sprintf("dryrun-%d", n),
Name: req.Name,
Status: "running",
PublicIPv4: dryRunIP(n),
}, nil
}
func (DryRunProvider) DeleteServer(context.Context, string) error { return nil }
func (DryRunProvider) CreatePrimaryIP(_ context.Context, req cloudprovider.CreatePrimaryIPRequest) (*cloudprovider.PrimaryIP, error) {
n := dryRunSeq.Add(1)
return &cloudprovider.PrimaryIP{
ID: fmt.Sprintf("dryrun-ip-%d", n),
Type: req.Type,
IP: dryRunIP(n),
}, nil
}
func (DryRunProvider) AssignPrimaryIP(context.Context, string, string) error { return nil }
func (DryRunProvider) UnassignPrimaryIP(context.Context, string) error { return nil }
func (DryRunProvider) DeletePrimaryIP(context.Context, string) error { return nil }
func (DryRunProvider) SetReverseDNS(context.Context, string, string) error { return nil }
// dryRunIP maps a sequence number into a deterministic 10.x.x.x address so the
// inet[] columns get a valid value.
func dryRunIP(n uint64) string {
return fmt.Sprintf("10.%d.%d.%d", (n>>16)&0xff, (n>>8)&0xff, (n&0xfe)+1)
}
var _ cloudprovider.Provider = DryRunProvider{}
-413
View File
@@ -1,413 +0,0 @@
// Package provisioning is the state machine that drives a provisioning_jobs
// row from "pending" through the lifecycle of creating a server, attaching
// IPs, setting rDNS, installing the worker binary, and verifying that the
// expected workers report in.
//
// The state machine is idempotent: each Run call resumes from the row's
// current state, so a backend crash mid-provision is recoverable. Each step
// records its progress to the database before attempting the next step.
//
// On failure at any step, state transitions to rolling_back and the inverse
// operations run (delete primary IPs, delete server). Cleanly failed jobs
// leave no orphaned resources at the cloud provider.
package provisioning
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"strings"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// UUIDv5 URL namespace, kept in sync with cmd/worker and the installer.
var uuidNamespaceURL = uuid.MustParse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")
// WorkerIDForIP returns the deterministic UUIDv5 that a worker process will
// adopt when it boots with the given WORKER_BIND_IP.
func WorkerIDForIP(ip string) uuid.UUID {
return uuid.NewSHA1(uuidNamespaceURL, []byte(ip))
}
// JobConfig is the in-row snapshot of what an admin (or the scale loop)
// asked for. Mirrors the relevant subset of provisioning_templates plus the
// rendered worker env that the installer needs.
type JobConfig struct {
Provider string `json:"provider"`
Location string `json:"location"`
Datacenter string `json:"datacenter,omitempty"`
ServerType string `json:"server_type"`
Image string `json:"image"`
ServerCount int `json:"server_count"`
IPv4PerServer int `json:"ipv4_per_server"`
IPv6PerServer int `json:"ipv6_per_server"`
Tier string `json:"tier"`
EgressKind string `json:"egress_kind"`
Labels map[string]string `json:"labels,omitempty"`
PlacementGroup string `json:"placement_group,omitempty"`
PrivateNetwork string `json:"private_network,omitempty"`
Firewall string `json:"firewall,omitempty"`
ImageTag string `json:"image_tag"`
WorkerEnv string `json:"worker_env"`
SSHKeyID string `json:"ssh_key_id"` // pre-uploaded to provider
SSHPrivKeyPEM []byte `json:"ssh_priv_key_pem"` // encrypted at rest in config blob
SSHPort int `json:"ssh_port,omitempty"`
SSHUser string `json:"ssh_user,omitempty"`
RDNSPattern string `json:"rdns_pattern,omitempty"` // e.g. "w-{{ip}}.workers.example.com"
}
// Service is the orchestrator.
type Service struct {
Jobs repository.ProvisioningJobRepository
Providers map[string]cloudprovider.Provider // keyed by provider name
// ProviderResolver, when set, resolves the provider client for a job
// (e.g. building a Hetzner client from the job's stored credential).
// Takes precedence over the static Providers map; the map is the
// test/fallback path.
ProviderResolver func(ctx context.Context, job *repository.ProvisioningJob) (cloudprovider.Provider, error)
Installer Installer
// VerifyTimeout is how long Run waits for expected workers to heartbeat
// before failing the job. Default 5 min.
VerifyTimeout time.Duration
// VerifyAllReady is called repeatedly during the verify step to check
// which of the expected workers have registered via heartbeat. Returns
// the subset that are alive.
VerifyAllReady func(ctx context.Context, expected []uuid.UUID) ([]uuid.UUID, error)
}
// Run drives the given job to terminal state (completed or failed). Safe to
// call concurrently with itself for different jobs; not safe for the same job.
func (s *Service) Run(ctx context.Context, jobID uuid.UUID) error {
job, err := s.Jobs.Get(ctx, jobID)
if err != nil {
return fmt.Errorf("provisioning: load job: %w", err)
}
if job == nil {
return fmt.Errorf("provisioning: job %s not found", jobID)
}
provider, err := s.resolveProvider(ctx, job)
if err != nil {
return s.fail(ctx, jobID, err)
}
var cfg JobConfig
if err := json.Unmarshal(job.Config, &cfg); err != nil {
return s.fail(ctx, jobID, fmt.Errorf("decode config: %w", err))
}
if cfg.SSHPort == 0 {
cfg.SSHPort = 22
}
if cfg.SSHUser == "" {
cfg.SSHUser = "root"
}
if cfg.ServerCount <= 0 {
cfg.ServerCount = 1
}
if cfg.IPv4PerServer <= 0 {
cfg.IPv4PerServer = 1
}
// One server per Run call. server_count > 1 means the admin / scale loop
// must enqueue multiple jobs, one per server. Keeps each job's failure
// domain small.
for state := job.State; ; {
switch state {
case models.ProvJobPending:
state = models.ProvJobCreatingServer
case models.ProvJobCreatingServer:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
server, err := s.createServer(ctx, provider, cfg, jobID)
if err != nil {
return s.rollback(ctx, jobID, provider, fmt.Errorf("create_server: %w", err))
}
if err := s.Jobs.RecordServer(ctx, jobID, server.ID); err != nil {
return s.rollback(ctx, jobID, provider, err)
}
if err := s.Jobs.AppendIPs(ctx, jobID, []string{}, []string{server.PublicIPv4}); err != nil {
return s.rollback(ctx, jobID, provider, err)
}
job.ProviderServerID = &server.ID
job.IPs = append(job.IPs, server.PublicIPv4)
state = models.ProvJobCreatingIPs
case models.ProvJobCreatingIPs:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
// IPv4PerServer=1 means the server's default IP is the only IP.
// IPv4PerServer>1 means create (n-1) extra Primary IPs.
extra := cfg.IPv4PerServer - 1
if extra > 0 {
ipIDs := make([]string, 0, extra)
ips := make([]string, 0, extra)
for i := 0; i < extra; i++ {
ip, err := provider.CreatePrimaryIP(ctx, cloudprovider.CreatePrimaryIPRequest{
Type: "ipv4",
Name: fmt.Sprintf("warmbly-%s-%d", jobID.String()[:8], i),
Datacenter: cfg.Datacenter,
Labels: cfg.Labels,
})
if err != nil {
return s.rollback(ctx, jobID, provider, fmt.Errorf("create_ip %d: %w", i, err))
}
ipIDs = append(ipIDs, ip.ID)
ips = append(ips, ip.IP)
}
if err := s.Jobs.AppendIPs(ctx, jobID, ipIDs, ips); err != nil {
return s.rollback(ctx, jobID, provider, err)
}
job.ProviderIPIDs = append(job.ProviderIPIDs, ipIDs...)
job.IPs = append(job.IPs, ips...)
}
state = models.ProvJobAssigningIPs
case models.ProvJobAssigningIPs:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
if job.ProviderServerID != nil {
for _, ipID := range job.ProviderIPIDs {
if err := provider.AssignPrimaryIP(ctx, ipID, *job.ProviderServerID); err != nil {
return s.rollback(ctx, jobID, provider, fmt.Errorf("assign_ip %s: %w", ipID, err))
}
}
}
state = models.ProvJobSettingRDNS
case models.ProvJobSettingRDNS:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
if cfg.RDNSPattern != "" {
for i, ipID := range job.ProviderIPIDs {
ip := ""
if i+1 < len(job.IPs) {
ip = job.IPs[i+1] // index 0 is the server's default IP
}
hostname := strings.ReplaceAll(cfg.RDNSPattern, "{{ip}}", strings.ReplaceAll(ip, ".", "-"))
if err := provider.SetReverseDNS(ctx, ipID, hostname); err != nil {
// rDNS failure is non-fatal — log and continue.
log.Warn().Err(err).Str("ip", ip).Str("hostname", hostname).Msg("provisioning: SetReverseDNS failed")
continue
}
// Verify the PTR actually took and forward-confirms (FCrDNS).
// A missing/mismatched PTR is a classic mailbox-provider
// rejection cause, so surface it; non-fatal (DNS may still be
// propagating). Best-effort — only verify when we know the IP.
if ip != "" {
ptr, ok, vErr := VerifyReverseDNS(ctx, ip)
switch {
case vErr != nil:
log.Warn().Err(vErr).Str("ip", ip).Msg("provisioning: rDNS verification lookup failed")
case !ok:
log.Warn().Str("ip", ip).Str("ptr", ptr).Str("expected", hostname).Msg("provisioning: rDNS not yet forward-confirmed (FCrDNS)")
default:
log.Info().Str("ip", ip).Str("ptr", ptr).Msg("provisioning: rDNS forward-confirmed")
}
}
}
}
state = models.ProvJobInstalling
case models.ProvJobInstalling:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
if len(job.IPs) == 0 {
return s.rollback(ctx, jobID, provider, fmt.Errorf("no IPs to install on"))
}
expected := make([]uuid.UUID, 0, len(job.IPs))
for _, ip := range job.IPs {
expected = append(expected, WorkerIDForIP(ip))
}
res, err := s.Installer.Install(ctx, InstallRequest{
Host: job.IPs[0], // use server's default IP for SSH
SSHPort: cfg.SSHPort,
SSHUser: cfg.SSHUser,
SSHKeyPEM: cfg.SSHPrivKeyPEM,
IPs: job.IPs,
WorkerEnv: cfg.WorkerEnv,
ImageTag: cfg.ImageTag,
ExpectedIDs: expected,
})
if err != nil {
return s.rollback(ctx, jobID, provider, fmt.Errorf("install: %w", err))
}
if len(res.InstalledWorkerIDs) > 0 {
if err := s.Jobs.AppendWorkerIDs(ctx, jobID, res.InstalledWorkerIDs); err != nil {
return s.rollback(ctx, jobID, provider, err)
}
}
state = models.ProvJobVerifying
case models.ProvJobVerifying:
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
return err
}
timeout := s.VerifyTimeout
if timeout == 0 {
timeout = 5 * time.Minute
}
vctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
expected := make([]uuid.UUID, 0, len(job.IPs))
for _, ip := range job.IPs {
expected = append(expected, WorkerIDForIP(ip))
}
ok, err := s.waitForHeartbeats(vctx, expected)
if err != nil {
return s.rollback(ctx, jobID, provider, fmt.Errorf("verify: %w", err))
}
if !ok {
return s.rollback(ctx, jobID, provider, fmt.Errorf("verify: timeout waiting for workers to register"))
}
state = models.ProvJobCompleted
case models.ProvJobCompleted:
return s.Jobs.MarkCompleted(ctx, jobID)
case models.ProvJobFailed, models.ProvJobRollingBack:
return fmt.Errorf("provisioning job %s already terminal: %s", jobID, state)
default:
return s.fail(ctx, jobID, fmt.Errorf("unknown state %q", state))
}
}
}
func (s *Service) resolveProvider(ctx context.Context, job *repository.ProvisioningJob) (cloudprovider.Provider, error) {
if s.ProviderResolver != nil {
return s.ProviderResolver(ctx, job)
}
if p, ok := s.Providers[job.Provider]; ok {
return p, nil
}
return nil, fmt.Errorf("no provider client for %q", job.Provider)
}
func (s *Service) createServer(ctx context.Context, p cloudprovider.Provider, cfg JobConfig, jobID uuid.UUID) (*cloudprovider.Server, error) {
sshKeys := []string{}
if cfg.SSHKeyID != "" {
sshKeys = []string{cfg.SSHKeyID}
}
return p.CreateServer(ctx, cloudprovider.CreateServerRequest{
Name: fmt.Sprintf("warmbly-%s-%s", cfg.Location, jobID.String()[:8]),
ServerType: cfg.ServerType,
Image: cfg.Image,
Location: cfg.Location,
Datacenter: cfg.Datacenter,
SSHKeyIDs: sshKeys,
UserData: renderCloudInit(cfg),
Labels: cfg.Labels,
PlacementGroup: cfg.PlacementGroup,
PrivateNetwork: cfg.PrivateNetwork,
Firewall: cfg.Firewall,
StartAfterCreate: true,
})
}
func renderCloudInit(cfg JobConfig) string {
// Minimal cloud-init: ensure Docker is present so install-worker.sh
// doesn't have to fetch it. The installer handles the rest.
return `#cloud-config
package_update: true
packages:
- curl
- ca-certificates
- docker.io
runcmd:
- [systemctl, enable, --now, docker]
`
}
func (s *Service) waitForHeartbeats(ctx context.Context, expected []uuid.UUID) (bool, error) {
if s.VerifyAllReady == nil {
// No verify hook wired — assume installer-side check is sufficient.
return true, nil
}
wantSet := map[uuid.UUID]struct{}{}
for _, id := range expected {
wantSet[id] = struct{}{}
}
tick := time.NewTicker(5 * time.Second)
defer tick.Stop()
for {
alive, err := s.VerifyAllReady(ctx, expected)
if err != nil {
return false, err
}
for _, id := range alive {
delete(wantSet, id)
}
if len(wantSet) == 0 {
return true, nil
}
select {
case <-ctx.Done():
return false, nil
case <-tick.C:
}
}
}
// rollback transitions the job to rolling_back, undoes provider-side
// resources we created, then marks the job failed.
func (s *Service) rollback(ctx context.Context, jobID uuid.UUID, p cloudprovider.Provider, rootCause error) error {
_ = s.Jobs.UpdateState(ctx, jobID, models.ProvJobRollingBack)
job, _ := s.Jobs.Get(ctx, jobID)
if job != nil {
for _, ipID := range job.ProviderIPIDs {
_ = p.UnassignPrimaryIP(ctx, ipID)
_ = p.DeletePrimaryIP(ctx, ipID)
}
if job.ProviderServerID != nil {
_ = p.DeleteServer(ctx, *job.ProviderServerID)
}
}
return s.fail(ctx, jobID, rootCause)
}
func (s *Service) fail(ctx context.Context, jobID uuid.UUID, err error) error {
_ = s.Jobs.MarkFailed(ctx, jobID, err.Error())
return err
}
// SubmitConfigJSON marshals a JobConfig to the raw JSON the row stores.
func SubmitConfigJSON(cfg JobConfig) (json.RawMessage, error) {
return json.Marshal(cfg)
}
// randomToken is a small helper for default name suffixes.
func randomToken(n int) string {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return ""
}
return hex.EncodeToString(b)
}
// ParseIPs is a small helper for callers that need to round-trip the
// INET[]→string→net.IP conversion when reading job rows.
func ParseIPs(raw []string) []net.IP {
out := make([]net.IP, 0, len(raw))
for _, s := range raw {
if ip := net.ParseIP(s); ip != nil {
out = append(out, ip)
}
}
return out
}
+125 -149
View File
@@ -1,15 +1,17 @@
// Package releases drives worker auto-update based on GitHub Releases.
// Package releases resolves which version the fleet should be running.
//
// All configuration is env-driven so this is safe to ship in self-hosted
// deployments. A self-hoster can:
// - point the poller at their own fork (RELEASES_GITHUB_REPO)
// - point the image at their own registry (RELEASES_WORKER_IMAGE_REPO)
// - disable the feature entirely (RELEASES_ENABLED=false)
// It does not update anything. In the pull model a node asks, on every
// heartbeat, what version it should be, compares that to what it is, and
// updates itself. This package's whole job is to keep the answer current: it
// reads GitHub Releases, picks the head of the configured channel, and writes
// the resolved tag into admin_settings where the heartbeat handler reads it.
//
// The service is push-driven, not poll-driven: a single check runs on
// backend boot to sync state, and from then on the GitHub webhook
// (POST /webhooks/github/releases) triggers checks. There is no recurring
// timer.
// All configuration is env-driven so this is safe in self-hosted deployments.
// A self-hoster can point the poller at their own fork
// (RELEASES_GITHUB_REPO), point nodes at their own registry
// (RELEASES_WORKER_IMAGE_REPO), or disable the feature entirely
// (RELEASES_ENABLED=false), in which case nodes are told nothing and leave
// themselves alone.
package releases
import (
@@ -17,17 +19,13 @@ import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/worker_orchestrator"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
@@ -42,14 +40,13 @@ type Config struct {
}
type Service struct {
cfg Config
credsRepo repository.CredentialsRepository
workerRepo repository.WorkerRepository
orchestrator *worker_orchestrator.Orchestrator
http *http.Client
cfg Config
settings repository.FleetSettingsRepository
http *http.Client
// In-memory cache of the latest resolved versions per channel.
// Populated on every CheckGitHub call; surfaced via GetState() to UI.
// In-memory view of the last check, surfaced to the dashboard. The
// authoritative resolved tag lives in admin_settings, not here, so every
// backend replica answers heartbeats identically.
stateMu sync.Mutex
state State
}
@@ -71,21 +68,14 @@ type ChannelView struct {
HTMLURL string `json:"html_url,omitempty"`
}
func New(
cfg Config,
credsRepo repository.CredentialsRepository,
workerRepo repository.WorkerRepository,
orchestrator *worker_orchestrator.Orchestrator,
) *Service {
func New(cfg Config, settings repository.FleetSettingsRepository) *Service {
if cfg.HTTPClient == nil {
cfg.HTTPClient = &http.Client{Timeout: 15 * time.Second}
}
return &Service{
cfg: cfg,
credsRepo: credsRepo,
workerRepo: workerRepo,
orchestrator: orchestrator,
http: cfg.HTTPClient,
cfg: cfg,
settings: settings,
http: cfg.HTTPClient,
state: State{
Channels: map[string]ChannelView{},
GithubRepo: cfg.GithubRepo,
@@ -95,15 +85,14 @@ func New(
}
}
// public
// CheckGitHub resolves stable + dev channels by hitting the GitHub Releases
// API, updates each profile that subscribes to a channel, and (if the
// profile has auto_update=true) rolls every assigned worker to the new
// image.
// CheckGitHub resolves the stable and dev channel heads and, unless the fleet
// is pinned, records the head of the configured channel as the version every
// node should converge on.
//
// Returns the list of profiles that changed and the resolved state.
func (s *Service) CheckGitHub(ctx context.Context) (changed []ProfileUpdate, err error) {
// It never writes an empty tag: a GitHub outage or an empty release list
// leaves the previous answer in place, because telling the fleet "no version"
// would read as "stop updating" at best and be acted on at worst.
func (s *Service) CheckGitHub(ctx context.Context) (*models.FleetReleaseState, error) {
if !s.cfg.Enabled {
return nil, errors.New("releases not enabled")
}
@@ -113,85 +102,108 @@ func (s *Service) CheckGitHub(ctx context.Context) (changed []ProfileUpdate, err
s.recordError(err.Error())
return nil, err
}
stable, dev := PickChannelHeads(releases)
now := time.Now()
s.stateMu.Lock()
s.state.LastCheckedAt = now
s.state.LastCheckedAt = time.Now()
s.state.LastError = ""
s.state.Channels = map[string]ChannelView{}
if stable != nil {
s.state.Channels["stable"] = s.channelView("stable", stable)
s.state.Channels[models.FleetChannelStable] = s.channelView(models.FleetChannelStable, stable)
}
if dev != nil {
s.state.Channels["dev"] = s.channelView("dev", dev)
s.state.Channels[models.FleetChannelDev] = s.channelView(models.FleetChannelDev, dev)
}
s.stateMu.Unlock()
for _, channel := range []models.ReleaseChannel{models.ReleaseChannelStable, models.ReleaseChannelDev} {
var target *Release
switch channel {
case models.ReleaseChannelStable:
target = stable
case models.ReleaseChannelDev:
target = dev
}
if target == nil {
continue
}
image := s.imageFor(target.TagName)
profiles, err := s.credsRepo.ListProfilesByChannel(ctx, channel)
if err != nil {
return changed, fmt.Errorf("list profiles for %s: %w", channel, err)
}
for _, p := range profiles {
if p.WorkerImage == image && p.ResolvedImageTag == target.TagName {
continue // already on the latest
}
if err := s.credsRepo.RecordResolvedTag(ctx, p.ID, image, target.TagName); err != nil {
log.Printf("releases: record tag for %s: %v", p.Name, err)
continue
}
pu := ProfileUpdate{
ProfileID: p.ID,
ProfileName: p.Name,
Channel: string(channel),
NewTag: target.TagName,
NewImage: image,
AutoApplied: p.AutoUpdate,
}
if p.AutoUpdate {
pu.Rollout = s.rollout(ctx, p.ID, image)
}
changed = append(changed, pu)
}
current, err := s.settings.GetRelease(ctx)
if err != nil {
return nil, err
}
if current == nil {
current = &models.FleetReleaseState{Channel: models.FleetChannelStable}
}
if current.Channel == models.FleetChannelPinned {
// Explicitly held. Channel heads are still reported to the dashboard so
// an operator can see what they are declining.
return current, nil
}
return changed, nil
var head *Release
switch current.Channel {
case models.FleetChannelDev:
head = dev
default:
head = stable
}
if head == nil || head.TagName == "" {
return current, nil
}
if head.TagName == current.Tag {
return current, nil
}
next := &models.FleetReleaseState{
Channel: current.Channel,
Tag: head.TagName,
ResolvedAt: time.Now(),
Source: "github:" + current.Channel,
}
if err := s.settings.SetRelease(ctx, next); err != nil {
return nil, err
}
log.Printf("releases: fleet target is now %s (channel %s); nodes will self-update on their next heartbeat",
next.Tag, next.Channel)
return next, nil
}
type ProfileUpdate struct {
ProfileID uuid.UUID `json:"profile_id"`
ProfileName string `json:"profile_name"`
Channel string `json:"channel"`
NewTag string `json:"new_tag"`
NewImage string `json:"new_image"`
AutoApplied bool `json:"auto_applied"`
Rollout []RolloutEntry `json:"rollout,omitempty"`
// SetChannel switches which channel the fleet follows and immediately
// re-resolves, so the change takes effect on the next heartbeat rather than
// whenever the next release happens to land.
func (s *Service) SetChannel(ctx context.Context, channel string) (*models.FleetReleaseState, error) {
switch channel {
case models.FleetChannelStable, models.FleetChannelDev, models.FleetChannelPinned:
default:
return nil, errors.New("unknown channel: " + channel)
}
current, err := s.settings.GetRelease(ctx)
if err != nil {
return nil, err
}
if current == nil {
current = &models.FleetReleaseState{}
}
current.Channel = channel
if err := s.settings.SetRelease(ctx, current); err != nil {
return nil, err
}
if channel == models.FleetChannelPinned {
return current, nil
}
return s.CheckGitHub(ctx)
}
type RolloutEntry struct {
WorkerID uuid.UUID `json:"worker_id"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
Skipped string `json:"skipped,omitempty"`
// SetTag pins the fleet to an explicit tag. Used to roll back: it switches the
// channel to pinned so the next GitHub check does not immediately undo it.
func (s *Service) SetTag(ctx context.Context, tag string) (*models.FleetReleaseState, error) {
if strings.TrimSpace(tag) == "" {
return nil, errors.New("tag required")
}
next := &models.FleetReleaseState{
Channel: models.FleetChannelPinned,
Tag: tag,
ResolvedAt: time.Now(),
Source: "manual",
}
if err := s.settings.SetRelease(ctx, next); err != nil {
return nil, err
}
return next, nil
}
// HandleWebhook validates the GitHub `release` event signature and triggers
// a CheckGitHub. Returns 401 on signature mismatch only the secret holder
// can fire updates.
// HandleWebhook validates the GitHub `release` event signature and triggers a
// re-resolve. Returns an error on signature mismatch: only the secret holder
// can move the fleet.
func (s *Service) HandleWebhook(ctx context.Context, body []byte, signature, eventType string) error {
if !s.cfg.Enabled {
return errors.New("releases not enabled")
@@ -200,30 +212,15 @@ func (s *Service) HandleWebhook(ctx context.Context, body []byte, signature, eve
return errors.New("webhook secret not configured")
}
if !verifySignature(s.cfg.WebhookSecret, body, signature) {
return errors.New("bad signature")
return errors.New("signature mismatch")
}
// Only react to release publish events. GitHub fires several action types
// for `release`; published/released are the ones that mean "new artifact
// is live."
if eventType == "release" {
var ev struct {
Action string `json:"action"`
}
_ = json.Unmarshal(body, &ev)
switch ev.Action {
case "published", "released", "edited", "":
_, err := s.CheckGitHub(ctx)
return err
default:
return nil // ignore (created/prereleased/deleted etc.)
}
if eventType != "release" {
return nil
}
// For ping or other events, just succeed silently.
return nil
_, err := s.CheckGitHub(ctx)
return err
}
// GetState returns the last known per-channel resolution. Cheap; reads from
// the in-memory cache populated by CheckGitHub.
func (s *Service) GetState() State {
s.stateMu.Lock()
defer s.stateMu.Unlock()
@@ -236,15 +233,15 @@ func (s *Service) GetState() State {
return st
}
// RunBootCheck is fire-and-forget: when the backend starts, sync state once
// so the dashboard isn't empty. Errors are logged, not surfaced.
// RunBootCheck syncs once on start so the dashboard is not empty and a fleet
// brought up after a release converges without waiting for a webhook.
func (s *Service) RunBootCheck(ctx context.Context) {
if !s.cfg.Enabled {
log.Printf("releases: disabled")
log.Printf("releases: disabled; nodes will not be told to update")
return
}
if s.cfg.GithubRepo == "" || s.cfg.WorkerImageRepo == "" {
log.Printf("releases: skipping boot check (RELEASES_GITHUB_REPO or RELEASES_WORKER_IMAGE_REPO unset)")
if s.cfg.GithubRepo == "" {
log.Printf("releases: skipping boot check (RELEASES_GITHUB_REPO unset)")
return
}
go func() {
@@ -256,28 +253,6 @@ func (s *Service) RunBootCheck(ctx context.Context) {
}()
}
// internals
func (s *Service) rollout(ctx context.Context, profileID uuid.UUID, image string) []RolloutEntry {
workers, err := s.workerRepo.ListWorkersByProfile(ctx, profileID)
if err != nil {
return []RolloutEntry{{OK: false, Error: "list workers: " + err.Error()}}
}
out := make([]RolloutEntry, 0, len(workers))
for _, w := range workers {
if w.InstallState != models.WorkerInstallStateInstalled {
out = append(out, RolloutEntry{WorkerID: w.ID, OK: false, Skipped: "not installed"})
continue
}
if err := s.orchestrator.UpdateToImage(ctx, w.ID, image); err != nil {
out = append(out, RolloutEntry{WorkerID: w.ID, OK: false, Error: err.Error()})
continue
}
out = append(out, RolloutEntry{WorkerID: w.ID, OK: true})
}
return out
}
func (s *Service) recordError(msg string) {
s.stateMu.Lock()
defer s.stateMu.Unlock()
@@ -287,6 +262,9 @@ func (s *Service) recordError(msg string) {
func (s *Service) imageFor(tag string) string {
repo := strings.TrimRight(s.cfg.WorkerImageRepo, "/")
if repo == "" {
return tag
}
return repo + ":" + tag
}
@@ -300,8 +278,6 @@ func (s *Service) channelView(name string, r *Release) ChannelView {
}
}
// HMAC verification
func verifySignature(secret string, body []byte, header string) bool {
if !strings.HasPrefix(header, "sha256=") {
return false
+30 -29
View File
@@ -12,6 +12,7 @@ import (
"github.com/warmbly/warmbly/internal/observability/errs"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/stripe/stripe-go/v76"
portalsession "github.com/stripe/stripe-go/v76/billingportal/session"
"github.com/stripe/stripe-go/v76/checkout/session"
@@ -1074,32 +1075,37 @@ func (s *stripeService) handleSubscriptionUpdated(ctx context.Context, event *st
if s.workerAssignment != nil {
isNowPaid := sub.HasPaidSubscription()
// Trial user converting to paid - migrate to premium workers.
// Use a bounded timeout context since these goroutines outlive the HTTP request.
if wasTrialOnly && isNowPaid {
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
s.workerAssignment.MigrateOrgToPremiumWorkers(bgCtx, sub.OrganizationID)
}()
}
// Converting a trial to paid no longer moves anything: workers are
// interchangeable, so an org's mailboxes are already wherever the
// placer thinks they belong.
_ = wasTrialOnly
_ = isNowPaid
// Handle dedicated worker migration on plan change
// Isolated egress is the only plan change with a placement effect, and
// it is a reservation, not a migration: the rotation loop converges the
// org's mailboxes onto the reserved worker on its own schedule, which
// keeps a plan change from re-authenticating every mailbox at once.
if newPlan != nil && newPlan.ID != oldPlanID {
hadDedicated := oldPlan != nil && oldPlan.DedicatedWorkers > 0
needsDedicated := newPlan.DedicatedWorkers > 0
hadIsolation := oldPlan.IsolatedEgress()
needsIsolation := newPlan.IsolatedEgress()
if !hadDedicated && needsDedicated {
orgID, subID := sub.OrganizationID, sub.ID
switch {
case !hadIsolation && needsIsolation:
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
s.workerAssignment.MigrateOrgToDedicated(bgCtx, sub.OrganizationID, sub.ID)
if err := s.workerAssignment.ReserveIsolatedWorker(bgCtx, orgID, subID); err != nil {
log.Warn().Err(err).Str("org_id", orgID.String()).Msg("stripe: reserve isolated worker failed")
}
}()
} else if hadDedicated && !needsDedicated {
case hadIsolation && !needsIsolation:
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
s.workerAssignment.MigrateOrgToShared(bgCtx, sub.OrganizationID)
if err := s.workerAssignment.ReleaseIsolatedWorker(bgCtx, orgID); err != nil {
log.Warn().Err(err).Str("org_id", orgID.String()).Msg("stripe: release isolated worker failed")
}
}()
}
}
@@ -1119,9 +1125,8 @@ func (s *stripeService) handleSubscriptionDeleted(ctx context.Context, event *st
return nil
}
// Check if org had dedicated workers
oldPlan, _ := s.planRepo.GetByID(ctx, sub.PlanID)
hadDedicated := oldPlan != nil && oldPlan.DedicatedWorkers > 0
hadIsolation := oldPlan.IsolatedEgress()
sub.Status = models.SubscriptionStatusCanceled
canceledAt := time.Now()
@@ -1131,21 +1136,17 @@ func (s *stripeService) handleSubscriptionDeleted(ctx context.Context, event *st
return errx.New(errx.Internal, "failed to update subscription")
}
// Handle worker migration - move back to free tier workers.
// Use bounded timeout context since these goroutines outlive the HTTP request.
if s.workerAssignment != nil {
// Cancelling releases the reserved worker back to the fleet. Nothing else
// moves: a cancelled org's mailboxes keep the workers they are on, which
// is both cheaper and better for them than a forced re-authentication.
if s.workerAssignment != nil && hadIsolation {
orgID := sub.OrganizationID
if hadDedicated {
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
s.workerAssignment.MigrateOrgToShared(bgCtx, orgID)
}()
}
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
s.workerAssignment.MigrateOrgToFreeWorkers(bgCtx, orgID)
if err := s.workerAssignment.ReleaseIsolatedWorker(bgCtx, orgID); err != nil {
log.Warn().Err(err).Str("org_id", orgID.String()).Msg("stripe: release isolated worker failed")
}
}()
}
File diff suppressed because it is too large Load Diff
-544
View File
@@ -1,544 +0,0 @@
// Contract tests for plan-aware worker assignment.
//
// These lock the rule that's at the heart of multi-tenant deliverability:
// who lands on which worker is a function of the org's subscription, not of
// the request shape or who happens to be online. If this contract ever
// regresses, free-tier orgs could leak onto premium workers (or vice versa)
// and tank the IPs of paying customers.
//
// Test style: hand-rolled stub repos that embed the interface as a nil
// field so only the methods AssignWorkerToEmail actually touches need
// real bodies — anything else panics, which is the desired loud failure.
package worker
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// stubs
type stubWorkerRepo struct {
repository.WorkerRepository // embed for the panic-on-unused-method behavior
dedicatedForOrg *models.Worker
sharedFree []models.Worker
sharedPremium []models.Worker
capacityFree []repository.WorkerCapacityRowDB
capacityPremium []repository.WorkerCapacityRowDB
workersByID map[uuid.UUID]models.Worker
placementHint *repository.EmailAccountPlacementHint
lastEmailWorkerAssigned uuid.UUID
lastEmailPoolTypeSet string
incrementedWorkerCounts map[uuid.UUID]int
loadScoreDeltas map[uuid.UUID]float64
// Dedicated auto-promotion knobs.
availableDedicated *models.Worker // GetAvailableDedicatedWorker result
promotableDedicated *models.Worker // PromoteIdlePremiumWorkerToDedicated result
dedicatedAssignCreated bool // CreateDedicatedAssignmentIfNotExists result
setWorkerTypeCalls map[uuid.UUID]models.WorkerType // records SetWorkerType calls
// Risk-band placement knobs.
riskBand models.EmailRiskBand // GetEmailAccountRiskBand result ("" → clean)
sharedByPool map[models.WorkerRiskPool][]models.Worker // GetSharedWorkersByTierAndPool result
promotedToPool *models.Worker // PromoteWorkerToPool result
}
func (r *stubWorkerRepo) GetDedicatedWorkerByOrgID(_ context.Context, _ uuid.UUID) (*models.Worker, error) {
return r.dedicatedForOrg, nil
}
func (r *stubWorkerRepo) GetSharedWorkersByTier(_ context.Context, freeTier bool) ([]models.Worker, error) {
if freeTier {
return r.sharedFree, nil
}
return r.sharedPremium, nil
}
func (r *stubWorkerRepo) UpdateEmailAccountWorker(_ context.Context, emailID, workerID uuid.UUID) error {
r.lastEmailWorkerAssigned = workerID
return nil
}
func (r *stubWorkerRepo) IncrementAccountCount(_ context.Context, workerID uuid.UUID) error {
if r.incrementedWorkerCounts == nil {
r.incrementedWorkerCounts = map[uuid.UUID]int{}
}
r.incrementedWorkerCounts[workerID]++
return nil
}
func (r *stubWorkerRepo) UpdateEmailAccountWarmupPoolType(_ context.Context, _ uuid.UUID, pool string) error {
r.lastEmailPoolTypeSet = pool
return nil
}
// Capacity-aware methods. Default behaviour: empty capacity view so the
// assignment service falls back to the legacy account-count path. Tests
// that want to exercise the capacity-aware path populate
// capacityFree/capacityPremium + workersByID.
func (r *stubWorkerRepo) ListCapacityCandidates(_ context.Context, freeTier bool, _ []models.WorkerHealthState) ([]repository.WorkerCapacityRowDB, error) {
if freeTier {
return r.capacityFree, nil
}
return r.capacityPremium, nil
}
func (r *stubWorkerRepo) GetByID(_ context.Context, id uuid.UUID) (*models.Worker, error) {
w, ok := r.workersByID[id]
if !ok {
return nil, nil
}
return &w, nil
}
func (r *stubWorkerRepo) GetEmailAccountPlacementHint(_ context.Context, _ uuid.UUID) (*repository.EmailAccountPlacementHint, error) {
return r.placementHint, nil
}
func (r *stubWorkerRepo) AddLoadScore(_ context.Context, workerID uuid.UUID, delta float64) error {
if r.loadScoreDeltas == nil {
r.loadScoreDeltas = map[uuid.UUID]float64{}
}
r.loadScoreDeltas[workerID] += delta
return nil
}
func (r *stubWorkerRepo) GetEmailAccountRiskBand(_ context.Context, _ uuid.UUID) (models.EmailRiskBand, error) {
if r.riskBand == "" {
return models.EmailRiskBandClean, nil
}
return r.riskBand, nil
}
func (r *stubWorkerRepo) GetAvailableDedicatedWorker(_ context.Context) (*models.Worker, error) {
return r.availableDedicated, nil
}
func (r *stubWorkerRepo) PromoteIdlePremiumWorkerToDedicated(_ context.Context) (*models.Worker, error) {
if r.promotableDedicated == nil {
return nil, nil
}
w := *r.promotableDedicated
w.WorkerType = models.WorkerTypeDedicated
return &w, nil
}
func (r *stubWorkerRepo) CreateDedicatedAssignmentIfNotExists(_ context.Context, a *models.DedicatedWorkerAssignment) (bool, error) {
if r.dedicatedAssignCreated {
// Bind it so the post-assign re-fetch in AssignWorkerToEmail finds it.
r.dedicatedForOrg = &models.Worker{ID: a.WorkerID, WorkerType: models.WorkerTypeDedicated, Active: true}
}
return r.dedicatedAssignCreated, nil
}
func (r *stubWorkerRepo) SetWorkerType(_ context.Context, id uuid.UUID, t models.WorkerType) error {
if r.setWorkerTypeCalls == nil {
r.setWorkerTypeCalls = map[uuid.UUID]models.WorkerType{}
}
r.setWorkerTypeCalls[id] = t
return nil
}
func (r *stubWorkerRepo) GetSharedWorkersByTierAndPool(_ context.Context, _ bool, pool models.WorkerRiskPool) ([]models.Worker, error) {
return r.sharedByPool[pool], nil
}
func (r *stubWorkerRepo) PromoteWorkerToPool(_ context.Context, _ bool, _ models.WorkerRiskPool) (*models.Worker, error) {
return r.promotedToPool, nil
}
type stubSubRepo struct {
repository.SubscriptionRepository
sub *models.Subscription
}
func (r *stubSubRepo) GetByOrganizationID(_ context.Context, _ uuid.UUID) (*models.Subscription, error) {
return r.sub, nil
}
type stubPlanRepo struct {
repository.PlanRepository
plan *models.Plan
}
func (r *stubPlanRepo) GetByID(_ context.Context, _ uuid.UUID) (*models.Plan, error) {
return r.plan, nil
}
// tests
func newWorker(id uuid.UUID, freeTier bool, wtype models.WorkerType) models.Worker {
return models.Worker{ID: id, FreeTier: freeTier, WorkerType: wtype, Active: true}
}
func TestAssign_FreeOrg_LandsOnFreeSharedWorker(t *testing.T) {
// Tier separation only exists when billing does. With BILLING_PROVIDER=none
// (the self-host default) there is no paid/free split to enforce — see
// TestAssign_SelfHost_PlacesEveryOrgAsPaid.
t.Setenv("BILLING_PROVIDER", "stripe")
freeWorker := newWorker(uuid.New(), true, models.WorkerTypeShared)
premiumWorker := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
sharedFree: []models.Worker{freeWorker},
sharedPremium: []models.Worker{premiumWorker},
}
svc := NewAssignmentService(wr, &stubSubRepo{sub: nil}, &stubPlanRepo{})
emailID := uuid.New()
got, err := svc.AssignWorkerToEmail(context.Background(), emailID, uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != freeWorker.ID {
t.Errorf("free org should land on free worker, got %s (free=%s)", got, freeWorker.ID)
}
if wr.lastEmailPoolTypeSet != "free" {
t.Errorf("free org should join the free warmup pool, got %q", wr.lastEmailPoolTypeSet)
}
}
func TestAssign_PaidOrg_LandsOnPremiumSharedWorker(t *testing.T) {
freeWorker := newWorker(uuid.New(), true, models.WorkerTypeShared)
premiumWorker := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
sharedFree: []models.Worker{freeWorker},
sharedPremium: []models.Worker{premiumWorker},
}
// Subscription is active but plan has no dedicated workers.
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 0}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != premiumWorker.ID {
t.Errorf("paid org should land on premium worker, got %s (premium=%s)", got, premiumWorker.ID)
}
if wr.lastEmailPoolTypeSet != "premium" {
t.Errorf("paid org should join the premium warmup pool, got %q", wr.lastEmailPoolTypeSet)
}
}
func TestAssign_PaidOrgWithDedicatedPlan_LandsOnDedicatedWorker(t *testing.T) {
dedicated := newWorker(uuid.New(), false, models.WorkerTypeDedicated)
premium := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
dedicatedForOrg: &dedicated,
sharedPremium: []models.Worker{premium},
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 1}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != dedicated.ID {
t.Errorf("paid org with dedicated plan + assignment should land on the dedicated worker, got %s", got)
}
}
func TestAssign_PaidOrgWithDedicatedPlanButNoAssignment_FallsBackToPremium(t *testing.T) {
// Dedicated plan, no bound worker, no free dedicated worker, AND nothing
// to promote (no idle premium spare). The add must still succeed by
// falling back to a premium shared worker rather than failing.
premium := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
dedicatedForOrg: nil, // org has the plan but no worker assigned yet
availableDedicated: nil, // dedicated pool is empty
promotableDedicated: nil, // and there's no idle premium spare to promote
sharedPremium: []models.Worker{premium},
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 1}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != premium.ID {
t.Errorf("org with dedicated plan but nothing to promote should fall back to premium, got %s", got)
}
}
func TestAssign_PaidOrgWithDedicatedPlanButNoAssignment_PromotesSpareWorker(t *testing.T) {
// Dedicated plan, no bound worker, dedicated pool empty — but an idle
// premium shared worker is available to promote. The mailbox must land on
// the promoted (now dedicated) worker, not on the shared pool.
spare := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
dedicatedForOrg: nil,
availableDedicated: nil,
promotableDedicated: &spare,
dedicatedAssignCreated: true, // bind succeeds
sharedPremium: []models.Worker{newWorker(uuid.New(), false, models.WorkerTypeShared)},
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 1}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != spare.ID {
t.Errorf("org with dedicated plan should land on the promoted spare worker %s, got %s", spare.ID, got)
}
}
func TestAssignDedicatedWorker_PromoteThenLoseBind_RevertsToShared(t *testing.T) {
// Promotion succeeds but the bind race is lost (a concurrent add bound the
// org first). The just-promoted worker must be reverted to shared so it
// isn't stranded as an unbound dedicated box.
spare := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
availableDedicated: nil, // dedicated pool empty → promote
promotableDedicated: &spare,
dedicatedAssignCreated: false, // lose the bind race
}
svc := NewAssignmentService(wr, &stubSubRepo{}, &stubPlanRepo{})
err := svc.AssignDedicatedWorker(context.Background(), uuid.New(), uuid.New())
if !errors.Is(err, ErrOrgAlreadyAssigned) {
t.Fatalf("expected ErrOrgAlreadyAssigned on lost bind race, got %v", err)
}
if got, ok := wr.setWorkerTypeCalls[spare.ID]; !ok || got != models.WorkerTypeShared {
t.Errorf("promoted worker must be reverted to shared after losing the bind race, got %v (called=%v)", got, ok)
}
}
func TestAssign_RiskyMailbox_LandsOnRiskyPoolWorker(t *testing.T) {
// A risky mailbox must land on a worker in the risky pool, never on a
// clean-pool worker, so it can't damage the reputation of trusted inboxes.
riskyWorker := newWorker(uuid.New(), false, models.WorkerTypeShared)
riskyWorker.RiskPool = models.WorkerRiskPoolRisky
cleanWorker := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{
riskBand: models.EmailRiskBandRisky,
sharedByPool: map[models.WorkerRiskPool][]models.Worker{
models.WorkerRiskPoolRisky: {riskyWorker},
models.WorkerRiskPoolClean: {cleanWorker},
},
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 0}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != riskyWorker.ID {
t.Errorf("risky mailbox must land on a risky-pool worker %s, got %s", riskyWorker.ID, got)
}
}
func TestAssign_RiskyMailbox_NoRiskyWorker_PromotesCleanWorker(t *testing.T) {
// When no risky-pool worker exists, we promote an idle clean worker into
// the risky pool rather than co-locating with trusted inboxes.
promoted := newWorker(uuid.New(), false, models.WorkerTypeShared)
promoted.RiskPool = models.WorkerRiskPoolRisky
wr := &stubWorkerRepo{
riskBand: models.EmailRiskBandRisky,
sharedByPool: map[models.WorkerRiskPool][]models.Worker{}, // risky pool empty
promotedToPool: &promoted,
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 0}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != promoted.ID {
t.Errorf("risky mailbox with empty pool should land on the promoted worker %s, got %s", promoted.ID, got)
}
}
func TestAssign_RiskyMailbox_NoWorkerNoPromotion_Refuses(t *testing.T) {
// Strict invariant: if there's no risky-pool worker and nothing to
// promote, refuse rather than place a risky inbox next to trusted ones.
wr := &stubWorkerRepo{
riskBand: models.EmailRiskBandQuarantine,
sharedByPool: map[models.WorkerRiskPool][]models.Worker{},
promotedToPool: nil,
}
sub := paidSub()
plan := &models.Plan{ID: sub.PlanID, DedicatedWorkers: 0}
svc := NewAssignmentService(wr, &stubSubRepo{sub: sub}, &stubPlanRepo{plan: plan})
_, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if !errors.Is(err, ErrNoAvailableWorkers) {
t.Errorf("strict placement should refuse with ErrNoAvailableWorkers, got %v", err)
}
}
func TestSelectSharedWorker_NoWorkers_Errors(t *testing.T) {
svc := NewAssignmentService(&stubWorkerRepo{}, &stubSubRepo{}, &stubPlanRepo{})
if _, err := svc.SelectSharedWorker(context.Background(), true); err != ErrNoAvailableWorkers {
t.Fatalf("expected ErrNoAvailableWorkers, got %v", err)
}
}
// paidSub returns a minimal subscription that HasPaidSubscription() will
// return true for: status == "active" AND StripeSubscriptionID set.
func paidSub() *models.Subscription {
sid := "sub_test_" + uuid.NewString()
return &models.Subscription{
ID: uuid.New(),
PlanID: uuid.New(),
Status: models.SubscriptionStatusActive,
StripeSubscriptionID: &sid,
}
}
// capacity-aware selection tests
//
// These exercise the new path: ListCapacityCandidates returns rows,
// the service computes utilization, sorts ASC, and lands the mailbox on
// the least-utilised worker that still has headroom for the weight.
func capacityRow(id uuid.UUID, base, load float64) repository.WorkerCapacityRowDB {
return repository.WorkerCapacityRowDB{
WorkerID: id,
WorkerType: models.WorkerTypeShared,
FreeTier: true,
EgressKind: models.WorkerEgressColdSMTP,
HealthState: models.WorkerHealthHealthy,
LoadScore: load,
BaseCapacity: base,
HealthMultiplier: 1.0,
AgeMultiplier: 1.0,
}
}
func TestSelectSharedWorker_CapacityAware_LeastUtilizedWins(t *testing.T) {
hot := uuid.New() // 14/16 utilised
cold := uuid.New() // 2/16 utilised
wr := &stubWorkerRepo{
capacityFree: []repository.WorkerCapacityRowDB{
capacityRow(hot, 16, 14),
capacityRow(cold, 16, 2),
},
workersByID: map[uuid.UUID]models.Worker{
hot: {ID: hot, FreeTier: true, WorkerType: models.WorkerTypeShared, Active: true},
cold: {ID: cold, FreeTier: true, WorkerType: models.WorkerTypeShared, Active: true},
},
}
svc := NewAssignmentService(wr, &stubSubRepo{}, &stubPlanRepo{})
got, err := svc.SelectSharedWorker(context.Background(), true)
if err != nil {
t.Fatalf("SelectSharedWorker: %v", err)
}
if got.ID != cold {
t.Errorf("least-utilized should win: got %s, want %s", got.ID, cold)
}
}
func TestSelectSharedWorker_CapacityAware_FiltersOutSaturated(t *testing.T) {
// Saturated worker (load == base) has zero headroom. Filtered out;
// the only remaining candidate wins.
saturated := uuid.New()
headroom := uuid.New()
wr := &stubWorkerRepo{
capacityFree: []repository.WorkerCapacityRowDB{
capacityRow(saturated, 16, 16),
capacityRow(headroom, 16, 4),
},
workersByID: map[uuid.UUID]models.Worker{
saturated: {ID: saturated, FreeTier: true, WorkerType: models.WorkerTypeShared, Active: true},
headroom: {ID: headroom, FreeTier: true, WorkerType: models.WorkerTypeShared, Active: true},
},
}
svc := NewAssignmentService(wr, &stubSubRepo{}, &stubPlanRepo{})
got, err := svc.SelectSharedWorker(context.Background(), true)
if err != nil {
t.Fatalf("SelectSharedWorker: %v", err)
}
if got.ID != headroom {
t.Errorf("saturated worker should be filtered, got %s, want %s", got.ID, headroom)
}
}
func TestSelectSharedWorker_CapacityAware_FallsBackWhenAllSaturated(t *testing.T) {
// Every worker is full; the legacy account_count path catches us so
// we don't fail the assignment outright. Falls back through
// selectSharedWorkerLegacy -> GetSharedWorkersByTier.
a := newWorker(uuid.New(), true, models.WorkerTypeShared)
wr := &stubWorkerRepo{
capacityFree: []repository.WorkerCapacityRowDB{
capacityRow(a.ID, 16, 16),
},
workersByID: map[uuid.UUID]models.Worker{a.ID: a},
sharedFree: []models.Worker{a},
}
svc := NewAssignmentService(wr, &stubSubRepo{}, &stubPlanRepo{})
got, err := svc.SelectSharedWorker(context.Background(), true)
if err != nil {
t.Fatalf("SelectSharedWorker: %v", err)
}
if got.ID != a.ID {
t.Errorf("fallback should still return the only worker, got %s", got.ID)
}
}
// With billing disabled every org places as paid. HasPaidSubscription requires a
// Stripe subscription id, so without this a self-host org is free tier forever
// and can only ever be placed onto free-tier workers — while a stock worker
// registers as premium, leaving no candidate and no way to connect a mailbox.
func TestAssign_SelfHost_PlacesEveryOrgAsPaid(t *testing.T) {
t.Setenv("BILLING_PROVIDER", "none")
premiumWorker := newWorker(uuid.New(), false, models.WorkerTypeShared)
wr := &stubWorkerRepo{sharedPremium: []models.Worker{premiumWorker}}
svc := NewAssignmentService(wr, &stubSubRepo{sub: nil}, &stubPlanRepo{})
got, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New())
if err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if *got != premiumWorker.ID {
t.Errorf("self-host org should place on the premium worker, got %s", got)
}
if wr.lastEmailPoolTypeSet != "premium" {
t.Errorf("self-host org should join the premium warmup pool, got %q", wr.lastEmailPoolTypeSet)
}
}
func TestAssign_UpdatesLoadScoreByMailboxWeight(t *testing.T) {
// AssignWorkerToEmail must bump load_score by MailboxWeight. With an
// OAuth provider the bump is 0.05; with cold SMTP it's 1.0; with
// warmup it's 0.4.
t.Setenv("BILLING_PROVIDER", "stripe")
freeWorker := newWorker(uuid.New(), true, models.WorkerTypeShared)
wr := &stubWorkerRepo{
sharedFree: []models.Worker{freeWorker},
placementHint: &repository.EmailAccountPlacementHint{Provider: "gmail-api", IsWarmup: false},
}
svc := NewAssignmentService(wr, &stubSubRepo{sub: nil}, &stubPlanRepo{})
if _, err := svc.AssignWorkerToEmail(context.Background(), uuid.New(), uuid.New()); err != nil {
t.Fatalf("AssignWorkerToEmail: %v", err)
}
if got := wr.loadScoreDeltas[freeWorker.ID]; got != 0.05 {
t.Errorf("load_score should be bumped by 0.05 for gmail-api, got %v", got)
}
}
+22 -19
View File
@@ -24,9 +24,7 @@ import (
// worker_capacity_view. Loaded by the repository; fed into ComputeCapacity.
type WorkerCapacityRow struct {
WorkerID uuid.UUID
WorkerType models.WorkerType
FreeTier bool
EgressKind models.WorkerEgressKind
Region string
HealthState models.WorkerHealthState
LoadScore float64
BaseCapacity float64
@@ -87,29 +85,34 @@ func clampUnit(x float64) float64 {
return x
}
// MailboxWeight is the per-mailbox load contribution. The numbers are
// deliberately on different scales:
// MailboxWeight is the per-mailbox load contribution, in cold-mailbox
// equivalents. It is the reason a worker no longer declares an egress
// category: each mailbox states its own cost, so one base capacity covers a
// worker carrying any mix.
//
// - Cold SMTP mailboxes are the bottleneck (one inbox, one IP-facing
// conversation, a hard ~50/day ceiling per CLAUDE.md sending policy).
// Weight = 1.0 so a cold_smtp worker with Base=16 caps at ~16 cold
// mailboxes.
// - smtp_imap mailboxes hold a real SMTP and IMAP conversation from the
// worker's own address, and Exchange Online caps SMTP AUTH at ~3
// concurrent connections and ~30 msg/min per mailbox. They are the
// bottleneck. Weight = 1.0, so a worker with Base=16 carries ~16 of them.
//
// - OAuth API mailboxes (Gmail API, Microsoft Graph) push through the
// provider's own infrastructure and don't bottleneck on a single
// IMAP/SMTP conversation. Weight = 0.05 so an oauth_api worker with
// Base=400 can carry hundreds of API mailboxes.
// - gmail and outlook mailboxes go through the Google and Microsoft Graph
// APIs. The provider absorbs the connection cost and the per-mailbox
// ceiling is its own quota, not ours. Weight = 0.05.
//
// - Warmup-only assignments are the cheapest because warmup volume is
// small and bursty by design. Weight = 0.4 regardless of provider so
// warmup-only workers don't get crowded out by their own cold-style
// mailbox accounting.
// - Warmup-only assignments are cheapest: warmup volume is small and bursty
// by design. Weight = 0.4 regardless of provider so warmup-only workers
// are not crowded out by their own cold-style accounting.
//
// The provider strings are the email_provider enum values as stored. An
// earlier version of this function switched on "gmail-api" and "graph-api",
// which no caller ever produced, so every non-warmup mailbox silently weighed
// 1.0 and the API-backed ones were over-accounted by 20x.
func MailboxWeight(provider string, isWarmup bool) float64 {
if isWarmup {
return 0.4
}
switch provider {
case "gmail-api", "graph-api":
switch models.InboxProvider(provider) {
case models.InboxProviderGoogle, models.InboxProviderOutlook:
return 0.05
default:
return 1.0
+6 -3
View File
@@ -154,10 +154,13 @@ func TestMailboxWeight_TableDriven(t *testing.T) {
warmup bool
want float64
}{
{"warmup_overrides_provider", "gmail-api", true, 0.4},
// The provider strings are the email_provider enum values as stored.
// The old table asserted "gmail-api"/"graph-api", which nothing ever
// wrote, so the API-mailbox weight never actually applied in production.
{"warmup_overrides_provider", "gmail", true, 0.4},
{"warmup_overrides_smtp_imap", "smtp_imap", true, 0.4},
{"gmail_api_cold", "gmail-api", false, 0.05},
{"graph_api_cold", "graph-api", false, 0.05},
{"gmail_cold", "gmail", false, 0.05},
{"outlook_cold", "outlook", false, 0.05},
{"smtp_imap_cold", "smtp_imap", false, 1.0},
{"empty_provider_cold", "", false, 1.0},
{"unknown_provider_cold", "exchange-rpc", false, 1.0},
+186
View File
@@ -0,0 +1,186 @@
// Placement is the scoring layer that replaced the old category filters.
//
// Workers used to be partitioned four ways (free_tier, worker_type, risk_pool,
// egress_kind) and placement was a filter: find a worker whose labels match
// the mailbox's labels. That model assumed the worker's IP was the sending
// identity, the way it is for a platform that talks to recipient MXs directly.
// Warmbly doesn't. A worker authenticates to the customer's own mailbox
// provider and that provider delivers from its own outbound pool, so:
//
// - the worker IP is invisible to recipient spam filtering (Google strips
// the submitting client's IP; Microsoft dropped X-Originating-IP), which
// means co-locating a spam-prone mailbox next to a healthy one cannot
// contaminate the healthy one's sending reputation
// - the worker IP is very visible to the mailbox provider, where it drives
// sign-in risk challenges, per-IP auth throttles (454 4.7.0) and per-IP
// rate limits (421 4.7.28)
//
// So the levers invert. IP *stability* per mailbox beats IP diversity, because
// moving a mailbox changes the client IP its provider sees and buys a security
// challenge for nothing. Placement stops being a partition problem and becomes
// a scoring problem: spend capacity well, keep mailboxes still, keep one
// client IP from crowding one provider, and keep any single worker from
// carrying too much of one customer.
package worker
import (
"sort"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
// Placement scoring weights. Relative magnitudes are the design: stickiness
// outranks packing, because a needless migration costs provider trust while a
// slightly hotter worker costs nothing a rebalance can't fix later.
const (
weightHeadroom = 1.0
weightIncumbent = 2.0
weightRegion = 0.6
weightIsolation = 1.5
weightBlastRadius = 0.8
weightProviderLoad = 0.7
)
// providerSoftCap is how many mailboxes of ONE provider a single worker is
// expected to carry before the placer starts steering elsewhere. It is a soft
// cap: exceeding it lowers the score, it never refuses a placement. Anchored
// on the provider-side limits rather than anything about our machines -
// Exchange Online allows ~3 concurrent SMTP AUTH connections and ~30 msg/min
// per mailbox, and Gmail rate-limits authentication per client IP, so the
// thing worth spreading is how many accounts of one provider sign in from one
// address.
const providerSoftCap = 24.0
// PlacementCandidate is one worker the placer may choose, with the live facts
// the score reads. Everything here is observed, none of it is configured.
type PlacementCandidate struct {
WorkerID uuid.UUID
Region string
Capacity Capacity
Health models.WorkerHealthState
// TotalMailboxes is every mailbox currently on this worker.
TotalMailboxes int
// OrgMailboxesHere is how many of them belong to the org being placed.
OrgMailboxesHere int
// ProviderMailboxesHere is how many of them use the same mailbox provider
// as the one being placed, from any org. Same client IP, same provider,
// same rate-limit bucket.
ProviderMailboxesHere int
}
// PlacementRequest describes the mailbox that needs a home.
type PlacementRequest struct {
// Weight is the mailbox's load contribution (see MailboxWeight).
Weight float64
// Region the mailbox's provider expects sign-ins from. Empty means no
// preference and scores neutral rather than penalising anything.
Region string
// CurrentWorkerID is the incumbent, when this is a re-placement. The
// incumbent gets a large bonus: staying put is the default.
CurrentWorkerID *uuid.UUID
// OrgMailboxesTotal is how many mailboxes the org owns in total, the
// denominator for the blast-radius term. Zero disables that term.
OrgMailboxesTotal int
// IsolatedEgress is the org entitlement that buys neighbours-free egress.
// It does not pin the org to a machine; it makes foreign tenants expensive
// to score against, so the fleet converges on isolation and self-heals
// when a worker dies instead of stranding the customer.
IsolatedEgress bool
}
// Eligible reports whether a candidate may host the mailbox at all. These are
// the only hard constraints left: the worker has to be able to do the work.
// Everything else is a preference expressed in the score.
func (c PlacementCandidate) Eligible(req PlacementRequest) bool {
switch c.Health {
case models.WorkerHealthHealthy, models.WorkerHealthWatch:
default:
return false
}
return c.Capacity.Effective-c.Capacity.Load >= req.Weight
}
// Score ranks an eligible candidate. Higher is better. The terms are additive
// and each one is traceable to a specific provider behaviour, which is what
// makes the number explainable in the decision log.
func (c PlacementCandidate) Score(req PlacementRequest) float64 {
var score float64
// Capacity: prefer the worker with the most room, so the fleet fills evenly.
utilization := c.Capacity.Utilization
if utilization > 1 {
utilization = 1
}
score += weightHeadroom * (1 - utilization)
// Stickiness: the incumbent wins ties and most non-ties. A mailbox that
// stays put keeps presenting the same client IP to its provider.
if req.CurrentWorkerID != nil && *req.CurrentWorkerID == c.WorkerID {
score += weightIncumbent
}
// Sign-in geography: a mailbox whose provider sees logins from the region
// it expects raises fewer risk challenges. Unknown on either side is
// neutral, never a penalty - most installs set no region at all.
if req.Region != "" && c.Region != "" && req.Region == c.Region {
score += weightRegion
}
// Neighbours: only orgs entitled to isolated egress pay attention to who
// else is on the box, and it is a preference, not a refusal.
if req.IsolatedEgress && c.TotalMailboxes > 0 {
foreign := float64(c.TotalMailboxes-c.OrgMailboxesHere) / float64(c.TotalMailboxes)
score -= weightIsolation * foreign
}
// Blast radius: if this worker dies or its IP gets throttled, how much of
// this customer's sending stops? Spread the org across workers.
if req.OrgMailboxesTotal > 0 {
concentration := float64(c.OrgMailboxesHere) / float64(req.OrgMailboxesTotal)
if concentration > 1 {
concentration = 1
}
score -= weightBlastRadius * concentration
}
// Provider crowding: many accounts of one provider authenticating from one
// address is what earns a per-IP auth throttle. Soft, and it saturates.
crowding := float64(c.ProviderMailboxesHere) / providerSoftCap
if crowding > 1 {
crowding = 1
}
score -= weightProviderLoad * crowding
return score
}
// SelectPlacement picks the best eligible candidate, or nil when none can take
// the mailbox. Deterministic: ties break on worker id so two concurrent
// placements of identical mailboxes agree, and a test can assert an outcome.
func SelectPlacement(candidates []PlacementCandidate, req PlacementRequest) *PlacementCandidate {
type scored struct {
cand PlacementCandidate
value float64
}
eligible := make([]scored, 0, len(candidates))
for _, c := range candidates {
if !c.Eligible(req) {
continue
}
eligible = append(eligible, scored{cand: c, value: c.Score(req)})
}
if len(eligible) == 0 {
return nil
}
sort.Slice(eligible, func(i, j int) bool {
if eligible[i].value != eligible[j].value {
return eligible[i].value > eligible[j].value
}
return eligible[i].cand.WorkerID.String() < eligible[j].cand.WorkerID.String()
})
best := eligible[0].cand
return &best
}
+177
View File
@@ -0,0 +1,177 @@
package worker
import (
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
// candidate builds a healthy candidate with the given capacity numbers.
func candidate(id uuid.UUID, effective, load float64) PlacementCandidate {
c := PlacementCandidate{
WorkerID: id,
Health: models.WorkerHealthHealthy,
Capacity: Capacity{Effective: effective, Load: load},
}
if effective > 0 {
c.Capacity.Utilization = load / effective
}
return c
}
func TestEligibleRequiresHealthAndHeadroom(t *testing.T) {
id := uuid.New()
req := PlacementRequest{Weight: 1.0}
if c := candidate(id, 16, 4); !c.Eligible(req) {
t.Fatal("healthy worker with headroom should be eligible")
}
if c := candidate(id, 16, 15.5); c.Eligible(req) {
t.Fatal("worker with less headroom than the mailbox weight should not be eligible")
}
for _, state := range []models.WorkerHealthState{
models.WorkerHealthThrottled,
models.WorkerHealthQuarantined,
models.WorkerHealthBlocked,
} {
c := candidate(id, 16, 0)
c.Health = state
if c.Eligible(req) {
t.Fatalf("state %s should not accept new mailboxes", state)
}
}
// watch still accepts: it is a warning, not a stop.
c := candidate(id, 16, 0)
c.Health = models.WorkerHealthWatch
if !c.Eligible(req) {
t.Fatal("watch state should still accept placements")
}
}
func TestIncumbentOutranksMarginallyEmptierWorker(t *testing.T) {
incumbent := uuid.New()
empty := uuid.New()
// The incumbent is busier, but staying put avoids a client-IP change.
cands := []PlacementCandidate{
candidate(incumbent, 16, 12),
candidate(empty, 16, 0),
}
got := SelectPlacement(cands, PlacementRequest{Weight: 1.0, CurrentWorkerID: &incumbent})
if got == nil || got.WorkerID != incumbent {
t.Fatalf("expected incumbent to win, got %v", got)
}
}
func TestIncumbentLosesWhenItHasNoHeadroom(t *testing.T) {
incumbent := uuid.New()
other := uuid.New()
cands := []PlacementCandidate{
candidate(incumbent, 16, 16),
candidate(other, 16, 8),
}
got := SelectPlacement(cands, PlacementRequest{Weight: 1.0, CurrentWorkerID: &incumbent})
if got == nil || got.WorkerID != other {
t.Fatal("a full incumbent must not win on stickiness alone")
}
}
func TestBlastRadiusSpreadsAnOrgAcrossWorkers(t *testing.T) {
crowded := uuid.New()
fresh := uuid.New()
// Both equally loaded overall, but the org already has all 10 of its
// mailboxes on `crowded`.
a := candidate(crowded, 16, 8)
a.OrgMailboxesHere = 10
b := candidate(fresh, 16, 8)
got := SelectPlacement([]PlacementCandidate{a, b}, PlacementRequest{
Weight: 1.0,
OrgMailboxesTotal: 10,
})
if got == nil || got.WorkerID != fresh {
t.Fatal("placement should spread an org rather than concentrate it")
}
}
func TestProviderCrowdingSteersAwayFromOneAddress(t *testing.T) {
crowded := uuid.New()
fresh := uuid.New()
a := candidate(crowded, 16, 8)
a.ProviderMailboxesHere = int(providerSoftCap)
b := candidate(fresh, 16, 8)
got := SelectPlacement([]PlacementCandidate{a, b}, PlacementRequest{Weight: 1.0})
if got == nil || got.WorkerID != fresh {
t.Fatal("many mailboxes of one provider on one address should push placement elsewhere")
}
}
func TestIsolatedEgressPrefersAWorkerWithoutForeignTenants(t *testing.T) {
shared := uuid.New()
ours := uuid.New()
a := candidate(shared, 16, 4)
a.TotalMailboxes = 10
a.OrgMailboxesHere = 1
b := candidate(ours, 16, 6)
b.TotalMailboxes = 6
b.OrgMailboxesHere = 6
got := SelectPlacement([]PlacementCandidate{a, b}, PlacementRequest{
Weight: 1.0,
IsolatedEgress: true,
})
if got == nil || got.WorkerID != ours {
t.Fatal("an isolated-egress org should avoid a worker full of other tenants")
}
}
func TestRegionMatchIsAPreferenceNotARequirement(t *testing.T) {
match := uuid.New()
other := uuid.New()
a := candidate(match, 16, 8)
a.Region = "eu-central"
b := candidate(other, 16, 8)
b.Region = "us-east"
got := SelectPlacement([]PlacementCandidate{a, b}, PlacementRequest{Weight: 1.0, Region: "eu-central"})
if got == nil || got.WorkerID != match {
t.Fatal("matching region should win between otherwise equal workers")
}
// With no region on the request, neither is penalised and the fleet still places.
if got := SelectPlacement([]PlacementCandidate{b}, PlacementRequest{Weight: 1.0}); got == nil {
t.Fatal("an unknown region must never make a worker ineligible")
}
}
func TestSelectPlacementReturnsNilWhenNothingFits(t *testing.T) {
full := candidate(uuid.New(), 16, 16)
if got := SelectPlacement([]PlacementCandidate{full}, PlacementRequest{Weight: 1.0}); got != nil {
t.Fatal("expected no placement when every worker is full")
}
if got := SelectPlacement(nil, PlacementRequest{Weight: 1.0}); got != nil {
t.Fatal("expected no placement from an empty fleet")
}
}
func TestSelectPlacementIsDeterministicOnTies(t *testing.T) {
a, b := uuid.New(), uuid.New()
cands := []PlacementCandidate{candidate(a, 16, 8), candidate(b, 16, 8)}
first := SelectPlacement(cands, PlacementRequest{Weight: 1.0})
// Same inputs, reversed order: the winner must not change.
second := SelectPlacement([]PlacementCandidate{cands[1], cands[0]}, PlacementRequest{Weight: 1.0})
if first == nil || second == nil || first.WorkerID != second.WorkerID {
t.Fatal("tie-breaking must not depend on candidate order")
}
}
+159
View File
@@ -0,0 +1,159 @@
// Rotation decides WHEN a mailbox is allowed to change workers. Placement
// (placement.go) decides where it would go; this decides whether it should go
// at all.
//
// The two are separate because the answer is usually no. Every migration
// changes the client IP the mailbox's provider sees, and providers treat a
// moving sign-in location as a risk signal: Google challenges the login,
// Microsoft and Google both throttle authentication per address. A mailbox
// that stays on one worker for months is in the best possible state, so the
// bar for moving one is deliberately high and rises with how little the move
// buys.
package worker
import (
"time"
"github.com/warmbly/warmbly/internal/models"
)
// Rotation thresholds.
const (
// RotationMinResidency is how long a mailbox stays put before an
// opportunistic move (packing, isolation) may touch it.
RotationMinResidency = 72 * time.Hour
// RotationElevatedResidency is the shorter floor that applies when the
// current worker is actively degrading. Still non-zero: a worker that
// dips into throttled and recovers within the hour should not have
// evacuated its mailboxes.
RotationElevatedResidency = 6 * time.Hour
// RotationHotUtilization is the utilization above which a worker is
// considered worth draining for packing reasons alone.
RotationHotUtilization = 0.85
// RotationMinScoreGain is how much better the target must score than the
// incumbent before an opportunistic move is worth its provider-trust
// cost. It sits below weightIncumbent so a genuinely better home can
// still win, but noise cannot.
RotationMinScoreGain = 0.5
)
// RotationUrgency is how badly a mailbox needs to leave its current worker.
type RotationUrgency int
const (
// RotationStay means the mailbox is where it should be.
RotationStay RotationUrgency = iota
// RotationOpportunistic is a move worth making only if a materially
// better home exists and the mailbox has served its full residency.
RotationOpportunistic
// RotationElevated is a degrading worker: move on the shorter residency,
// and take any eligible home rather than holding out for a better score.
RotationElevated
// RotationImmediate is a worker that cannot do the work at all. Residency
// and score gain are both ignored; anywhere eligible beats staying.
RotationImmediate
)
// RotationInput is the live state of one mailbox and the worker under it.
type RotationInput struct {
WorkerActive bool
WorkerLive bool // heartbeating inside the liveness window
WorkerHealth models.WorkerHealthState
WorkerUtilization float64
// Residency is how long the mailbox has been on this worker. A zero
// value (no worker_assigned_at recorded) is treated as "long enough",
// because the column was backfilled at migration time and a NULL there
// means an old assignment, not a fresh one.
Residency time.Duration
// OnSomeoneElsesReservedWorker is true when this mailbox sits on a worker
// another organization has reserved. Those have to leave, or the
// reservation means nothing.
OnSomeoneElsesReservedWorker bool
// AwayFromOwnReservedWorker is true when this mailbox's own organization
// has a reserved worker and the mailbox is not on it.
AwayFromOwnReservedWorker bool
}
// EvaluateRotation returns how urgently the mailbox should move and a reason
// string suitable for the decision log. The reason is always populated when
// the urgency is not RotationStay.
func EvaluateRotation(in RotationInput) (RotationUrgency, string) {
// The worker cannot execute anything: a command queued for it is never
// run and never answered, so this is not a deliverability trade-off.
if !in.WorkerActive {
return RotationImmediate, "worker is inactive"
}
if !in.WorkerLive {
return RotationImmediate, "worker stopped heartbeating"
}
switch in.WorkerHealth {
case models.WorkerHealthBlocked:
return RotationImmediate, "worker is blocked"
case models.WorkerHealthQuarantined:
return RotationImmediate, "worker is quarantined"
case models.WorkerHealthThrottled:
return RotationElevated, "worker is throttled"
}
if in.OnSomeoneElsesReservedWorker {
// Elevated, not opportunistic: this mailbox has to leave for the
// reservation to mean anything, and an opportunistic move would be
// weighed against the incumbent's stickiness bonus and refused every
// tick, so the stranger would never go and the row would re-enter the
// scan budget forever.
return RotationElevated, "worker is reserved for another organization"
}
if in.AwayFromOwnReservedWorker {
return RotationOpportunistic, "organization has a reserved worker elsewhere"
}
if in.WorkerUtilization > RotationHotUtilization {
return RotationOpportunistic, "worker over capacity"
}
return RotationStay, ""
}
// MayMove applies the residency floor for an urgency level. Splitting this out
// keeps the "should it move" question separable from "is it allowed to yet",
// which is the part that stops a flapping worker from evacuating twice.
func MayMove(urgency RotationUrgency, residency time.Duration) bool {
switch urgency {
case RotationImmediate:
return true
case RotationElevated:
return residency == 0 || residency >= RotationElevatedResidency
case RotationOpportunistic:
return residency == 0 || residency >= RotationMinResidency
default:
return false
}
}
// WorthMoving decides whether a chosen target justifies the move.
//
// Urgent moves take anything eligible, because staying is not an option.
// Opportunistic moves have to clear RotationMinScoreGain on top of the
// incumbent's own stickiness bonus, which is what keeps the fleet from
// churning.
//
// A mandated target skips the comparison entirely. It was chosen by an
// entitlement rather than by scoring, and it usually scores lower than the
// incumbent precisely because the incumbent is the incumbent; weighing the two
// would refuse the move on every tick and the mailbox would never arrive.
func WorthMoving(urgency RotationUrgency, incumbentScore, targetScore float64, mandated bool) bool {
switch urgency {
case RotationImmediate, RotationElevated:
return true
case RotationOpportunistic:
return mandated || targetScore-incumbentScore >= RotationMinScoreGain
default:
return false
}
}
+168
View File
@@ -0,0 +1,168 @@
package worker
import (
"testing"
"time"
"github.com/warmbly/warmbly/internal/models"
)
func healthyInput() RotationInput {
return RotationInput{
WorkerActive: true,
WorkerLive: true,
WorkerHealth: models.WorkerHealthHealthy,
Residency: 30 * 24 * time.Hour,
}
}
func TestHealthyWorkerNeverRotates(t *testing.T) {
urgency, reason := EvaluateRotation(healthyInput())
if urgency != RotationStay {
t.Fatalf("a settled mailbox on a healthy worker must stay, got %v (%s)", urgency, reason)
}
}
func TestDeadOrBlockedWorkerIsImmediate(t *testing.T) {
cases := map[string]func(RotationInput) RotationInput{
"inactive": func(in RotationInput) RotationInput { in.WorkerActive = false; return in },
"no heartbeat": func(in RotationInput) RotationInput {
in.WorkerLive = false
return in
},
"blocked": func(in RotationInput) RotationInput {
in.WorkerHealth = models.WorkerHealthBlocked
return in
},
"quarantined": func(in RotationInput) RotationInput {
in.WorkerHealth = models.WorkerHealthQuarantined
return in
},
}
for name, mutate := range cases {
in := mutate(healthyInput())
in.Residency = time.Minute // fresh: residency must not hold it back
urgency, reason := EvaluateRotation(in)
if urgency != RotationImmediate {
t.Fatalf("%s: expected immediate, got %v", name, urgency)
}
if reason == "" {
t.Fatalf("%s: urgent rotations must carry a reason", name)
}
if !MayMove(urgency, in.Residency) {
t.Fatalf("%s: residency must not block an immediate move", name)
}
}
}
func TestThrottledWorkerWaitsOutTheShortResidency(t *testing.T) {
in := healthyInput()
in.WorkerHealth = models.WorkerHealthThrottled
in.Residency = time.Hour
urgency, _ := EvaluateRotation(in)
if urgency != RotationElevated {
t.Fatalf("throttled should be elevated, got %v", urgency)
}
if MayMove(urgency, time.Hour) {
t.Fatal("an elevated move must respect the short residency floor")
}
if !MayMove(urgency, RotationElevatedResidency) {
t.Fatal("an elevated move should be allowed once the short floor passes")
}
}
func TestHotWorkerIsOpportunisticAndNeedsAMateriallyBetterHome(t *testing.T) {
in := healthyInput()
in.WorkerUtilization = RotationHotUtilization + 0.05
urgency, _ := EvaluateRotation(in)
if urgency != RotationOpportunistic {
t.Fatalf("an over-capacity worker should be opportunistic, got %v", urgency)
}
if MayMove(urgency, time.Hour) {
t.Fatal("an opportunistic move must respect the full residency floor")
}
if !MayMove(urgency, RotationMinResidency) {
t.Fatal("an opportunistic move should be allowed after full residency")
}
// A marginal improvement is not worth the provider-trust cost.
if WorthMoving(urgency, 1.0, 1.0+RotationMinScoreGain/2, false) {
t.Fatal("a marginal score gain must not justify a migration")
}
if !WorthMoving(urgency, 1.0, 1.0+RotationMinScoreGain, false) {
t.Fatal("a material score gain should justify a migration")
}
}
func TestUrgentMovesTakeAnythingEligible(t *testing.T) {
// Staying is not an option, so a worse-scoring destination still wins.
if !WorthMoving(RotationImmediate, 5.0, 0.1, false) {
t.Fatal("an immediate move must accept any eligible destination")
}
if !WorthMoving(RotationElevated, 5.0, 0.1, false) {
t.Fatal("an elevated move must accept any eligible destination")
}
}
func TestReservedWorkerDriftRotatesBothWays(t *testing.T) {
// A stranger has to leave for the reservation to mean anything, and the
// destination does not have to be better than where it is. Weighing it
// against the incumbent's stickiness bonus would refuse the move forever.
stranger := healthyInput()
stranger.OnSomeoneElsesReservedWorker = true
urgency, _ := EvaluateRotation(stranger)
if urgency != RotationElevated {
t.Fatalf("a mailbox on someone else's reserved worker must be evicted, got %v", urgency)
}
if !WorthMoving(urgency, 5.0, 0.1, false) {
t.Fatal("evicting a stranger must accept any eligible destination")
}
// Pulling the owner back is opportunistic, but the placer marks the
// reserved worker as mandated so the score comparison does not block it.
owner := healthyInput()
owner.AwayFromOwnReservedWorker = true
urgency, _ = EvaluateRotation(owner)
if urgency != RotationOpportunistic {
t.Fatalf("a mailbox away from its own reserved worker should be pulled back, got %v", urgency)
}
if !WorthMoving(urgency, 5.0, 0.1, true) {
t.Fatal("the reserved worker is mandated, so the move must not be score-gated")
}
}
func TestUnknownResidencyIsTreatedAsSettled(t *testing.T) {
// worker_assigned_at was backfilled at migration time, so a zero value
// means an old assignment, not a brand new one. Reading it as "brand new"
// would freeze every pre-migration mailbox in place.
if !MayMove(RotationOpportunistic, 0) {
t.Fatal("unknown residency must not block an opportunistic move")
}
}
// A reserved worker usually scores lower than the incumbent, because the
// incumbent carries the stickiness bonus. Weighing the two would refuse the
// move on every tick and an isolated-egress organization would never arrive on
// the worker it is paying for.
func TestMandatedTargetIgnoresTheScoreComparison(t *testing.T) {
if WorthMoving(RotationOpportunistic, 5.0, 0.1, false) {
t.Fatal("an ordinary opportunistic move must not accept a worse target")
}
if !WorthMoving(RotationOpportunistic, 5.0, 0.1, true) {
t.Fatal("an entitlement-chosen target must move regardless of score")
}
if WorthMoving(RotationStay, 0, 100, true) {
t.Fatal("mandated must not override RotationStay")
}
}
func TestStayNeverMoves(t *testing.T) {
if MayMove(RotationStay, RotationMinResidency*10) {
t.Fatal("RotationStay must never permit a move")
}
if WorthMoving(RotationStay, 0, 100, false) {
t.Fatal("RotationStay must never be worth moving")
}
}
@@ -1,54 +0,0 @@
package worker_orchestrator
import (
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"golang.org/x/crypto/ssh"
)
// GenerateKeypair produces an ed25519 SSH keypair.
// Returns:
// - publicKey in OpenSSH authorized_keys format ("ssh-ed25519 AAAA... warmbly")
// - privateKey in OpenSSH PEM format (PEM-encoded openssh-key-v1)
func GenerateKeypair() (publicKey string, privateKeyPEM string, err error) {
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return "", "", fmt.Errorf("ed25519: %w", err)
}
sshPub, err := ssh.NewPublicKey(pub)
if err != nil {
return "", "", fmt.Errorf("ssh.NewPublicKey: %w", err)
}
authorizedKey := ssh.MarshalAuthorizedKey(sshPub)
publicKey = string(authorizedKey[:len(authorizedKey)-1]) + " warmbly-worker"
pemBlock, err := ssh.MarshalPrivateKey(priv, "")
if err != nil {
return "", "", fmt.Errorf("ssh.MarshalPrivateKey: %w", err)
}
privateKeyPEM = string(pem.EncodeToMemory(pemBlock))
return publicKey, privateKeyPEM, nil
}
// FingerprintSHA256 returns the SHA256 fingerprint of a host key, formatted
// the way OpenSSH displays it: "SHA256:<base64-no-padding>".
func FingerprintSHA256(key ssh.PublicKey) string {
sum := sha256.Sum256(key.Marshal())
return "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:])
}
// ParsePrivateKey decodes an OpenSSH PEM private key into an ssh.Signer.
func ParsePrivateKey(pemBytes []byte) (ssh.Signer, error) {
if len(pemBytes) == 0 {
return nil, errors.New("empty private key")
}
return ssh.ParsePrivateKey(pemBytes)
}
@@ -1,690 +0,0 @@
// Package worker_orchestrator drives the lifecycle of remote worker VPSes
// over SSH.
//
// The control plane stores per-worker connection info and an ed25519 private
// key (encrypted via the platform cipher service) in Postgres. Operations
// (Install, Restart, Update, Uninstall, Status, TailLogs, RotateKeys,
// TestConnection) open a fresh SSH session, run the relevant command, and
// update the workers row.
//
// The install payload is the project's scripts/install-worker.sh — uploaded
// to /tmp on the target and executed with the worker's env config baked in.
//
// Identity model: the worker's UUID is the workers.id row. The VPS's
// hostname is set to that UUID via `docker run --hostname` inside the
// installer, so cmd/worker reads its identity from os.Hostname().
package worker_orchestrator
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/cipher"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
"golang.org/x/crypto/ssh"
)
// platformCipherID is the UUID under which the cipher service stores the
// DEK that encrypts platform-level secrets (worker SSH keys). The zero UUID
// is not used by any real organization, so it cleanly partitions platform
// secrets from organization secrets in the encrypted-keys store.
var platformCipherID = uuid.Nil
// WorkerEnvConfig is the set of env vars the worker container needs at run
// time. The orchestrator writes these into /etc/warmbly/worker.env on the
// target. Provide credentials that scope only what the worker can reach.
type WorkerEnvConfig struct {
AppEnv string // "dev" or "prod"
WorkerImage string // e.g. ghcr.io/warmbly/worker:latest
KafkaBootstrap string
KafkaSASLUsername string
KafkaSASLPassword string
SchemaRegistryURL string
SchemaRegistryKey string
SchemaRegistrySecret string
RedisURL string
AWSRegion string
AWSAccessKeyID string
AWSSecretAccessKey string
EncryptedKeysBackendURL string
EncryptedKeysWorkerToken string
// Decryption material. A worker fetches the org's encrypted data key over
// the internal API and opens it locally, then opens mailbox credentials
// with the credentials key, so without these it can authenticate to the
// backend and still be unable to send a single message.
KMSProvider string
KMSLocalMasterKey string
KMSAWSKeyID string
CredentialsEncryptionKey string
// Blob storage. A worker binary defaults this to s3, so an unset value on a
// no-cloud deployment makes the worker look for AWS at boot.
BlobProvider string
BlobBucket string
BlobFSRoot string
EventBusProvider string
NATSURL string
CodecProvider string
// Provider OAuth client credentials the worker needs to refresh delegated
// mailbox tokens locally (Cfg is not shipped in the AddWorkerEmail payload).
BoxGoogleClientID string
BoxGoogleClientSecret string
BoxOutlookClientID string
BoxOutlookClientSecret string
}
type Orchestrator struct {
repo repository.WorkerRepository
credentialsRepo repository.CredentialsRepository
cipher cipher.CipherService
defaultEnv WorkerEnvConfig // fallback when worker has no profile assigned
installerPath string // absolute path to scripts/install-worker.sh
}
func New(
repo repository.WorkerRepository,
credentialsRepo repository.CredentialsRepository,
cipherSvc cipher.CipherService,
defaultEnv WorkerEnvConfig,
installerPath string,
) *Orchestrator {
return &Orchestrator{
repo: repo,
credentialsRepo: credentialsRepo,
cipher: cipherSvc,
defaultEnv: defaultEnv,
installerPath: installerPath,
}
}
// EncryptSecret is a small public helper for handlers writing new platform
// secrets (AWS keys, Kafka passwords, etc.) into the credentials tables.
// Uses the same platform-identity DEK as worker SSH keys.
func (o *Orchestrator) EncryptSecret(ctx context.Context, plaintext string) (string, error) {
if plaintext == "" {
return "", nil
}
return o.encryptPrivateKey(ctx, plaintext)
}
// public ops
// TestConnection opens an SSH session and runs `true`. Useful for the
// dashboard "test" button after the admin pastes the pubkey into the VPS.
func (o *Orchestrator) TestConnection(ctx context.Context, workerID uuid.UUID) error {
return o.withSession(ctx, workerID, func(d *dialer) error {
_, err := d.Run(ctx, "true")
return err
})
}
// Install uploads the installer + env file, then runs the installer to bring
// the worker container up.
func (o *Orchestrator) Install(ctx context.Context, workerID uuid.UUID) error {
_ = o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateProvisioning, "")
err := o.withSession(ctx, workerID, func(d *dialer) error {
envContent, image, err := o.renderEnvFile(ctx, workerID)
if err != nil {
return err
}
if err := d.Upload(ctx, "/tmp/warmbly-worker.env", envContent, "0600"); err != nil {
return fmt.Errorf("upload env: %w", err)
}
installerBytes, err := os.ReadFile(o.installerPath)
if err != nil {
return fmt.Errorf("read installer: %w", err)
}
if err := d.Upload(ctx, "/tmp/install-worker.sh", string(installerBytes), "0755"); err != nil {
return fmt.Errorf("upload installer: %w", err)
}
cmd := fmt.Sprintf(
"sudo /tmp/install-worker.sh --non-interactive --worker-id %s --image %s --env-file /tmp/warmbly-worker.env",
workerID.String(), shellQuote(image),
)
out, err := d.Run(ctx, cmd)
if err != nil {
return fmt.Errorf("installer failed: %w\n%s", err, tail(out, 40))
}
return nil
})
if err != nil {
_ = o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateError, err.Error())
return err
}
_ = o.repo.MarkConfigApplied(ctx, workerID, time.Now())
return o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateInstalled, "")
}
func (o *Orchestrator) InstallerScript() ([]byte, error) {
return os.ReadFile(o.installerPath)
}
// RenderEnrollmentEnv returns a complete dotenv payload for the one-command
// enrollment flow. The token exchange authenticates the caller; this method
// only renders the config the installer writes to disk.
func (o *Orchestrator) RenderEnrollmentEnv(ctx context.Context, workerID uuid.UUID) (string, string, error) {
envContent, image, err := o.renderEnvFile(ctx, workerID)
if err != nil {
return "", "", err
}
var b strings.Builder
b.WriteString("# Warmbly worker enrollment config\n")
b.WriteString("WORKER_ID=")
b.WriteString(workerID.String())
b.WriteString("\n")
b.WriteString("WARMBLY_WORKER_IMAGE=")
b.WriteString(image)
b.WriteString("\n")
b.WriteString(envContent)
return b.String(), image, nil
}
// ApplyConfig re-writes /etc/warmbly/worker.env from the worker's current
// profile + AWS creds and restarts the service. Cheaper than Install — does
// not touch Docker or the installer script. Use after a credential change.
func (o *Orchestrator) ApplyConfig(ctx context.Context, workerID uuid.UUID) error {
err := o.withSession(ctx, workerID, func(d *dialer) error {
envContent, _, err := o.renderEnvFile(ctx, workerID)
if err != nil {
return err
}
// install-worker.sh writes to /etc/warmbly/worker.env at install time;
// we replace that file directly here so it's atomic relative to a
// restart.
if err := d.Upload(ctx, "/etc/warmbly/worker.env", envContent, "0600"); err != nil {
return fmt.Errorf("upload env: %w", err)
}
out, err := d.Run(ctx, "sudo systemctl restart warmbly-worker.service")
if err != nil {
return fmt.Errorf("restart: %w: %s", err, tail(out, 20))
}
return nil
})
if err != nil {
return err
}
return o.repo.MarkConfigApplied(ctx, workerID, time.Now())
}
func (o *Orchestrator) Restart(ctx context.Context, workerID uuid.UUID) error {
return o.withSession(ctx, workerID, func(d *dialer) error {
out, err := d.Run(ctx, "sudo systemctl restart warmbly-worker.service")
if err != nil {
return fmt.Errorf("restart: %w: %s", err, tail(out, 20))
}
return nil
})
}
// Update pulls a new image and restarts the worker. Resolves the target image
// from the worker's profile (if one is assigned); otherwise from the
// orchestrator's default. The installer rewrites the systemd unit with the
// new image so subsequent restarts pick it up.
func (o *Orchestrator) Update(ctx context.Context, workerID uuid.UUID) error {
_, image, err := o.renderEnvFile(ctx, workerID)
if err != nil {
return err
}
return o.UpdateToImage(ctx, workerID, image)
}
// UpdateToImage rolls a worker to a specific image tag. Used by the release
// service when an auto-update channel resolves a new tag.
func (o *Orchestrator) UpdateToImage(ctx context.Context, workerID uuid.UUID, image string) error {
if image == "" {
return errors.New("no image specified")
}
err := o.withSession(ctx, workerID, func(d *dialer) error {
// Make sure the installer is present — it lands in /tmp during Install
// but a backend redeploy can blow it away if /tmp is cleaned.
installerBytes, ierr := os.ReadFile(o.installerPath)
if ierr != nil {
return fmt.Errorf("read installer: %w", ierr)
}
if err := d.Upload(ctx, "/tmp/install-worker.sh", string(installerBytes), "0755"); err != nil {
return fmt.Errorf("upload installer: %w", err)
}
cmd := fmt.Sprintf("sudo /tmp/install-worker.sh --update --image %s", shellQuote(image))
out, err := d.Run(ctx, cmd)
if err != nil {
return fmt.Errorf("update: %w: %s", err, tail(out, 30))
}
return nil
})
if err != nil {
return err
}
tag := imageTag(image)
_ = o.repo.MarkImageVersion(ctx, workerID, tag)
return nil
}
// imageTag extracts the human-readable tag part of an image reference. Used
// for the workers.image_version column.
//
// ghcr.io/foo/worker:v1.2.3 → "v1.2.3"
// ghcr.io/foo/worker → "latest"
// ghcr.io/foo/worker@sha256:abc → "abc[:12]"
func imageTag(image string) string {
if i := strings.LastIndex(image, "@sha256:"); i >= 0 {
d := image[i+len("@sha256:"):]
if len(d) > 12 {
d = d[:12]
}
return "sha256:" + d
}
if i := strings.LastIndex(image, ":"); i >= 0 && !strings.Contains(image[i:], "/") {
return image[i+1:]
}
return "latest"
}
func (o *Orchestrator) Uninstall(ctx context.Context, workerID uuid.UUID) error {
_ = o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateUninstalling, "")
err := o.withSession(ctx, workerID, func(d *dialer) error {
out, err := d.Run(ctx, "sudo /tmp/install-worker.sh --uninstall || true")
_ = out
return err
})
if err != nil {
_ = o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateError, err.Error())
return err
}
return o.repo.UpdateInstallState(ctx, workerID, models.WorkerInstallStateUninstalled, "")
}
// Status snapshot from the target. Cheap, on-demand.
type StatusResult struct {
ServiceActive bool `json:"service_active"`
ContainerUp bool `json:"container_up"`
ContainerImage string `json:"container_image"`
Uptime string `json:"uptime"`
Raw string `json:"raw"`
}
func (o *Orchestrator) Status(ctx context.Context, workerID uuid.UUID) (*StatusResult, error) {
var result *StatusResult
err := o.withSession(ctx, workerID, func(d *dialer) error {
out, _ := d.Run(ctx,
"systemctl is-active warmbly-worker.service 2>/dev/null; "+
"echo '---'; "+
"docker inspect --format='{{.State.Status}} {{.Config.Image}} {{.State.StartedAt}}' warmbly-worker 2>/dev/null; "+
"echo '---'; "+
"uptime",
)
result = parseStatus(out)
return nil
})
return result, err
}
// SystemUpdate runs OS package upgrades on the worker VPS. Detects the
// distro and dispatches to apt / dnf / yum / pacman / apk. Returns the
// combined output (often long) so the dashboard can render it verbatim.
//
// Also checks whether a reboot is required afterward so the admin can be
// prompted; reboots are never automatic.
type SystemUpdateResult struct {
Output string `json:"output"`
RebootRequired bool `json:"reboot_required"`
}
func (o *Orchestrator) SystemUpdate(ctx context.Context, workerID uuid.UUID) (*SystemUpdateResult, error) {
var result *SystemUpdateResult
err := o.withSession(ctx, workerID, func(d *dialer) error {
// One big shell script: detect package manager, run upgrade, then
// check for reboot-required markers.
script := `set -e
detect() {
if command -v apt-get >/dev/null 2>&1; then echo apt
elif command -v dnf >/dev/null 2>&1; then echo dnf
elif command -v yum >/dev/null 2>&1; then echo yum
elif command -v pacman >/dev/null 2>&1; then echo pacman
elif command -v apk >/dev/null 2>&1; then echo apk
else echo unknown; fi
}
PM="$(detect)"
echo "== package manager: $PM"
case "$PM" in
apt)
sudo DEBIAN_FRONTEND=noninteractive apt-get update -y
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -o Dpkg::Options::="--force-confold"
sudo DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
;;
dnf|yum)
sudo "$PM" upgrade -y
sudo "$PM" autoremove -y || true
;;
pacman)
sudo pacman -Syu --noconfirm
;;
apk)
sudo apk update
sudo apk upgrade
;;
*)
echo "unsupported package manager"; exit 1
;;
esac
echo "== reboot check"
REBOOT=0
[ -f /var/run/reboot-required ] && REBOOT=1
command -v needs-restarting >/dev/null 2>&1 && needs-restarting -r >/dev/null 2>&1 || true
# kernel mismatch heuristic for everyone else
RUN_KERNEL="$(uname -r)"
NEW_KERNEL="$(ls -1 /lib/modules 2>/dev/null | sort -V | tail -1 || echo "$RUN_KERNEL")"
[ "$RUN_KERNEL" != "$NEW_KERNEL" ] && REBOOT=1
echo "==REBOOT_REQUIRED:$REBOOT"
`
out, err := d.Run(ctx, script)
result = &SystemUpdateResult{
Output: out,
RebootRequired: strings.Contains(out, "==REBOOT_REQUIRED:1"),
}
return err
})
return result, err
}
// RebootWorker requests an OS reboot. Worker comes back online once the VPS
// restarts and the systemd unit auto-starts.
func (o *Orchestrator) RebootWorker(ctx context.Context, workerID uuid.UUID) error {
return o.withSession(ctx, workerID, func(d *dialer) error {
// nohup + delay so the SSH session can close cleanly before reboot
_, _ = d.Run(ctx, "sudo sh -c 'nohup shutdown -r +1 >/dev/null 2>&1 &'")
return nil
})
}
func (o *Orchestrator) TailLogs(ctx context.Context, workerID uuid.UUID, lines int) (string, error) {
if lines <= 0 || lines > 1000 {
lines = 200
}
var logs string
err := o.withSession(ctx, workerID, func(d *dialer) error {
out, err := d.Run(ctx, fmt.Sprintf("sudo journalctl -u warmbly-worker -n %d --no-pager", lines))
logs = out
return err
})
return logs, err
}
// RotateKeys generates a new keypair, installs it on the target via the old
// connection, then persists the new keypair. The old key remains in
// authorized_keys until the admin removes it manually — we never auto-delete
// keys we didn't put there.
func (o *Orchestrator) RotateKeys(ctx context.Context, workerID uuid.UUID) (newPublicKey string, err error) {
newPub, newPriv, err := GenerateKeypair()
if err != nil {
return "", err
}
encPriv, err := o.encryptPrivateKey(ctx, newPriv)
if err != nil {
return "", err
}
err = o.withSession(ctx, workerID, func(d *dialer) error {
// Append the new pubkey to ~/.ssh/authorized_keys idempotently.
shellCmd := fmt.Sprintf(
"mkdir -p ~/.ssh && chmod 700 ~/.ssh && "+
"touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys && "+
"grep -qxF %s ~/.ssh/authorized_keys || echo %s >> ~/.ssh/authorized_keys",
shellQuote(strings.TrimSpace(newPub)), shellQuote(strings.TrimSpace(newPub)),
)
_, err := d.Run(ctx, shellCmd)
return err
})
if err != nil {
return "", err
}
if err := o.repo.RotateSSHKey(ctx, workerID, newPub, encPriv); err != nil {
return "", err
}
return newPub, nil
}
// helpers
// withSession decrypts the worker's private key, dials, runs `fn`, pins the
// host fingerprint on first connect.
func (o *Orchestrator) withSession(ctx context.Context, workerID uuid.UUID, fn func(*dialer) error) error {
creds, err := o.repo.GetWorkerSSHCredentials(ctx, workerID)
if err != nil {
return err
}
if creds == nil {
return errors.New("worker not found")
}
if creds.SSHHost == "" {
return errors.New("worker has no SSH host configured")
}
signer, err := o.loadSigner(ctx, creds.SSHPrivateKeyEncrypted)
if err != nil {
return fmt.Errorf("load private key: %w", err)
}
d, err := dial(ctx, dialOptions{
host: creds.SSHHost,
port: creds.SSHPort,
user: creds.SSHUser,
signer: signer,
expectedFingerprint: creds.SSHHostFingerprint,
timeout: 20 * time.Second,
})
if err != nil {
return err
}
defer d.Close()
if creds.SSHHostFingerprint == "" && d.observedFingerprint != "" {
_ = o.repo.UpdateHostFingerprint(ctx, workerID, d.observedFingerprint)
}
return fn(d)
}
func (o *Orchestrator) loadSigner(ctx context.Context, encryptedPrivateKey string) (ssh.Signer, error) {
if encryptedPrivateKey == "" {
return nil, errors.New("no private key stored")
}
c, err := o.cipher.Cipher(ctx, platformCipherID)
if err != nil {
return nil, fmt.Errorf("cipher: %w", err)
}
pem, err := c.Decrypt(ctx, encryptedPrivateKey)
if err != nil {
return nil, fmt.Errorf("decrypt: %w", err)
}
signer, err := ParsePrivateKey([]byte(pem))
if err != nil {
return nil, fmt.Errorf("parse private key: %w", err)
}
return signer, nil
}
func (o *Orchestrator) encryptPrivateKey(ctx context.Context, pem string) (string, error) {
c, err := o.cipher.Cipher(ctx, platformCipherID)
if err != nil {
return "", err
}
return c.Encrypt(ctx, pem)
}
// EncryptPrivateKey is the entry point used by the admin service when first
// creating a worker (before any orchestrator op has run).
func (o *Orchestrator) EncryptPrivateKey(ctx context.Context, pem string) (string, error) {
return o.encryptPrivateKey(ctx, pem)
}
// renderEnvFile produces the contents of /etc/warmbly/worker.env for this
// worker. Resolution order:
// 1. If the worker has a profile assigned, fetch the profile + linked AWS
// credentials, decrypt each secret with the cipher service, and use
// those values.
// 2. Otherwise fall back to the orchestrator's defaultEnv (the backend's
// own process env). This lets dev/sim work without setting up profiles.
//
// Also returns the image to run, since profiles can pin their own.
func (o *Orchestrator) renderEnvFile(ctx context.Context, workerID uuid.UUID) (envContent string, image string, err error) {
w, err := o.repo.GetWorkerDetail(ctx, workerID)
if err != nil {
return "", "", err
}
if w == nil {
return "", "", errors.New("worker not found")
}
env := o.defaultEnv
image = o.defaultEnv.WorkerImage
if w.ProfileID != nil {
pe, err := o.credentialsRepo.GetProfileEncrypted(ctx, *w.ProfileID)
if err != nil {
return "", "", fmt.Errorf("load profile: %w", err)
}
if pe == nil {
return "", "", fmt.Errorf("profile %s not found", w.ProfileID)
}
c, err := o.cipher.Cipher(ctx, platformCipherID)
if err != nil {
return "", "", fmt.Errorf("cipher: %w", err)
}
decrypt := func(s string) (string, error) {
if s == "" {
return "", nil
}
return c.Decrypt(ctx, s)
}
env.AppEnv = pe.Profile.AppEnv
env.WorkerImage = pe.Profile.WorkerImage
image = pe.Profile.WorkerImage
env.KafkaBootstrap = pe.Profile.KafkaBootstrapServers
env.KafkaSASLUsername = pe.Profile.KafkaSASLUsername
if env.KafkaSASLPassword, err = decrypt(pe.KafkaSASLPasswordEncrypted); err != nil {
return "", "", fmt.Errorf("decrypt kafka pw: %w", err)
}
env.SchemaRegistryURL = pe.Profile.SchemaRegistryURL
env.SchemaRegistryKey = pe.Profile.SchemaRegistryKey
if env.SchemaRegistrySecret, err = decrypt(pe.SchemaRegistrySecretEncrypted); err != nil {
return "", "", fmt.Errorf("decrypt schema secret: %w", err)
}
if env.RedisURL, err = decrypt(pe.RedisURLEncrypted); err != nil {
return "", "", fmt.Errorf("decrypt redis url: %w", err)
}
// AWS credentials live in their own row.
if pe.Profile.AWSCredentialID != nil {
aws, err := o.credentialsRepo.GetAWSCreds(ctx, *pe.Profile.AWSCredentialID)
if err != nil {
return "", "", fmt.Errorf("load aws creds: %w", err)
}
if aws != nil {
env.AWSRegion = aws.Region
env.AWSAccessKeyID = aws.AccessKeyID
if env.AWSSecretAccessKey, err = decrypt(aws.SecretAccessKeyEncrypted); err != nil {
return "", "", fmt.Errorf("decrypt aws secret: %w", err)
}
}
}
}
if image == "" {
image = "ghcr.io/warmbly/worker:latest"
}
var b strings.Builder
write := func(k, v string) {
if v == "" {
return
}
b.WriteString(k)
b.WriteString("=")
b.WriteString(v)
b.WriteString("\n")
}
write("APP_ENV", env.AppEnv)
write("AWS_CONFIG_ENABLED", "false")
write("AWS_REGION", env.AWSRegion)
write("AWS_ACCESS_KEY_ID", env.AWSAccessKeyID)
write("AWS_SECRET_ACCESS_KEY", env.AWSSecretAccessKey)
write("ENCRYPTED_KEYS_PROVIDER", "http")
write("ENCRYPTED_KEYS_BACKEND_URL", env.EncryptedKeysBackendURL)
write("ENCRYPTED_KEYS_WORKER_TOKEN", env.EncryptedKeysWorkerToken)
// A bare worker binary defaults KMS_PROVIDER to "aws", so this must be
// written explicitly or a self-host worker exits at boot looking for AWS.
write("KMS_PROVIDER", env.KMSProvider)
write("KMS_LOCAL_MASTER_KEY", env.KMSLocalMasterKey)
write("KMS_AWS_KEY_ID", env.KMSAWSKeyID)
write("CREDENTIALS_ENCRYPTION_KEY", env.CredentialsEncryptionKey)
write("BLOB_PROVIDER", env.BlobProvider)
write("BLOB_BUCKET", env.BlobBucket)
write("BLOB_FS_ROOT", env.BlobFSRoot)
write("KAFKA_BOOTSTRAP_SERVERS", env.KafkaBootstrap)
write("KAFKA_SASL_USERNAME", env.KafkaSASLUsername)
write("KAFKA_SASL_PASSWORD", env.KafkaSASLPassword)
write("SCHEMA_REGISTRY_URL", env.SchemaRegistryURL)
write("SCHEMA_REGISTRY_KEY", env.SchemaRegistryKey)
write("SCHEMA_REGISTRY_SECRET", env.SchemaRegistrySecret)
write("REDIS", env.RedisURL)
write("EVENTBUS_PROVIDER", env.EventBusProvider)
write("NATS_URL", env.NATSURL)
write("CODEC_PROVIDER", env.CodecProvider)
write("BOX_GOOGLE_CLIENT_ID", env.BoxGoogleClientID)
write("BOX_GOOGLE_CLIENT_SECRET", env.BoxGoogleClientSecret)
write("BOX_OUTLOOK_CLIENT_ID", env.BoxOutlookClientID)
write("BOX_OUTLOOK_CLIENT_SECRET", env.BoxOutlookClientSecret)
return b.String(), image, nil
}
func parseStatus(raw string) *StatusResult {
r := &StatusResult{Raw: raw}
parts := strings.Split(raw, "---")
if len(parts) >= 1 {
r.ServiceActive = strings.TrimSpace(parts[0]) == "active"
}
if len(parts) >= 2 {
fields := strings.Fields(parts[1])
if len(fields) >= 1 {
r.ContainerUp = fields[0] == "running"
}
if len(fields) >= 2 {
r.ContainerImage = fields[1]
}
}
if len(parts) >= 3 {
r.Uptime = strings.TrimSpace(parts[2])
}
return r
}
func tail(s string, lines int) string {
parts := strings.Split(strings.TrimRight(s, "\n"), "\n")
if len(parts) <= lines {
return s
}
return strings.Join(parts[len(parts)-lines:], "\n")
}
@@ -1,180 +0,0 @@
package worker_orchestrator
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"strconv"
"strings"
"time"
"golang.org/x/crypto/ssh"
)
// dialer wraps an *ssh.Client with helpers tailored for what the orchestrator
// actually needs: run commands, upload small files, with TOFU host-key pinning.
type dialer struct {
client *ssh.Client
// fingerprint observed during this connect. Caller stores it back into the
// workers row on first connect for future verification.
observedFingerprint string
}
type dialOptions struct {
host string
port int
user string
signer ssh.Signer
password string // optional; used only when signer is nil
expectedFingerprint string // empty on first connect (TOFU)
timeout time.Duration
}
func dial(ctx context.Context, opts dialOptions) (*dialer, error) {
if opts.port == 0 {
opts.port = 22
}
if opts.user == "" {
opts.user = "root"
}
if opts.timeout == 0 {
opts.timeout = 15 * time.Second
}
var auths []ssh.AuthMethod
if opts.signer != nil {
auths = append(auths, ssh.PublicKeys(opts.signer))
}
if opts.password != "" {
auths = append(auths, ssh.Password(opts.password))
}
if len(auths) == 0 {
return nil, errors.New("no SSH auth method configured")
}
d := &dialer{}
hostKeyCallback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
fp := FingerprintSHA256(key)
d.observedFingerprint = fp
if opts.expectedFingerprint == "" {
return nil // TOFU: accept and pin
}
if fp != opts.expectedFingerprint {
return fmt.Errorf("host key mismatch: pinned %s, got %s", opts.expectedFingerprint, fp)
}
return nil
}
cfg := &ssh.ClientConfig{
User: opts.user,
Auth: auths,
HostKeyCallback: hostKeyCallback,
Timeout: opts.timeout,
}
addr := net.JoinHostPort(opts.host, strconv.Itoa(opts.port))
// Honour context cancellation while net dial would otherwise block.
type dialResult struct {
conn net.Conn
err error
}
resCh := make(chan dialResult, 1)
go func() {
c, err := (&net.Dialer{Timeout: opts.timeout}).DialContext(ctx, "tcp", addr)
resCh <- dialResult{c, err}
}()
var rawConn net.Conn
select {
case <-ctx.Done():
return nil, ctx.Err()
case r := <-resCh:
if r.err != nil {
return nil, fmt.Errorf("dial %s: %w", addr, r.err)
}
rawConn = r.conn
}
c, chans, reqs, err := ssh.NewClientConn(rawConn, addr, cfg)
if err != nil {
_ = rawConn.Close()
return nil, fmt.Errorf("ssh handshake: %w", err)
}
d.client = ssh.NewClient(c, chans, reqs)
return d, nil
}
func (d *dialer) Close() error {
if d.client == nil {
return nil
}
return d.client.Close()
}
// Run executes a command, returning combined stdout/stderr. Stderr is appended
// to stdout because most of what we run is `set -e` shell that interleaves.
func (d *dialer) Run(ctx context.Context, cmd string) (string, error) {
sess, err := d.client.NewSession()
if err != nil {
return "", fmt.Errorf("new session: %w", err)
}
defer sess.Close()
var out bytes.Buffer
sess.Stdout = &out
sess.Stderr = &out
done := make(chan error, 1)
go func() { done <- sess.Run(cmd) }()
select {
case <-ctx.Done():
_ = sess.Signal(ssh.SIGINT)
return out.String(), ctx.Err()
case err := <-done:
return out.String(), err
}
}
// Upload writes content to a remote path via stdin to `tee`. Avoids requiring
// an scp/sftp subsystem on the target.
func (d *dialer) Upload(ctx context.Context, remotePath, content string, mode string) error {
sess, err := d.client.NewSession()
if err != nil {
return fmt.Errorf("new session: %w", err)
}
defer sess.Close()
stdin, err := sess.StdinPipe()
if err != nil {
return err
}
cmd := fmt.Sprintf("install -D -m %s /dev/stdin %s", mode, shellQuote(remotePath))
if err := sess.Start(cmd); err != nil {
return fmt.Errorf("start tee: %w", err)
}
if _, err := io.WriteString(stdin, content); err != nil {
return err
}
if err := stdin.Close(); err != nil {
return err
}
done := make(chan error, 1)
go func() { done <- sess.Wait() }()
select {
case <-ctx.Done():
_ = sess.Signal(ssh.SIGINT)
return ctx.Err()
case err := <-done:
return err
}
}
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
@@ -1,132 +0,0 @@
// Package cloudprovider abstracts over cloud-VPS APIs so the provisioning
// state machine can target multiple providers without baking Hetzner-specific
// types into the orchestration layer.
//
// One implementation today (Hetzner). The interface is intentionally small;
// adding OVH or Vultr later means implementing six methods.
package cloudprovider
import "context"
// Provider is the surface the provisioning state machine talks to.
type Provider interface {
Name() string
// Catalog — what's available to provision against. Used by the admin
// dropdowns when an operator is composing a template.
Locations(ctx context.Context) ([]Location, error)
ServerTypes(ctx context.Context) ([]ServerType, error)
Images(ctx context.Context) ([]Image, error)
// Auth check, called from the admin "Test connection" button.
Verify(ctx context.Context) error
// Provisioning. Each returns the provider-native ID + IPv4 so the state
// machine can record it for later cleanup.
CreateServer(ctx context.Context, req CreateServerRequest) (*Server, error)
DeleteServer(ctx context.Context, serverID string) error
// Primary IP lifecycle. ipv4_per_server=1 in a template means "use the
// IP that came with the server" — these calls are only made for extras.
CreatePrimaryIP(ctx context.Context, req CreatePrimaryIPRequest) (*PrimaryIP, error)
AssignPrimaryIP(ctx context.Context, ipID, serverID string) error
UnassignPrimaryIP(ctx context.Context, ipID string) error
DeletePrimaryIP(ctx context.Context, ipID string) error
SetReverseDNS(ctx context.Context, ipID, hostname string) error
}
// Location is a region / datacenter where servers can be created. JSON tags
// match the admin UI's HetznerLocation type so the catalog endpoints serialize
// straight through without a translation layer.
type Location struct {
Name string `json:"name"` // "fsn1", "hil", etc.
Description string `json:"description"` // "Falkenstein DC Park 1"
City string `json:"city"`
Country string `json:"country"` // ISO-3166 alpha-2
Network string `json:"network_zone"` // continent or "EU"/"US" grouping for UI
}
// ServerTypeLocationPrice is the price of one ServerType in one location.
// Hetzner prices vary per location (US regions carry a premium), so the UI
// keys the displayed price off the selected location rather than one flat
// number.
type ServerTypeLocationPrice struct {
Location string `json:"location"`
PriceMonthlyEUR float64 `json:"price_monthly_eur"`
PriceHourlyEUR float64 `json:"price_hourly_eur"`
}
// ServerType is one purchasable VPS shape. JSON tags are snake_case to match
// the admin UI's HetznerServerType type — without them encoding/json would
// emit PascalCase and the UI would read every field as undefined.
type ServerType struct {
Name string `json:"name"` // "cx22", "cpx11"
Description string `json:"description"`
Cores int `json:"cores"`
Memory float64 `json:"memory_gb"` // GiB
Disk int `json:"disk_gb"` // GiB
StorageType string `json:"storage_type,omitempty"` // "local" / "network"
CPUType string `json:"cpu_type,omitempty"` // "shared" / "dedicated"
Architecture string `json:"architecture,omitempty"` // "x86" / "arm"
// PriceMonthlyEUR is the headline (cheapest-location) gross monthly price.
// Prices carries the full per-location breakdown the UI prefers.
PriceMonthlyEUR float64 `json:"price_monthly_eur"`
PriceHourlyEUR float64 `json:"price_hourly_eur,omitempty"`
PriceMonthlyUSD float64 `json:"price_monthly_usd,omitempty"`
// PriceIPv4MonthlyEUR is the gross monthly cost of one extra Primary IPv4.
// Uniform across server types (it's an IP, not a machine attribute) so it's
// populated from a documented provider constant.
PriceIPv4MonthlyEUR float64 `json:"price_ipv4_monthly_eur,omitempty"`
IncludedTrafficTB float64 `json:"included_traffic_tb,omitempty"`
Prices []ServerTypeLocationPrice `json:"prices,omitempty"`
}
// Image is an OS image available for new servers.
type Image struct {
Name string `json:"name"` // "ubuntu-22.04"
Description string `json:"description"`
OSFlavor string `json:"os_flavor"`
OSVersion string `json:"os_version"`
}
// CreateServerRequest is what the state machine passes to CreateServer.
type CreateServerRequest struct {
Name string
ServerType string
Image string
Location string
Datacenter string // overrides Location when set
SSHKeyIDs []string
UserData string // cloud-init
Labels map[string]string
PlacementGroup string
PrivateNetwork string
Firewall string
StartAfterCreate bool
}
// Server is what CreateServer returns.
type Server struct {
ID string
Name string
Status string
PublicIPv4 string
PublicIPv6 string
}
// CreatePrimaryIPRequest configures one extra Primary IP. The IP that
// comes free with a server is created by CreateServer, not here.
type CreatePrimaryIPRequest struct {
Type string // "ipv4" / "ipv6"
Name string
Datacenter string // must match the server's datacenter
Labels map[string]string
}
// PrimaryIP is what CreatePrimaryIP returns.
type PrimaryIP struct {
ID string
Type string
IP string
AssignedToServerID string // empty when unassigned
}
@@ -1,407 +0,0 @@
// Package hetzner implements cloudprovider.Provider over the Hetzner Cloud
// REST API (https://docs.hetzner.cloud/).
//
// Auth is a single bearer token (Project API token). One token is one
// project; multi-project operators register multiple cloud_credentials rows.
package hetzner
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
)
const (
defaultBaseURL = "https://api.hetzner.cloud/v1"
defaultTimeout = 30 * time.Second
// primaryIPv4MonthlyEUR is the gross monthly list price of one Hetzner
// Primary IPv4 (~€0.50 net + VAT). It's uniform across server types and
// locations, so we surface it as a documented constant rather than a
// per-type field. Refine via GET /pricing (primary_ips) if exactness ever
// matters for billing; today it's an informational estimate only.
primaryIPv4MonthlyEUR = 0.60
)
// Client is the Hetzner Cloud API client implementing cloudprovider.Provider.
type Client struct {
baseURL string
token string
http *http.Client
}
// Option customizes the Client. WithHTTPClient and WithBaseURL are useful
// for tests against httptest.Server.
type Option func(*Client)
func WithBaseURL(u string) Option { return func(c *Client) { c.baseURL = u } }
func WithHTTPClient(h *http.Client) Option { return func(c *Client) { c.http = h } }
// New returns a Client authenticated with the given Hetzner project token.
func New(token string, opts ...Option) (*Client, error) {
if token == "" {
return nil, errors.New("hetzner: token is required")
}
c := &Client{
baseURL: defaultBaseURL,
token: token,
http: &http.Client{Timeout: defaultTimeout},
}
for _, o := range opts {
o(c)
}
return c, nil
}
func (c *Client) Name() string { return "hetzner" }
// ---------------------------------------------------------------------------
// Plumbing
// ---------------------------------------------------------------------------
func (c *Client) do(ctx context.Context, method, path string, body any, out any) error {
var rdr io.Reader
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("hetzner: marshal body: %w", err)
}
rdr = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, rdr)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+c.token)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("hetzner: %s %s: %w", method, path, err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("hetzner: read body: %w", err)
}
if resp.StatusCode >= 400 {
var e struct {
Error struct {
Code, Message string
} `json:"error"`
}
_ = json.Unmarshal(respBody, &e)
if e.Error.Message != "" {
return fmt.Errorf("hetzner: %s %s: %d %s (%s)",
method, path, resp.StatusCode, e.Error.Message, e.Error.Code)
}
return fmt.Errorf("hetzner: %s %s: %d %s",
method, path, resp.StatusCode, string(respBody))
}
if out == nil {
return nil
}
if err := json.Unmarshal(respBody, out); err != nil {
return fmt.Errorf("hetzner: decode response: %w", err)
}
return nil
}
// Verify hits a cheap authenticated endpoint to confirm the token is valid.
func (c *Client) Verify(ctx context.Context) error {
var out struct {
Datacenters []map[string]any `json:"datacenters"`
}
return c.do(ctx, http.MethodGet, "/datacenters", nil, &out)
}
// ---------------------------------------------------------------------------
// Catalog: locations, server_types, images
// ---------------------------------------------------------------------------
type apiLocation struct {
ID int `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Country string `json:"country"`
City string `json:"city"`
NetworkZone string `json:"network_zone"`
}
func (c *Client) Locations(ctx context.Context) ([]cloudprovider.Location, error) {
var out struct {
Locations []apiLocation `json:"locations"`
}
if err := c.do(ctx, http.MethodGet, "/locations", nil, &out); err != nil {
return nil, err
}
locs := make([]cloudprovider.Location, 0, len(out.Locations))
for _, l := range out.Locations {
locs = append(locs, cloudprovider.Location{
Name: l.Name,
Description: l.Description,
City: l.City,
Country: l.Country,
Network: l.NetworkZone,
})
}
return locs, nil
}
type apiPrice struct {
Location string `json:"location"`
PriceHourly struct {
Gross string `json:"gross"`
Net string `json:"net"`
} `json:"price_hourly"`
PriceMonthly struct {
Gross string `json:"gross"`
Net string `json:"net"`
} `json:"price_monthly"`
}
type apiServerType struct {
ID int `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Cores int `json:"cores"`
Memory float64 `json:"memory"`
Disk int `json:"disk"`
StorageType string `json:"storage_type"`
CPUType string `json:"cpu_type"`
Architecture string `json:"architecture"`
Prices []apiPrice `json:"prices"`
}
func (c *Client) ServerTypes(ctx context.Context) ([]cloudprovider.ServerType, error) {
var out struct {
ServerTypes []apiServerType `json:"server_types"`
}
if err := c.do(ctx, http.MethodGet, "/server_types?per_page=100", nil, &out); err != nil {
return nil, err
}
types := make([]cloudprovider.ServerType, 0, len(out.ServerTypes))
for _, t := range out.ServerTypes {
st := cloudprovider.ServerType{
Name: t.Name,
Description: t.Description,
Cores: t.Cores,
Memory: t.Memory,
Disk: t.Disk,
StorageType: t.StorageType,
CPUType: t.CPUType,
Architecture: t.Architecture,
PriceIPv4MonthlyEUR: primaryIPv4MonthlyEUR,
}
// Hetzner returns prices as decimal strings, per location. Carry the
// full per-location breakdown and use the cheapest location for the
// headline price (the UI shows the price for the selected location).
for _, p := range t.Prices {
gross, _ := strconv.ParseFloat(p.PriceMonthly.Gross, 64)
hourly, _ := strconv.ParseFloat(p.PriceHourly.Gross, 64)
st.Prices = append(st.Prices, cloudprovider.ServerTypeLocationPrice{
Location: p.Location,
PriceMonthlyEUR: gross,
PriceHourlyEUR: hourly,
})
if st.PriceMonthlyEUR == 0 || gross < st.PriceMonthlyEUR {
st.PriceMonthlyEUR = gross
st.PriceHourlyEUR = hourly
}
}
types = append(types, st)
}
return types, nil
}
func (c *Client) Images(ctx context.Context) ([]cloudprovider.Image, error) {
var out struct {
Images []struct {
ID int `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
OSFlavor string `json:"os_flavor"`
OSVersion string `json:"os_version"`
Type string `json:"type"`
} `json:"images"`
}
if err := c.do(ctx, http.MethodGet, "/images?type=system&per_page=100", nil, &out); err != nil {
return nil, err
}
imgs := make([]cloudprovider.Image, 0, len(out.Images))
for _, i := range out.Images {
if i.Type != "system" {
continue
}
imgs = append(imgs, cloudprovider.Image{
Name: i.Name,
Description: i.Description,
OSFlavor: i.OSFlavor,
OSVersion: i.OSVersion,
})
}
return imgs, nil
}
// ---------------------------------------------------------------------------
// Server lifecycle
// ---------------------------------------------------------------------------
type createServerReq struct {
Name string `json:"name"`
ServerType string `json:"server_type"`
Image string `json:"image"`
Location string `json:"location,omitempty"`
Datacenter string `json:"datacenter,omitempty"`
SSHKeys []string `json:"ssh_keys,omitempty"`
UserData string `json:"user_data,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
PlacementGroup *string `json:"placement_group,omitempty"`
Networks []string `json:"networks,omitempty"`
Firewalls []firewallRef `json:"firewalls,omitempty"`
StartAfterCreate bool `json:"start_after_create"`
}
type firewallRef struct {
Firewall string `json:"firewall"`
}
type apiServer struct {
ID int `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
PublicNet struct {
IPv4 struct {
IP string `json:"ip"`
} `json:"ipv4"`
IPv6 struct {
IP string `json:"ip"`
} `json:"ipv6"`
} `json:"public_net"`
}
func (c *Client) CreateServer(ctx context.Context, req cloudprovider.CreateServerRequest) (*cloudprovider.Server, error) {
body := createServerReq{
Name: req.Name,
ServerType: req.ServerType,
Image: req.Image,
Location: req.Location,
Datacenter: req.Datacenter,
SSHKeys: req.SSHKeyIDs,
UserData: req.UserData,
Labels: req.Labels,
StartAfterCreate: req.StartAfterCreate,
}
if req.PlacementGroup != "" {
body.PlacementGroup = &req.PlacementGroup
}
if req.PrivateNetwork != "" {
body.Networks = []string{req.PrivateNetwork}
}
if req.Firewall != "" {
body.Firewalls = []firewallRef{{Firewall: req.Firewall}}
}
var out struct {
Server apiServer `json:"server"`
}
if err := c.do(ctx, http.MethodPost, "/servers", body, &out); err != nil {
return nil, err
}
return &cloudprovider.Server{
ID: strconv.Itoa(out.Server.ID),
Name: out.Server.Name,
Status: out.Server.Status,
PublicIPv4: out.Server.PublicNet.IPv4.IP,
PublicIPv6: out.Server.PublicNet.IPv6.IP,
}, nil
}
func (c *Client) DeleteServer(ctx context.Context, serverID string) error {
return c.do(ctx, http.MethodDelete, "/servers/"+serverID, nil, nil)
}
// ---------------------------------------------------------------------------
// Primary IPs
// ---------------------------------------------------------------------------
type createPrimaryIPReq struct {
Type string `json:"type"`
Name string `json:"name"`
Datacenter string `json:"datacenter,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
AssigneeType string `json:"assignee_type"`
}
type apiPrimaryIP struct {
ID int `json:"id"`
Type string `json:"type"`
IP string `json:"ip"`
AssigneeID *int `json:"assignee_id"`
AssigneeType string `json:"assignee_type"`
AutoDelete bool `json:"auto_delete"`
}
func (c *Client) CreatePrimaryIP(ctx context.Context, req cloudprovider.CreatePrimaryIPRequest) (*cloudprovider.PrimaryIP, error) {
body := createPrimaryIPReq{
Type: req.Type,
Name: req.Name,
Datacenter: req.Datacenter,
Labels: req.Labels,
AssigneeType: "server",
}
var out struct {
PrimaryIP apiPrimaryIP `json:"primary_ip"`
}
if err := c.do(ctx, http.MethodPost, "/primary_ips", body, &out); err != nil {
return nil, err
}
return &cloudprovider.PrimaryIP{
ID: strconv.Itoa(out.PrimaryIP.ID),
Type: out.PrimaryIP.Type,
IP: out.PrimaryIP.IP,
}, nil
}
func (c *Client) AssignPrimaryIP(ctx context.Context, ipID, serverID string) error {
sid, err := strconv.Atoi(serverID)
if err != nil {
return fmt.Errorf("hetzner: assign: invalid server id %q", serverID)
}
body := struct {
AssigneeType string `json:"assignee_type"`
AssigneeID int `json:"assignee_id"`
}{AssigneeType: "server", AssigneeID: sid}
return c.do(ctx, http.MethodPost, "/primary_ips/"+ipID+"/actions/assign", body, nil)
}
func (c *Client) UnassignPrimaryIP(ctx context.Context, ipID string) error {
return c.do(ctx, http.MethodPost, "/primary_ips/"+ipID+"/actions/unassign", nil, nil)
}
func (c *Client) DeletePrimaryIP(ctx context.Context, ipID string) error {
return c.do(ctx, http.MethodDelete, "/primary_ips/"+ipID, nil, nil)
}
func (c *Client) SetReverseDNS(ctx context.Context, ipID, hostname string) error {
body := struct {
IP string `json:"ip"`
DNSPtr string `json:"dns_ptr"`
}{DNSPtr: hostname}
return c.do(ctx, http.MethodPost, "/primary_ips/"+ipID+"/actions/change_dns_ptr", body, nil)
}
// Compile-time check.
var _ cloudprovider.Provider = (*Client)(nil)
@@ -1,195 +0,0 @@
package hetzner
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
)
func newTestClient(t *testing.T, handler http.HandlerFunc) (*Client, *httptest.Server) {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
c, err := New("test-token", WithBaseURL(srv.URL))
if err != nil {
t.Fatal(err)
}
return c, srv
}
func TestNew_RejectsEmptyToken(t *testing.T) {
if _, err := New(""); err == nil {
t.Fatal("expected error on empty token")
}
}
func TestDo_SendsBearerToken(t *testing.T) {
gotAuth := ""
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
_, _ = w.Write([]byte(`{"datacenters":[]}`))
})
if err := c.Verify(context.Background()); err != nil {
t.Fatal(err)
}
if gotAuth != "Bearer test-token" {
t.Fatalf("auth header: got %q", gotAuth)
}
}
func TestDo_SurfaceAPIError(t *testing.T) {
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":{"code":"unauthorized","message":"invalid token"}}`))
})
err := c.Verify(context.Background())
if err == nil || !strings.Contains(err.Error(), "invalid token") {
t.Fatalf("expected unauthorized error, got %v", err)
}
}
func TestLocations_Parsing(t *testing.T) {
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/locations" {
t.Fatalf("expected /locations, got %s", r.URL.Path)
}
_, _ = w.Write([]byte(`{
"locations": [
{"id":1,"name":"fsn1","description":"Falkenstein DC Park 1","country":"DE","city":"Falkenstein","network_zone":"eu-central"},
{"id":2,"name":"hil","description":"Hillsboro DC1","country":"US","city":"Hillsboro","network_zone":"us-west"}
]
}`))
})
locs, err := c.Locations(context.Background())
if err != nil {
t.Fatal(err)
}
if len(locs) != 2 || locs[0].Name != "fsn1" || locs[1].Country != "US" {
t.Fatalf("parse mismatch: %#v", locs)
}
}
func TestServerTypes_PicksCheapestPrice(t *testing.T) {
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{
"server_types": [
{
"id": 1, "name": "cx22", "description": "CX22",
"cores": 2, "memory": 4, "disk": 40,
"storage_type": "local", "cpu_type": "shared", "architecture": "x86",
"prices": [
{"location": "fsn1", "price_monthly": {"gross": "5.83", "net": "4.90"}},
{"location": "hil", "price_monthly": {"gross": "7.05", "net": "5.92"}}
]
}
]
}`))
})
types, err := c.ServerTypes(context.Background())
if err != nil {
t.Fatal(err)
}
if len(types) != 1 {
t.Fatalf("want 1 type, got %d", len(types))
}
if types[0].PriceMonthlyEUR != 5.83 {
t.Fatalf("want cheapest price 5.83, got %v", types[0].PriceMonthlyEUR)
}
if types[0].Cores != 2 || types[0].Memory != 4 || types[0].Disk != 40 {
t.Fatalf("specs mismatch: %#v", types[0])
}
}
func TestCreateServer_PostsAndParsesResponse(t *testing.T) {
var receivedBody createServerReq
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/servers" {
t.Fatalf("expected POST /servers, got %s %s", r.Method, r.URL.Path)
}
if err := json.NewDecoder(r.Body).Decode(&receivedBody); err != nil {
t.Fatal(err)
}
_, _ = w.Write([]byte(`{
"server": {
"id": 42, "name": "warmbly-fsn1-001", "status": "initializing",
"public_net": {
"ipv4": {"ip": "1.2.3.4"},
"ipv6": {"ip": "2a01::1"}
}
}
}`))
})
req := cloudprovider.CreateServerRequest{
Name: "warmbly-fsn1-001",
ServerType: "cx22",
Image: "ubuntu-22.04",
Location: "fsn1",
SSHKeyIDs: []string{"key-1"},
UserData: "#cloud-config\nrunmd: []",
Labels: map[string]string{"warmbly": "true"},
StartAfterCreate: true,
}
s, err := c.CreateServer(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if s.ID != "42" {
t.Fatalf("server id: got %q want %q", s.ID, "42")
}
if s.PublicIPv4 != "1.2.3.4" {
t.Fatalf("server ipv4: got %q", s.PublicIPv4)
}
if receivedBody.ServerType != "cx22" {
t.Fatalf("posted body mismatch: %#v", receivedBody)
}
if !receivedBody.StartAfterCreate {
t.Fatal("StartAfterCreate not propagated")
}
}
func TestCreatePrimaryIP_DefaultsAssigneeServer(t *testing.T) {
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
var body createPrimaryIPReq
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatal(err)
}
if body.AssigneeType != "server" {
t.Fatalf("want assignee_type=server, got %q", body.AssigneeType)
}
_, _ = w.Write([]byte(`{"primary_ip":{"id":7,"type":"ipv4","ip":"5.6.7.8"}}`))
})
ip, err := c.CreatePrimaryIP(context.Background(), cloudprovider.CreatePrimaryIPRequest{
Type: "ipv4", Name: "warmbly-ip-1", Datacenter: "fsn1-dc14",
})
if err != nil {
t.Fatal(err)
}
if ip.ID != "7" || ip.IP != "5.6.7.8" {
t.Fatalf("parse mismatch: %#v", ip)
}
}
func TestSetReverseDNS_PostsToAction(t *testing.T) {
c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
t.Fatalf("want POST, got %s", r.Method)
}
if !strings.HasSuffix(r.URL.Path, "/actions/change_dns_ptr") {
t.Fatalf("unexpected path: %s", r.URL.Path)
}
_, _ = w.Write([]byte(`{"action":{"id":1,"status":"running"}}`))
})
if err := c.SetReverseDNS(context.Background(), "7", "w.example.com"); err != nil {
t.Fatal(err)
}
}
func TestProviderInterfaceConformance(t *testing.T) {
var _ cloudprovider.Provider = (*Client)(nil)
}
@@ -0,0 +1,88 @@
-- Restores the four worker categories. The original per-worker values are not
-- recoverable, so every worker comes back as a shared, premium, clean,
-- cold_smtp box, which is the default a fresh install would have produced.
BEGIN;
DROP MATERIALIZED VIEW IF EXISTS worker_capacity_view;
DROP INDEX IF EXISTS idx_email_accounts_worker_assigned_at;
ALTER TABLE email_accounts DROP COLUMN IF EXISTS worker_assigned_at;
ALTER TABLE workers DROP COLUMN IF EXISTS region;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'worker_risk_pool') THEN
CREATE TYPE public.worker_risk_pool AS ENUM ('clean', 'risky', 'quarantine');
END IF;
END
$$;
ALTER TABLE workers
ADD COLUMN IF NOT EXISTS free_tier boolean DEFAULT false NOT NULL,
ADD COLUMN IF NOT EXISTS worker_type text DEFAULT 'shared'::text NOT NULL,
ADD COLUMN IF NOT EXISTS risk_pool public.worker_risk_pool DEFAULT 'clean'::public.worker_risk_pool NOT NULL,
ADD COLUMN IF NOT EXISTS egress_kind text DEFAULT 'cold_smtp'::text NOT NULL;
ALTER TABLE workers
ADD CONSTRAINT workers_egress_kind_check
CHECK ((egress_kind = ANY (ARRAY['cold_smtp'::text, 'oauth_api'::text, 'warmup_only'::text])));
ALTER TABLE provisioning_templates
ADD COLUMN IF NOT EXISTS tier text DEFAULT 'shared_premium'::text NOT NULL,
ADD COLUMN IF NOT EXISTS egress_kind text DEFAULT 'cold_smtp'::text NOT NULL;
ALTER TABLE provisioning_templates
ADD CONSTRAINT provisioning_templates_tier_check
CHECK ((tier = ANY (ARRAY['shared_free'::text, 'shared_premium'::text, 'dedicated'::text])));
ALTER TABLE provisioning_templates
ADD CONSTRAINT provisioning_templates_egress_kind_check
CHECK ((egress_kind = ANY (ARRAY['cold_smtp'::text, 'oauth_api'::text, 'warmup_only'::text])));
CREATE MATERIALIZED VIEW worker_capacity_view AS
WITH aggregated AS (
SELECT worker_health_samples.worker_id,
sum(worker_health_samples.sends_attempted) AS sends_attempted_1h,
sum(worker_health_samples.sends_succeeded) AS sends_succeeded_1h,
sum(worker_health_samples.bounces_hard) AS bounces_hard_1h,
sum(worker_health_samples.bounces_soft) AS bounces_soft_1h,
sum(worker_health_samples.complaints) AS complaints_1h,
sum(worker_health_samples.auth_errors) AS auth_errors_1h
FROM public.worker_health_samples
WHERE (worker_health_samples.observed_at > (now() - '01:00:00'::interval))
GROUP BY worker_health_samples.worker_id
)
SELECT w.id AS worker_id,
w.worker_type,
w.free_tier,
w.egress_kind,
w.health_state,
w.load_score,
(
CASE w.egress_kind
WHEN 'cold_smtp'::text THEN 16
WHEN 'oauth_api'::text THEN 400
WHEN 'warmup_only'::text THEN 25
ELSE 16
END)::numeric AS base_capacity,
GREATEST(0.0, LEAST(1.0, ((1.0 - LEAST(0.5, (((COALESCE(a.bounces_hard_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (5)::numeric))) - LEAST(0.5, (((COALESCE(a.complaints_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (100)::numeric))))) AS health_multiplier,
LEAST(1.0, (EXTRACT(epoch FROM (now() - w.created_at)) / ((72 * 3600))::numeric)) AS age_multiplier,
COALESCE(a.sends_attempted_1h, (0)::bigint) AS sends_attempted_1h,
COALESCE(a.sends_succeeded_1h, (0)::bigint) AS sends_succeeded_1h,
COALESCE(a.bounces_hard_1h, (0)::bigint) AS bounces_hard_1h,
COALESCE(a.bounces_soft_1h, (0)::bigint) AS bounces_soft_1h,
COALESCE(a.complaints_1h, (0)::bigint) AS complaints_1h,
COALESCE(a.auth_errors_1h, (0)::bigint) AS auth_errors_1h
FROM (public.workers w
LEFT JOIN aggregated a ON ((a.worker_id = w.id)))
WHERE w.active
WITH NO DATA;
CREATE UNIQUE INDEX worker_capacity_view_pk ON public.worker_capacity_view USING btree (worker_id);
REFRESH MATERIALIZED VIEW public.worker_capacity_view;
COMMIT;
@@ -0,0 +1,101 @@
-- One kind of worker.
--
-- Workers used to carry four operator- or plan-chosen categories (free_tier,
-- worker_type, risk_pool, egress_kind) and placement was a filter over them.
-- None of the four survives contact with how this product actually sends:
-- a worker never talks to a recipient MX, it authenticates to the customer's
-- own mailbox provider, which then does the delivery from its own outbound
-- pool. The worker's IP is therefore invisible to recipient spam filtering
-- (Google strips the submitting client IP) and matters only to the provider,
-- as a login-trust and connection-concurrency surface.
--
-- So co-locating a "risky" mailbox next to a clean one cannot contaminate the
-- clean one's sending reputation, and segregating free from paid buys nothing
-- a health signal doesn't already buy. Placement becomes a score over live
-- health, load, affinity and blast radius; the categories go.
BEGIN;
DROP MATERIALIZED VIEW IF EXISTS worker_capacity_view;
ALTER TABLE workers
DROP COLUMN IF EXISTS free_tier,
DROP COLUMN IF EXISTS worker_type,
DROP COLUMN IF EXISTS risk_pool,
DROP COLUMN IF EXISTS egress_kind;
DROP TYPE IF EXISTS worker_risk_pool;
-- Region is an affinity hint, never a partition. A mailbox scores better on a
-- worker whose egress geolocates near where its provider expects the account
-- to sign in from; an empty region simply scores neutral.
ALTER TABLE workers ADD COLUMN IF NOT EXISTS region text NOT NULL DEFAULT '';
-- Residency bookkeeping. Moving a mailbox changes the client IP its provider
-- sees, which is a trust event worth avoiding, so the placement loop enforces
-- a minimum residency and a per-mailbox cooldown. The old rebalancer
-- documented a 24h cooldown but had no column to enforce it with.
ALTER TABLE email_accounts
ADD COLUMN IF NOT EXISTS worker_assigned_at timestamptz;
UPDATE email_accounts
SET worker_assigned_at = now()
WHERE worker_id IS NOT NULL
AND worker_assigned_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_email_accounts_worker_assigned_at
ON email_accounts (worker_assigned_at)
WHERE worker_id IS NOT NULL;
-- Provisioning templates categorised the machines they created. They now
-- describe only the machine shape; what lands on it is the placer's call.
ALTER TABLE provisioning_templates
DROP CONSTRAINT IF EXISTS provisioning_templates_tier_check,
DROP CONSTRAINT IF EXISTS provisioning_templates_egress_kind_check;
ALTER TABLE provisioning_templates
DROP COLUMN IF EXISTS tier,
DROP COLUMN IF EXISTS egress_kind;
-- base_capacity is now one number for every worker, in cold-mailbox
-- equivalents. It does not need to branch on an egress category because each
-- mailbox already contributes its own weight (see worker.MailboxWeight): an
-- OAuth-API mailbox costs 0.05 and a cold SMTP mailbox costs 1.0, so a worker
-- carrying either kind converges on the same load number without the worker
-- having to declare which kind it expects.
CREATE MATERIALIZED VIEW worker_capacity_view AS
WITH aggregated AS (
SELECT worker_health_samples.worker_id,
sum(worker_health_samples.sends_attempted) AS sends_attempted_1h,
sum(worker_health_samples.sends_succeeded) AS sends_succeeded_1h,
sum(worker_health_samples.bounces_hard) AS bounces_hard_1h,
sum(worker_health_samples.bounces_soft) AS bounces_soft_1h,
sum(worker_health_samples.complaints) AS complaints_1h,
sum(worker_health_samples.auth_errors) AS auth_errors_1h
FROM public.worker_health_samples
WHERE (worker_health_samples.observed_at > (now() - '01:00:00'::interval))
GROUP BY worker_health_samples.worker_id
)
SELECT w.id AS worker_id,
w.region,
w.health_state,
w.load_score,
(16)::numeric AS base_capacity,
GREATEST(0.0, LEAST(1.0, ((1.0 - LEAST(0.5, (((COALESCE(a.bounces_hard_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (5)::numeric))) - LEAST(0.5, (((COALESCE(a.complaints_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (100)::numeric))))) AS health_multiplier,
LEAST(1.0, (EXTRACT(epoch FROM (now() - w.created_at)) / ((72 * 3600))::numeric)) AS age_multiplier,
COALESCE(a.sends_attempted_1h, (0)::bigint) AS sends_attempted_1h,
COALESCE(a.sends_succeeded_1h, (0)::bigint) AS sends_succeeded_1h,
COALESCE(a.bounces_hard_1h, (0)::bigint) AS bounces_hard_1h,
COALESCE(a.bounces_soft_1h, (0)::bigint) AS bounces_soft_1h,
COALESCE(a.complaints_1h, (0)::bigint) AS complaints_1h,
COALESCE(a.auth_errors_1h, (0)::bigint) AS auth_errors_1h
FROM (public.workers w
LEFT JOIN aggregated a ON ((a.worker_id = w.id)))
WHERE w.active
WITH NO DATA;
CREATE UNIQUE INDEX worker_capacity_view_pk ON public.worker_capacity_view USING btree (worker_id);
REFRESH MATERIALIZED VIEW public.worker_capacity_view;
COMMIT;
@@ -0,0 +1,78 @@
-- Puts the machine columns back on workers and drops the node registry.
-- Consumer nodes have nowhere to go in the old shape, so they are lost.
BEGIN;
DROP MATERIALIZED VIEW IF EXISTS worker_capacity_view;
ALTER TABLE workers DROP CONSTRAINT IF EXISTS workers_id_is_a_node;
ALTER TABLE workers
ADD COLUMN IF NOT EXISTS name character varying(255) DEFAULT ''::character varying NOT NULL,
ADD COLUMN IF NOT EXISTS notes text,
ADD COLUMN IF NOT EXISTS ip_addr text DEFAULT ''::text NOT NULL,
ADD COLUMN IF NOT EXISTS region text DEFAULT ''::text NOT NULL,
ADD COLUMN IF NOT EXISTS active boolean DEFAULT false,
ADD COLUMN IF NOT EXISTS last_seen_at timestamp with time zone,
ADD COLUMN IF NOT EXISTS image_version text DEFAULT ''::text NOT NULL,
ADD COLUMN IF NOT EXISTS last_error text;
UPDATE workers w
SET name = n.name,
notes = NULLIF(n.notes, ''),
ip_addr = n.address,
region = n.region,
active = n.active,
last_seen_at = n.last_seen_at,
image_version = n.version,
last_error = NULLIF(n.last_error, '')
FROM fleet_nodes n
WHERE n.id = w.id;
-- Drop tag rows that belong to non-worker nodes before the key points back at
-- workers, or the constraint cannot be created.
DELETE FROM worker_tags t WHERE NOT EXISTS (SELECT 1 FROM workers w WHERE w.id = t.worker_id);
ALTER TABLE worker_tags DROP CONSTRAINT IF EXISTS worker_tags_node_id_fkey;
ALTER TABLE worker_tags
ADD CONSTRAINT worker_tags_worker_id_fkey
FOREIGN KEY (worker_id) REFERENCES workers(id) ON DELETE CASCADE;
DROP TABLE IF EXISTS fleet_nodes;
CREATE MATERIALIZED VIEW worker_capacity_view AS
WITH aggregated AS (
SELECT worker_health_samples.worker_id,
sum(worker_health_samples.sends_attempted) AS sends_attempted_1h,
sum(worker_health_samples.sends_succeeded) AS sends_succeeded_1h,
sum(worker_health_samples.bounces_hard) AS bounces_hard_1h,
sum(worker_health_samples.bounces_soft) AS bounces_soft_1h,
sum(worker_health_samples.complaints) AS complaints_1h,
sum(worker_health_samples.auth_errors) AS auth_errors_1h
FROM public.worker_health_samples
WHERE (worker_health_samples.observed_at > (now() - '01:00:00'::interval))
GROUP BY worker_health_samples.worker_id
)
SELECT w.id AS worker_id,
w.region,
w.health_state,
w.load_score,
(16)::numeric AS base_capacity,
GREATEST(0.0, LEAST(1.0, ((1.0 - LEAST(0.5, (((COALESCE(a.bounces_hard_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (5)::numeric))) - LEAST(0.5, (((COALESCE(a.complaints_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (100)::numeric))))) AS health_multiplier,
LEAST(1.0, (EXTRACT(epoch FROM (now() - w.created_at)) / ((72 * 3600))::numeric)) AS age_multiplier,
COALESCE(a.sends_attempted_1h, (0)::bigint) AS sends_attempted_1h,
COALESCE(a.sends_succeeded_1h, (0)::bigint) AS sends_succeeded_1h,
COALESCE(a.bounces_hard_1h, (0)::bigint) AS bounces_hard_1h,
COALESCE(a.bounces_soft_1h, (0)::bigint) AS bounces_soft_1h,
COALESCE(a.complaints_1h, (0)::bigint) AS complaints_1h,
COALESCE(a.auth_errors_1h, (0)::bigint) AS auth_errors_1h
FROM (public.workers w
LEFT JOIN aggregated a ON ((a.worker_id = w.id)))
WHERE w.active
WITH NO DATA;
CREATE UNIQUE INDEX worker_capacity_view_pk ON public.worker_capacity_view USING btree (worker_id);
REFRESH MATERIALIZED VIEW public.worker_capacity_view;
COMMIT;
@@ -0,0 +1,128 @@
-- The fleet becomes pull-based, and every process that runs on a machine you
-- own becomes a node.
--
-- A node enrols with a join token, heartbeats, reports what version it is
-- running and what it is using, and asks the control plane what version it
-- SHOULD be running. Nothing reaches into a node. That replaces the push
-- model, where the backend held SSH keys, bought servers through a cloud API,
-- and shelled in to install, restart and upgrade them.
--
-- fleet_nodes is the registry every role shares. `workers` becomes a pure
-- placement extension: the columns that describe a *machine* (name, address,
-- region, version, liveness) move to the node, and the columns that describe
-- *what mail it carries* stay. workers.id IS the node id, enforced by the
-- foreign key, so "every worker is a node" cannot drift.
BEGIN;
CREATE TABLE fleet_nodes (
id uuid PRIMARY KEY,
role text NOT NULL,
name text NOT NULL DEFAULT '',
notes text NOT NULL DEFAULT '',
-- Where it is and how to reach it. region is the placement hint; address
-- is whatever the node reports as its outbound address.
region text NOT NULL DEFAULT '',
address text NOT NULL DEFAULT '',
-- version is what the node reports it is running. desired_version is
-- resolved per role by the control plane; pinned_version overrides it for
-- one node, so a single machine can be held back or canaried.
version text NOT NULL DEFAULT '',
pinned_version text NOT NULL DEFAULT '',
active boolean NOT NULL DEFAULT true,
last_seen_at timestamp with time zone,
enrolled_at timestamp with time zone NOT NULL DEFAULT now(),
-- Usage, overwritten on every beat. Deliberately a snapshot and not a
-- history: worker_health_samples already keeps the time series capacity
-- math needs, and a per-node metrics table would grow without a reader.
cpu_percent numeric(5,2),
memory_mb integer,
goroutines integer,
uptime_seconds bigint,
last_error text NOT NULL DEFAULT '',
created_at timestamp with time zone NOT NULL DEFAULT now(),
updated_at timestamp with time zone NOT NULL DEFAULT now(),
CONSTRAINT fleet_nodes_role_check CHECK (role = ANY (ARRAY['worker'::text, 'consumer'::text]))
);
CREATE INDEX idx_fleet_nodes_role_seen ON fleet_nodes (role, last_seen_at DESC);
CREATE INDEX idx_fleet_nodes_live ON fleet_nodes (last_seen_at) WHERE active;
-- Existing workers become nodes, carrying over what they already reported.
INSERT INTO fleet_nodes (id, role, name, notes, region, address, version, active,
last_seen_at, enrolled_at, last_error, created_at, updated_at)
SELECT w.id, 'worker', w.name, COALESCE(w.notes, ''), COALESCE(w.region, ''),
COALESCE(w.ip_addr, ''), COALESCE(w.image_version, ''), COALESCE(w.active, false),
w.last_seen_at, w.created_at, COALESCE(w.last_error, ''), w.created_at, w.updated_at
FROM workers w
ON CONFLICT (id) DO NOTHING;
-- The capacity view reads the machine's region through the node now.
DROP MATERIALIZED VIEW IF EXISTS worker_capacity_view;
ALTER TABLE workers
DROP COLUMN IF EXISTS name,
DROP COLUMN IF EXISTS notes,
DROP COLUMN IF EXISTS ip_addr,
DROP COLUMN IF EXISTS region,
DROP COLUMN IF EXISTS active,
DROP COLUMN IF EXISTS last_seen_at,
DROP COLUMN IF EXISTS image_version,
DROP COLUMN IF EXISTS last_error;
ALTER TABLE workers
ADD CONSTRAINT workers_id_is_a_node
FOREIGN KEY (id) REFERENCES fleet_nodes(id) ON DELETE CASCADE;
-- Tags describe a machine, so they belong to the node. Repointing the foreign
-- key lets a consumer carry them too; the table keeps its name because
-- renaming it buys nothing a comment does not.
ALTER TABLE worker_tags DROP CONSTRAINT IF EXISTS worker_tags_worker_id_fkey;
ALTER TABLE worker_tags
ADD CONSTRAINT worker_tags_node_id_fkey
FOREIGN KEY (worker_id) REFERENCES fleet_nodes(id) ON DELETE CASCADE;
CREATE MATERIALIZED VIEW worker_capacity_view AS
WITH aggregated AS (
SELECT worker_health_samples.worker_id,
sum(worker_health_samples.sends_attempted) AS sends_attempted_1h,
sum(worker_health_samples.sends_succeeded) AS sends_succeeded_1h,
sum(worker_health_samples.bounces_hard) AS bounces_hard_1h,
sum(worker_health_samples.bounces_soft) AS bounces_soft_1h,
sum(worker_health_samples.complaints) AS complaints_1h,
sum(worker_health_samples.auth_errors) AS auth_errors_1h
FROM public.worker_health_samples
WHERE (worker_health_samples.observed_at > (now() - '01:00:00'::interval))
GROUP BY worker_health_samples.worker_id
)
SELECT w.id AS worker_id,
n.region,
w.health_state,
w.load_score,
(16)::numeric AS base_capacity,
GREATEST(0.0, LEAST(1.0, ((1.0 - LEAST(0.5, (((COALESCE(a.bounces_hard_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (5)::numeric))) - LEAST(0.5, (((COALESCE(a.complaints_1h, (0)::bigint))::numeric / (NULLIF(a.sends_attempted_1h, 0))::numeric) * (100)::numeric))))) AS health_multiplier,
LEAST(1.0, (EXTRACT(epoch FROM (now() - w.created_at)) / ((72 * 3600))::numeric)) AS age_multiplier,
COALESCE(a.sends_attempted_1h, (0)::bigint) AS sends_attempted_1h,
COALESCE(a.sends_succeeded_1h, (0)::bigint) AS sends_succeeded_1h,
COALESCE(a.bounces_hard_1h, (0)::bigint) AS bounces_hard_1h,
COALESCE(a.bounces_soft_1h, (0)::bigint) AS bounces_soft_1h,
COALESCE(a.complaints_1h, (0)::bigint) AS complaints_1h,
COALESCE(a.auth_errors_1h, (0)::bigint) AS auth_errors_1h
FROM ((public.workers w
JOIN public.fleet_nodes n ON ((n.id = w.id)))
LEFT JOIN aggregated a ON ((a.worker_id = w.id)))
WHERE n.active
WITH NO DATA;
CREATE UNIQUE INDEX worker_capacity_view_pk ON public.worker_capacity_view USING btree (worker_id);
REFRESH MATERIALIZED VIEW public.worker_capacity_view;
COMMIT;
@@ -0,0 +1,148 @@
-- Recreates the push-based provisioning tables and the SSH columns. Their
-- contents are gone: the servers, keys and templates they described were not
-- carried forward, so this restores the shape and nothing else.
BEGIN;
CREATE TYPE public.worker_install_state AS ENUM (
'pending',
'provisioning',
'installed',
'error',
'uninstalling',
'uninstalled'
);
ALTER TABLE workers
ADD COLUMN IF NOT EXISTS ssh_host text,
ADD COLUMN IF NOT EXISTS ssh_port integer DEFAULT 22 NOT NULL,
ADD COLUMN IF NOT EXISTS ssh_user character varying(64) DEFAULT 'root'::character varying NOT NULL,
ADD COLUMN IF NOT EXISTS ssh_public_key text,
ADD COLUMN IF NOT EXISTS ssh_private_key_encrypted text,
ADD COLUMN IF NOT EXISTS ssh_host_fingerprint text,
ADD COLUMN IF NOT EXISTS install_state public.worker_install_state DEFAULT 'pending'::public.worker_install_state NOT NULL,
ADD COLUMN IF NOT EXISTS enrollment_token_hash text,
ADD COLUMN IF NOT EXISTS enrollment_token_expires_at timestamp with time zone,
ADD COLUMN IF NOT EXISTS profile_id uuid,
ADD COLUMN IF NOT EXISTS config_applied_at timestamp with time zone;
CREATE TABLE public.aws_credentials (
id uuid DEFAULT gen_random_uuid() NOT NULL,
name character varying(120) NOT NULL,
description text DEFAULT ''::text NOT NULL,
region character varying(40) NOT NULL,
access_key_id text NOT NULL,
secret_access_key_encrypted text NOT NULL,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL
);
CREATE TABLE public.worker_profiles (
id uuid DEFAULT gen_random_uuid() NOT NULL,
name character varying(120) NOT NULL,
description text DEFAULT ''::text NOT NULL,
app_env character varying(20) DEFAULT 'prod'::character varying NOT NULL,
worker_image text DEFAULT 'ghcr.io/warmbly/worker:latest'::text NOT NULL,
kafka_bootstrap_servers text DEFAULT ''::text NOT NULL,
kafka_sasl_username text DEFAULT ''::text NOT NULL,
kafka_sasl_password_encrypted text DEFAULT ''::text NOT NULL,
schema_registry_url text DEFAULT ''::text NOT NULL,
schema_registry_key text DEFAULT ''::text NOT NULL,
schema_registry_secret_encrypted text DEFAULT ''::text NOT NULL,
redis_url_encrypted text DEFAULT ''::text NOT NULL,
aws_credential_id uuid,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL,
release_channel public.release_channel DEFAULT 'pinned'::public.release_channel NOT NULL,
auto_update boolean DEFAULT false NOT NULL,
resolved_image_tag text DEFAULT ''::text NOT NULL,
last_release_check_at timestamp with time zone
);
CREATE TABLE public.provisioning_templates (
id uuid DEFAULT gen_random_uuid() NOT NULL,
name text NOT NULL,
description text,
provider text NOT NULL,
location text NOT NULL,
datacenter text,
server_type text NOT NULL,
image text DEFAULT 'ubuntu-22.04'::text NOT NULL,
server_count integer DEFAULT 1 NOT NULL,
ipv4_per_server integer DEFAULT 1 NOT NULL,
ipv6_per_server integer DEFAULT 1 NOT NULL,
worker_profile_id uuid,
tier text NOT NULL,
egress_kind text DEFAULT 'cold_smtp'::text NOT NULL,
labels jsonb DEFAULT '{}'::jsonb NOT NULL,
placement_group text,
private_network text,
firewall text,
is_auto_template boolean DEFAULT false NOT NULL,
est_monthly_cost numeric(10,2),
est_cost_currency text DEFAULT 'EUR'::text,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT provisioning_templates_egress_kind_check CHECK ((egress_kind = ANY (ARRAY['cold_smtp'::text, 'oauth_api'::text, 'warmup_only'::text]))),
CONSTRAINT provisioning_templates_ipv4_per_server_check CHECK (((ipv4_per_server >= 1) AND (ipv4_per_server <= 64))),
CONSTRAINT provisioning_templates_server_count_check CHECK (((server_count >= 1) AND (server_count <= 100))),
CONSTRAINT provisioning_templates_tier_check CHECK ((tier = ANY (ARRAY['shared_free'::text, 'shared_premium'::text, 'dedicated'::text])))
);
CREATE TABLE public.provisioning_policy (
provider text NOT NULL,
enabled boolean DEFAULT true NOT NULL,
auto_provision boolean DEFAULT false NOT NULL,
max_per_day integer DEFAULT 2 NOT NULL,
max_per_month integer DEFAULT 30 NOT NULL,
monthly_budget numeric(10,2) DEFAULT 500,
budget_currency text DEFAULT 'EUR'::text,
cooldown_min integer DEFAULT 60 NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL
);
CREATE TABLE public.provisioning_jobs (
id uuid DEFAULT gen_random_uuid() NOT NULL,
state text DEFAULT 'pending'::text NOT NULL,
triggered_by text NOT NULL,
provider text NOT NULL,
credential_id uuid,
template_id uuid,
config jsonb NOT NULL,
provider_server_id text,
provider_ip_ids text[],
ips inet[],
worker_ids uuid[],
est_monthly_cost numeric(10,2),
cost_currency text DEFAULT 'EUR'::text,
error text,
attempts integer DEFAULT 0 NOT NULL,
last_step_at timestamp with time zone,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL,
completed_at timestamp with time zone,
CONSTRAINT provisioning_jobs_state_check CHECK ((state = ANY (ARRAY['pending'::text, 'creating_server'::text, 'creating_ips'::text, 'assigning_ips'::text, 'setting_rdns'::text, 'installing'::text, 'verifying'::text, 'completed'::text, 'failed'::text, 'rolling_back'::text])))
);
ALTER TABLE ONLY public.aws_credentials ADD CONSTRAINT aws_credentials_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.aws_credentials ADD CONSTRAINT aws_credentials_name_key UNIQUE (name);
ALTER TABLE ONLY public.worker_profiles ADD CONSTRAINT worker_profiles_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.worker_profiles ADD CONSTRAINT worker_profiles_name_key UNIQUE (name);
ALTER TABLE ONLY public.provisioning_templates ADD CONSTRAINT provisioning_templates_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.provisioning_templates ADD CONSTRAINT provisioning_templates_name_key UNIQUE (name);
ALTER TABLE ONLY public.provisioning_policy ADD CONSTRAINT provisioning_policy_pkey PRIMARY KEY (provider);
ALTER TABLE ONLY public.provisioning_jobs ADD CONSTRAINT provisioning_jobs_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.provisioning_jobs
ADD CONSTRAINT provisioning_jobs_template_id_fkey FOREIGN KEY (template_id) REFERENCES public.provisioning_templates(id) ON DELETE SET NULL;
ALTER TABLE ONLY public.provisioning_templates
ADD CONSTRAINT provisioning_templates_worker_profile_id_fkey FOREIGN KEY (worker_profile_id) REFERENCES public.worker_profiles(id) ON DELETE SET NULL;
ALTER TABLE ONLY public.worker_profiles
ADD CONSTRAINT worker_profiles_aws_credential_id_fkey FOREIGN KEY (aws_credential_id) REFERENCES public.aws_credentials(id) ON DELETE RESTRICT;
ALTER TABLE ONLY public.workers
ADD CONSTRAINT workers_profile_id_fkey FOREIGN KEY (profile_id) REFERENCES public.worker_profiles(id) ON DELETE SET NULL;
COMMIT;
@@ -0,0 +1,41 @@
-- Removes the push half of fleet management.
--
-- Servers used to be bought through a cloud API from a stored template, then
-- configured over SSH from a keypair the backend held. A node now joins by
-- running one command on a machine you already have, and keeps itself current
-- by asking the control plane what version it should be. None of the pieces
-- below have a job left:
--
-- provisioning_templates/_jobs/_policy bought and tracked cloud servers
-- aws_credentials, worker_profiles templated env for machines we configured
-- workers.ssh_*, install_state the SSH channel and its lifecycle
-- workers.enrollment_token_* per-worker tokens, replaced by the
-- instance join token in admin_settings
--
-- decision_log stays: it is the audit trail of what the control loops decided,
-- which still matters.
BEGIN;
ALTER TABLE workers
DROP COLUMN IF EXISTS ssh_host,
DROP COLUMN IF EXISTS ssh_port,
DROP COLUMN IF EXISTS ssh_user,
DROP COLUMN IF EXISTS ssh_public_key,
DROP COLUMN IF EXISTS ssh_private_key_encrypted,
DROP COLUMN IF EXISTS ssh_host_fingerprint,
DROP COLUMN IF EXISTS install_state,
DROP COLUMN IF EXISTS enrollment_token_hash,
DROP COLUMN IF EXISTS enrollment_token_expires_at,
DROP COLUMN IF EXISTS profile_id,
DROP COLUMN IF EXISTS config_applied_at;
DROP TABLE IF EXISTS provisioning_jobs;
DROP TABLE IF EXISTS provisioning_templates;
DROP TABLE IF EXISTS provisioning_policy;
DROP TABLE IF EXISTS worker_profiles;
DROP TABLE IF EXISTS aws_credentials;
DROP TYPE IF EXISTS worker_install_state;
COMMIT;
+15 -14
View File
@@ -142,16 +142,15 @@ type UnbanUserRequest struct {
// AdminWorkerDetail represents a worker with admin-relevant details
type AdminWorkerDetail struct {
ID uuid.UUID `json:"id"`
Name string `json:"name"`
Notes string `json:"notes"`
IPAddr string `json:"ip_addr"`
Active bool `json:"active"`
FreeTier bool `json:"free_tier"`
WorkerType WorkerType `json:"worker_type"`
AccountCount int `json:"account_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ID uuid.UUID `json:"id"`
Name string `json:"name"`
Notes string `json:"notes"`
IPAddr string `json:"ip_addr"`
Active bool `json:"active"`
Region string `json:"region"`
AccountCount int `json:"account_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
// Statistics
EmailsSentToday int `json:"emails_sent_today"`
@@ -169,10 +168,12 @@ type AdminWorkersResult struct {
// AdminUpdateWorker represents the request to update a worker
type AdminUpdateWorker struct {
Name *string `json:"name,omitempty"`
Notes *string `json:"notes,omitempty"`
Active *bool `json:"active,omitempty"`
WorkerType *WorkerType `json:"worker_type,omitempty"`
Name *string `json:"name,omitempty"`
Notes *string `json:"notes,omitempty"`
Active *bool `json:"active,omitempty"`
// Region is a sign-in geography hint the placer scores on. It is the only
// worker attribute an operator sets, and leaving it empty is fine.
Region *string `json:"region,omitempty"`
}
// AdminWorkerEmail represents an email account connected to a worker, including
+2 -12
View File
@@ -201,12 +201,9 @@ type AdminFleetWorkerRow struct {
Name string `json:"name"`
IPAddr string `json:"ip_addr"`
Active bool `json:"active"`
FreeTier bool `json:"free_tier"`
WorkerType WorkerType `json:"worker_type"`
RiskPool WorkerRiskPool `json:"risk_pool"`
EgressKind WorkerEgressKind `json:"egress_kind"`
Region string `json:"region"`
HealthState WorkerHealthState `json:"health_state"`
InstallState string `json:"install_state"`
Version string `json:"version"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
Live bool `json:"live"`
AccountCount int `json:"account_count"`
@@ -256,13 +253,6 @@ type AdminDedicatedAssignment struct {
AccountCount int `json:"account_count"`
}
// AdminConvertDedicatedRequest is the body of POST /admin/workers/:id/convert-dedicated.
type AdminConvertDedicatedRequest struct {
OrganizationID string `json:"organization_id"`
SubscriptionID string `json:"subscription_id"`
DrainToWorkerID *string `json:"drain_to_worker_id"`
}
// ---- workspace transfers ----
// AdminTransferJob is an export or import job with its workspace attached.
+8 -4
View File
@@ -394,10 +394,14 @@ type ContactSentEmail struct {
SequenceName *string `json:"step_name,omitempty"`
// Engagement (from campaign_contact_progress, may be nil).
OpenedAt *time.Time `json:"opened_at,omitempty"`
ClickedAt *time.Time `json:"clicked_at,omitempty"`
RepliedAt *time.Time `json:"replied_at,omitempty"`
BouncedAt *time.Time `json:"bounced_at,omitempty"`
// OpenedAt is a person's open. An automated fetch (client prefetch,
// security gateway) lands in MachineOpenedAt instead, so the two are
// never mistaken for each other.
OpenedAt *time.Time `json:"opened_at,omitempty"`
MachineOpenedAt *time.Time `json:"machine_opened_at,omitempty"`
ClickedAt *time.Time `json:"clicked_at,omitempty"`
RepliedAt *time.Time `json:"replied_at,omitempty"`
BouncedAt *time.Time `json:"bounced_at,omitempty"`
}
type ContactSentEmailsResult struct {
+135
View File
@@ -0,0 +1,135 @@
package models
import (
"time"
"github.com/google/uuid"
)
// NodeRole is what a process on a machine does. Both roles share the same
// lifecycle - enrol, heartbeat, report usage, self-update - and differ only in
// what else the control plane knows about them: a worker additionally carries
// mailbox placement, a consumer carries nothing extra.
type NodeRole string
const (
NodeRoleWorker NodeRole = "worker"
NodeRoleConsumer NodeRole = "consumer"
)
// Valid reports whether the role is one the control plane accepts. Anything
// else is refused at enrolment rather than stored and puzzled over later.
func (r NodeRole) Valid() bool {
return r == NodeRoleWorker || r == NodeRoleConsumer
}
// NodeLivenessWindow is how long after its last beat a node is still treated as
// live. Generous relative to the 90s beat interval so one slow request, a GC
// pause or a brief network blip never looks like a dead machine.
const NodeLivenessWindow = 5 * time.Minute
// FleetNode is one Warmbly process running on a machine you own.
//
// Everything here is reported BY the node or resolved FOR it. Nothing is
// configured on it by hand: a node is identified by the id it enrols with, and
// the only operator-set fields are cosmetic (name, notes) or an explicit
// override (pinned_version).
type FleetNode struct {
ID uuid.UUID `json:"id"`
Role NodeRole `json:"role"`
Name string `json:"name"`
Notes string `json:"notes"`
// Region is the sign-in geography hint placement scores on. Address is
// whatever the node reports as its outbound address.
Region string `json:"region"`
Address string `json:"address"`
// Version is what the node reports it is running. DesiredVersion is what
// the control plane wants it to run, resolved per role unless
// PinnedVersion overrides it for this one machine.
Version string `json:"version"`
PinnedVersion string `json:"pinned_version,omitempty"`
DesiredVersion string `json:"desired_version,omitempty"`
Active bool `json:"active"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
EnrolledAt time.Time `json:"enrolled_at"`
Usage NodeUsage `json:"usage"`
// MailboxCount is how many mailboxes this node carries. Set for workers
// only; nil for a consumer, which carries none by definition.
MailboxCount *int `json:"mailbox_count,omitempty"`
LastError string `json:"last_error,omitempty"`
Tags []string `json:"tags,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// NodeUsage is the resource snapshot a node reports on every beat. A snapshot
// and not a history on purpose: worker_health_samples already keeps the time
// series capacity math reads, and a second per-node metrics table would grow
// without anyone querying it.
type NodeUsage struct {
CPUPercent *float64 `json:"cpu_percent,omitempty"`
MemoryMB *int `json:"memory_mb,omitempty"`
Goroutines *int `json:"goroutines,omitempty"`
UptimeSeconds *int64 `json:"uptime_seconds,omitempty"`
}
// Live reports whether the node has beaten recently enough to be given work.
// Computed rather than stored so it can never go stale in the row.
func (n *FleetNode) Live() bool {
if n == nil || !n.Active || n.LastSeenAt == nil {
return false
}
return time.Since(*n.LastSeenAt) <= NodeLivenessWindow
}
// NeedsUpdate reports whether the node is running something other than what
// the control plane wants. An empty DesiredVersion means "no opinion", which
// happens before the first release is resolved and must never be read as
// "downgrade to nothing".
func (n *FleetNode) NeedsUpdate() bool {
if n == nil || n.DesiredVersion == "" {
return false
}
return n.Version != n.DesiredVersion
}
// NodeHeartbeat is what a node POSTs on every beat.
type NodeHeartbeat struct {
NodeID uuid.UUID `json:"node_id"`
Role NodeRole `json:"role"`
Name string `json:"name,omitempty"`
Region string `json:"region,omitempty"`
Address string `json:"address,omitempty"`
Version string `json:"version,omitempty"`
Usage NodeUsage `json:"usage"`
// LastError is whatever went wrong since the previous beat, for the
// dashboard. Empty clears it.
LastError string `json:"last_error,omitempty"`
// Booted is set on the first beat of a fresh process. A worker holds its
// mailboxes in memory only, so the backend reloads them right away instead
// of leaving it to the reconciler's next pass.
Booted bool `json:"booted,omitempty"`
// Stopping is set on the farewell beat, so the row goes inactive at once
// rather than staying selectable until the beat ages out.
Stopping bool `json:"stopping,omitempty"`
}
// NodeHeartbeatReply is the control plane's answer. It is the only channel by
// which a node is told to do anything, which is what keeps the model pull-only.
type NodeHeartbeatReply struct {
// DesiredVersion is the image tag this node should be running. The node
// compares it to its own and updates itself when they differ. Empty means
// the control plane has no opinion yet; the node must leave itself alone.
DesiredVersion string `json:"desired_version,omitempty"`
// LivenessSeconds tells the node how long its beat is trusted for, so the
// beat interval and the server's window can never drift apart.
LivenessSeconds int `json:"liveness_seconds"`
}
+52
View File
@@ -0,0 +1,52 @@
package models
import "time"
// Admin settings keys for the fleet. Both live in `admin_settings` so every
// backend replica agrees and the values survive a restart.
const (
// FleetSettingsKeyRelease holds the resolved release the fleet should be
// running (FleetReleaseState).
FleetSettingsKeyRelease = "fleet.release"
// FleetSettingsKeyJoinToken holds the bcrypt-style hash of the instance
// join token. The token itself is shown once, at issue time, and never
// stored.
FleetSettingsKeyJoinToken = "fleet.join_token"
)
// ReleaseChannelStable and friends name which GitHub releases the fleet
// follows.
const (
FleetChannelStable = "stable"
FleetChannelDev = "dev"
// FleetChannelPinned freezes the fleet at whatever Tag currently says.
// Nodes still self-update TO that tag; they just stop following new ones.
FleetChannelPinned = "pinned"
)
// FleetReleaseState is the answer to "what version should my nodes be running".
//
// One tag covers every role: worker and consumer ship from the same repository
// and the same release, so a node needs the tag and already knows its own
// image name. Keeping it to one value is what makes "everything is on the same
// version" checkable at a glance instead of a per-role matrix.
type FleetReleaseState struct {
Channel string `json:"channel"`
// Tag is the resolved release, e.g. "v1.4.2". Empty means nothing has been
// resolved yet, which every consumer of this must read as "no opinion"
// rather than "downgrade to nothing".
Tag string `json:"tag"`
ResolvedAt time.Time `json:"resolved_at"`
// Source records how Tag was set, so an operator can tell an automatic
// resolution from a manual pin.
Source string `json:"source,omitempty"`
}
// DesiredVersion is the tag nodes should converge on, or "" when the control
// plane has no opinion.
func (s *FleetReleaseState) DesiredVersion() string {
if s == nil {
return ""
}
return s.Tag
}
+5 -17
View File
@@ -2,8 +2,10 @@ package models
// EmailRiskBand classifies a mailbox by reputation risk. The rebalancer
// derives this from WarmupHealthState and writes it into
// email_accounts.risk_band; nothing else should set it. Workers pick up
// mailboxes whose band matches their risk_pool.
// email_accounts.risk_band; nothing else should set it. It drives warmup
// partner selection and per-mailbox pacing, not worker placement: a mailbox
// landing in spam does not contaminate the machine it sends from, because the
// machine is not the sending identity.
type EmailRiskBand string
const (
@@ -13,7 +15,7 @@ const (
)
// RiskBandFromHealth maps the warmup health state machine into the simpler
// three-bucket risk_band that workers cluster by. The mapping is one-way
// three-bucket risk_band. The mapping is one-way
// (collapses watch/throttled/quarantined into the recovery pool) — the
// reverse direction is meaningless.
//
@@ -34,17 +36,3 @@ func RiskBandFromHealth(s WarmupHealthState) EmailRiskBand {
return EmailRiskBandClean
}
}
// MatchingRiskPool returns the worker risk_pool that should host mailboxes
// of this band. The naming intentionally mirrors so a band of X always
// goes to a pool of X — keeps the rebalancer trivial.
func (b EmailRiskBand) MatchingRiskPool() WorkerRiskPool {
switch b {
case EmailRiskBandRisky:
return WorkerRiskPoolRisky
case EmailRiskBandQuarantine:
return WorkerRiskPoolQuarantine
default:
return WorkerRiskPoolClean
}
}
+20
View File
@@ -51,6 +51,10 @@ type Plan struct {
StripeProductID *string `json:"stripe_product_id,omitempty"`
// Worker tier settings
// DedicatedWorkers is the isolated-egress entitlement, kept under its
// original column name. Read it through IsolatedEgress() rather than
// comparing it directly: the number never meant "how many machines you
// get", only "does this plan reserve egress for you".
DedicatedWorkers int `json:"dedicated_workers"`
DailyCampaignLimit *int `json:"daily_campaign_limit,omitempty"`
@@ -243,3 +247,19 @@ type StripeWebhookEvent struct {
ProcessedAt time.Time `json:"processed_at"`
Payload map[string]interface{} `json:"payload,omitempty"`
}
// IsolatedEgress reports whether the plan reserves a worker for the
// organization, so its mailboxes always authenticate to their providers from
// an address no other tenant sends from.
//
// This is the deliverability shape of what used to be sold as a "dedicated
// worker". The customer-visible promise is about the sign-in address being
// theirs alone, which is the part that actually affects them: providers score
// login trust and apply per-IP auth throttles on that address, so an
// organization running many mailboxes benefits from not sharing it. It was
// never about the machine, and pinning them to one was the wrong shape - a
// reserved worker that dies used to strand the customer, where a preference
// simply re-converges.
func (p *Plan) IsolatedEgress() bool {
return p != nil && p.DedicatedWorkers > 0
}
+24 -87
View File
@@ -7,36 +7,6 @@ import (
"golang.org/x/oauth2"
)
// WorkerType represents the type of worker
type WorkerType string
const (
WorkerTypeShared WorkerType = "shared"
WorkerTypeDedicated WorkerType = "dedicated"
)
// WorkerRiskPool buckets shared workers by acceptable mailbox risk level.
// Dedicated workers don't use it (one customer per worker — no
// cross-tenant contamination risk).
type WorkerRiskPool string
const (
WorkerRiskPoolClean WorkerRiskPool = "clean"
WorkerRiskPoolRisky WorkerRiskPool = "risky"
WorkerRiskPoolQuarantine WorkerRiskPool = "quarantine"
)
// WorkerEgressKind describes how a worker is wired up to actually send mail.
// Different egress profiles ship with very different safe capacities, which
// is why the capacity view branches on this column to derive base_capacity.
type WorkerEgressKind string
const (
WorkerEgressColdSMTP WorkerEgressKind = "cold_smtp"
WorkerEgressOAuthAPI WorkerEgressKind = "oauth_api"
WorkerEgressWarmupOnly WorkerEgressKind = "warmup_only"
)
// WorkerHealthState is the rolled-up health label maintained by the
// assignment loop. Authoritative for "can this worker accept new
// mailboxes" placement decisions. Mirrors the warmup health vocabulary
@@ -51,80 +21,47 @@ const (
WorkerHealthBlocked WorkerHealthState = "blocked"
)
// Worker is a node that carries mailboxes. It is a flat view over
// workers JOIN fleet_nodes: the placement columns (AccountCount, HealthState,
// LoadScore) live on `workers`, and the machine columns below are hydrated
// from the node, because they describe the box rather than the mail on it.
type Worker struct {
ID uuid.UUID `json:"id"`
Name string `json:"name"`
Notes string `json:"notes"`
IPAddr string `json:"ip_addr"`
Active bool `json:"active"`
FreeTier bool `json:"free_tier"`
WorkerType WorkerType `json:"worker_type"`
AccountCount int `json:"account_count"`
RiskPool WorkerRiskPool `json:"risk_pool"`
EgressKind WorkerEgressKind `json:"egress_kind"`
HealthState WorkerHealthState `json:"health_state"`
LoadScore float64 `json:"load_score"`
// SSH management (none of these expose secret material — the encrypted
// private key is fetched separately via GetWorkerSSHCredentials).
SSHHost string `json:"ssh_host,omitempty"`
SSHPort int `json:"ssh_port,omitempty"`
SSHUser string `json:"ssh_user,omitempty"`
SSHPublicKey string `json:"ssh_public_key,omitempty"`
SSHHostFingerprint string `json:"ssh_host_fingerprint,omitempty"`
InstallState WorkerInstallState `json:"install_state"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
LastError string `json:"last_error,omitempty"`
// From the node. Read-only here; a worker never writes them.
Name string `json:"name"`
Notes string `json:"notes"`
IPAddr string `json:"ip_addr"`
Active bool `json:"active"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
LastError string `json:"last_error,omitempty"`
Version string `json:"version,omitempty"`
// Profile assignment. Nil means "use backend env defaults".
ProfileID *uuid.UUID `json:"profile_id,omitempty"`
ConfigAppliedAt *time.Time `json:"config_applied_at,omitempty"`
// Region is a sign-in geography hint for placement, not a partition. A
// mailbox scores better on a worker whose egress geolocates near where
// its provider expects logins from; empty scores neutral.
Region string `json:"region"`
// Image tag the worker is currently running, captured on every successful
// Update. Used for the "v1.2.3 → v1.2.4" badge in the dashboard.
ImageVersion string `json:"image_version,omitempty"`
// Admin-applied free-form tags (eu-west, hetzner, warmup-only, ...).
// Auto-derived "smart" labels (tier:free, pool:risky, state:error) are
// computed client-side from the worker row and never stored here.
// Admin-applied free-form tags (eu-west, spare, ...). Auto-derived
// "smart" labels (health:watch) are computed client-side and never stored.
Tags []string `json:"tags,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// WorkerInstallState mirrors the worker_install_state enum.
type WorkerInstallState string
const (
WorkerInstallStatePending WorkerInstallState = "pending"
WorkerInstallStateProvisioning WorkerInstallState = "provisioning"
WorkerInstallStateInstalled WorkerInstallState = "installed"
WorkerInstallStateError WorkerInstallState = "error"
WorkerInstallStateUninstalling WorkerInstallState = "uninstalling"
WorkerInstallStateUninstalled WorkerInstallState = "uninstalled"
)
// WorkerSSHCredentials carries the encrypted private key alongside the
// connection info. Only the orchestrator should ever fetch this; the field is
// never serialised to admin clients.
type WorkerSSHCredentials struct {
WorkerID uuid.UUID
SSHHost string
SSHPort int
SSHUser string
SSHPublicKey string
SSHPrivateKeyEncrypted string
SSHHostFingerprint string
}
type UpdateWorker struct {
IPAddr *string `json:"ip_addr"`
Active *bool `json:"active"`
WorkerType *WorkerType `json:"worker_type,omitempty"`
Active *bool `json:"active"`
Region *string `json:"region,omitempty"`
}
// DedicatedWorkerAssignment represents a dedicated worker assignment to a user
// DedicatedWorkerAssignment binds an organization entitled to isolated egress
// to a worker. It is a preference the placer converges on, not a hard pin: the
// worker itself carries no category, and if it dies the org's mailboxes place
// normally rather than stranding.
type DedicatedWorkerAssignment struct {
ID uuid.UUID `json:"id"`
WorkerID uuid.UUID `json:"worker_id"`
+220
View File
@@ -0,0 +1,220 @@
// Package nodeagent is the node half of the pull-based fleet.
//
// Every Warmbly process that runs on a machine you own embeds it. The agent
// heartbeats to the control plane, reports what it is running and what it is
// using, and reads back the version it should be running. It never applies an
// update itself: replacing a running container from inside that container is
// how you get a process that cannot finish the job. Instead it writes the
// target where the host-side updater (a systemd timer installed by `warmbly
// join`) can see it, and that restarts the service.
package nodeagent
import (
"bytes"
"context"
"encoding/json"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
// DefaultInterval is used until the control plane says otherwise. The first
// reply carries the server's liveness window and the agent re-paces itself to
// a third of it, so the two can never drift apart.
const DefaultInterval = 90 * time.Second
type Config struct {
NodeID uuid.UUID
Role models.NodeRole
Name string
Region string
Address string
// Version is what this build reports. Empty means "unknown", which the
// control plane must not read as "needs updating".
Version string
BaseURL string
Token string
// TargetVersionPath is where the resolved target is written for the
// host-side updater to read. Empty disables that, which is what you want
// in dev where nothing supervises the process.
TargetVersionPath string
HTTPClient *http.Client
}
type Agent struct {
cfg Config
http *http.Client
started time.Time
// lastErr is reported on the next beat and then cleared, so the dashboard
// shows what went wrong without it sticking forever.
lastErr atomic.Pointer[string]
}
func New(cfg Config) *Agent {
if cfg.HTTPClient == nil {
cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second}
}
return &Agent{cfg: cfg, http: cfg.HTTPClient, started: time.Now()}
}
// ReportError attaches a message to the next heartbeat. Safe from any
// goroutine; the newest message wins.
func (a *Agent) ReportError(msg string) {
a.lastErr.Store(&msg)
}
// Run beats until ctx is cancelled, then sends one farewell beat so the node
// goes inactive immediately rather than staying selectable until it ages out.
//
// A failed beat is logged and retried on the next tick. It is never fatal: a
// node that cannot reach the control plane should keep doing the work it
// already has, not stop.
func (a *Agent) Run(ctx context.Context) {
if a.cfg.BaseURL == "" || a.cfg.Token == "" {
log.Println("nodeagent: no backend URL or token; heartbeats disabled")
return
}
interval := DefaultInterval
if reply := a.beat(ctx, true, false); reply != nil {
interval = paceFrom(reply.LivenessSeconds)
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
// Fresh context: ctx is already cancelled, and without the
// farewell the row stays selectable until the beat ages out, so
// the control plane keeps handing work to a process that has exited.
byeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
a.beat(byeCtx, false, true)
cancel()
return
case <-ticker.C:
if reply := a.beat(ctx, false, false); reply != nil {
if next := paceFrom(reply.LivenessSeconds); next != interval {
interval = next
ticker.Reset(interval)
}
}
}
}
}
// paceFrom beats three times per liveness window, so two lost beats in a row
// still do not look like a dead machine.
func paceFrom(livenessSeconds int) time.Duration {
if livenessSeconds <= 0 {
return DefaultInterval
}
d := time.Duration(livenessSeconds) * time.Second / 3
if d < 15*time.Second {
return 15 * time.Second
}
return d
}
func (a *Agent) beat(ctx context.Context, booted, stopping bool) *models.NodeHeartbeatReply {
beat := models.NodeHeartbeat{
NodeID: a.cfg.NodeID,
Role: a.cfg.Role,
Name: a.cfg.Name,
Region: a.cfg.Region,
Address: a.cfg.Address,
Version: a.cfg.Version,
Usage: sampleUsage(a.started),
Booted: booted,
Stopping: stopping,
}
if p := a.lastErr.Swap(nil); p != nil {
beat.LastError = *p
}
body, err := json.Marshal(beat)
if err != nil {
return nil
}
url := strings.TrimRight(a.cfg.BaseURL, "/") + "/api/v1/internal/fleet/heartbeat"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
log.Println("nodeagent: build heartbeat:", err)
return nil
}
req.Header.Set("Authorization", "Bearer "+a.cfg.Token)
req.Header.Set("Content-Type", "application/json")
resp, err := a.http.Do(req)
if err != nil {
log.Println("nodeagent: heartbeat failed:", err)
return nil
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
log.Println("nodeagent: heartbeat returned status", resp.StatusCode)
return nil
}
var reply models.NodeHeartbeatReply
if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil {
return nil
}
a.applyTarget(reply.DesiredVersion)
return &reply
}
// applyTarget records the version the control plane wants. Writing a file the
// host-side updater polls keeps the decision (control plane) separate from the
// mechanism (systemd timer), which is what lets a node replace itself without
// asking a process to kill and restart its own container.
//
// An empty target is ignored rather than written: "no opinion" must never
// become "downgrade to nothing".
func (a *Agent) applyTarget(version string) {
if version == "" || a.cfg.TargetVersionPath == "" || version == a.cfg.Version {
return
}
dir := filepath.Dir(a.cfg.TargetVersionPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
log.Println("nodeagent: cannot create target dir:", err)
return
}
tmp := a.cfg.TargetVersionPath + ".tmp"
if err := os.WriteFile(tmp, []byte(version+"\n"), 0o644); err != nil {
log.Println("nodeagent: cannot write target version:", err)
return
}
// Rename so the updater never reads a half-written file.
if err := os.Rename(tmp, a.cfg.TargetVersionPath); err != nil {
log.Println("nodeagent: cannot publish target version:", err)
return
}
log.Printf("nodeagent: control plane wants %s (running %s); the host updater will apply it",
version, a.cfg.Version)
}
func sampleUsage(started time.Time) models.NodeUsage {
var m runtime.MemStats
runtime.ReadMemStats(&m)
mem := int(m.Sys / 1024 / 1024)
goroutines := runtime.NumGoroutine()
uptime := int64(time.Since(started).Seconds())
return models.NodeUsage{
MemoryMB: &mem,
Goroutines: &goroutines,
UptimeSeconds: &uptime,
}
}
@@ -1,84 +0,0 @@
package repository
import (
"strings"
"testing"
"github.com/warmbly/warmbly/internal/pkg/encrypt"
)
func testRepo(t *testing.T) *emailRepository {
t.Helper()
enc, err := encrypt.NewEncrypterFromHex(strings.Repeat("ab", 32))
if err != nil {
t.Fatalf("build encrypter: %v", err)
}
return &emailRepository{Encrypt: enc}
}
func TestSealCredentialRoundTrip(t *testing.T) {
r := testRepo(t)
for _, plain := range []string{
"ya29.a0ARGnu0-fake-google-access-token",
"1//0gFAKE_google_refresh_token",
"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.fake.graph-token",
"",
} {
sealed, err := r.sealCredential(plain)
if err != nil {
t.Fatalf("seal %q: %v", plain, err)
}
if plain != "" && sealed == plain {
t.Fatalf("seal %q returned the plaintext unchanged", plain)
}
opened, legacy, err := r.openCredential(sealed)
if err != nil {
t.Fatalf("open %q: %v", plain, err)
}
if legacy {
t.Errorf("open %q reported a sealed value as legacy plaintext", plain)
}
if opened != plain {
t.Errorf("round trip of %q returned %q", plain, opened)
}
}
}
// Rows written before OAuth tokens were sealed must still be readable, and must
// be reported as legacy so the caller re-seals them.
func TestOpenCredentialDetectsLegacyPlaintext(t *testing.T) {
r := testRepo(t)
for _, stored := range []string{
"ya29.a0ARGnu0-fake-google-access-token",
"1//0gFAKE_google_refresh_token",
"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.fake.graph-token",
"seed-fake-access-token",
} {
opened, legacy, err := r.openCredential(stored)
if err != nil {
t.Fatalf("open %q: %v", stored, err)
}
if !legacy {
t.Errorf("open %q was not flagged as legacy plaintext", stored)
}
if opened != stored {
t.Errorf("open %q returned %q, want the stored value verbatim", stored, opened)
}
}
}
// Without CREDENTIALS_ENCRYPTION_KEY the repository must fail closed rather
// than fall back to writing provider tokens in the clear.
func TestSealCredentialFailsClosedWithoutEncrypter(t *testing.T) {
r := &emailRepository{}
if _, err := r.sealCredential("ya29.token"); err == nil {
t.Fatal("sealCredential succeeded with no encrypter configured")
}
if _, _, err := r.openCredential("ya29.token"); err == nil {
t.Fatal("openCredential succeeded with no encrypter configured")
}
}

Some files were not shown because too many files have changed in this diff Show More