diff --git a/.github/aws-deploy/package-lock.json b/.github/aws-deploy/package-lock.json index b2cde4668..a6370dff0 100644 --- a/.github/aws-deploy/package-lock.json +++ b/.github/aws-deploy/package-lock.json @@ -8,7 +8,49 @@ "name": "warmbly-aws-deploy", "version": "1.0.0", "dependencies": { - "@aws-sdk/client-ecs": "3.1155.0" + "@aws-sdk/client-cloudwatch": "3.1155.0", + "@aws-sdk/client-cloudwatch-logs": "3.1155.0", + "@aws-sdk/client-ecs": "3.1155.0", + "@aws-sdk/client-secrets-manager": "3.1155.0" + } + }, + "node_modules/@aws-sdk/client-cloudwatch": { + "version": "3.1155.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1155.0.tgz", + "integrity": "sha512-lIiqQLr6HYQWtaosd5n0Y+FBCawblHaxA3kDl+Wve5n8eRIY1Ru3y4rWqwL27nlanJHQxozVBVHBKV2fXOvqSg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/credential-provider-node": "^3.972.88", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/middleware-compression": "^4.6.2", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudwatch-logs": { + "version": "3.1155.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.1155.0.tgz", + "integrity": "sha512-uAEyhj2FucEVboNvFmQfBg6f7DNikzLqrMlv++MrFdDSPNdTjI8EHHigyaOf94ySfkn+ViX/1dRrN1a5BJV9Qw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/credential-provider-node": "^3.972.88", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" } }, "node_modules/@aws-sdk/client-ecs": { @@ -30,6 +72,25 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-secrets-manager": { + "version": "3.1155.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1155.0.tgz", + "integrity": "sha512-Y3LUtxo/027dxgLm4agqlTGCr1ucK1oSgp4Fv5bDYkahP52Q0qp5be8lVAwn7Lef/jRHGvpV4CwmrY4ZCJTtJQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/credential-provider-node": "^3.972.88", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/core": { "version": "3.978.5", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.5.tgz", @@ -324,6 +385,21 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-compression": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.6.2.tgz", + "integrity": "sha512-Q9d+luiRjyHT6kCL/9NyGpdZJgodh4vtvfHC6H8SqoVKrV2k9RoyI9/IloVfdCYks3/2DIi7BsYYLcda5KZS0A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "fflate": "0.8.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/node-http-handler": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", @@ -370,6 +446,12 @@ "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", "license": "MIT" }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", diff --git a/.github/aws-deploy/package.json b/.github/aws-deploy/package.json index 0e763b29c..b3e414f1c 100644 --- a/.github/aws-deploy/package.json +++ b/.github/aws-deploy/package.json @@ -3,6 +3,9 @@ "private": true, "version": "1.0.0", "dependencies": { - "@aws-sdk/client-ecs": "3.1155.0" + "@aws-sdk/client-cloudwatch": "3.1155.0", + "@aws-sdk/client-cloudwatch-logs": "3.1155.0", + "@aws-sdk/client-ecs": "3.1155.0", + "@aws-sdk/client-secrets-manager": "3.1155.0" } } diff --git a/.github/scripts/aws-deploy.cjs b/.github/scripts/aws-deploy.cjs index e8c49a14e..4706dd96d 100644 --- a/.github/scripts/aws-deploy.cjs +++ b/.github/scripts/aws-deploy.cjs @@ -1,8 +1,26 @@ +// Deploys one published release to the AWS control plane: render every task +// definition from the revision Terraform last registered, swap in the release +// digests, gate on a dependency check and the migration, roll the services in +// waves with a bake, then hand the workers a canary rollout. It never builds, +// never reads a secret value, and restores everything it changed on failure. const { - ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand, + ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand, RegisterTaskDefinitionCommand, RunTaskCommand, DescribeTasksCommand, UpdateServiceCommand, } = require("@aws-sdk/client-ecs"); +const { SecretsManagerClient, DescribeSecretCommand, ListSecretVersionIdsCommand } = require("@aws-sdk/client-secrets-manager"); +const { CloudWatchClient, DescribeAlarmsCommand } = require("@aws-sdk/client-cloudwatch"); +const { CloudWatchLogsClient, GetLogEventsCommand } = require("@aws-sdk/client-cloudwatch-logs"); const { createHash } = require("node:crypto"); +const { readFileSync } = require("node:fs"); + +const SERVICES = ["backend", "consumer", "forms", "tracking", "realtime"]; +const WAVES = [["backend"], ["consumer"], ["forms", "tracking", "realtime"]]; +const MODES = ["deploy", "rollback", "redeploy", "refresh-secrets"]; +const RELEASE_TAG = "warmbly-release"; +const VARIANT_TAG = "warmbly-image-variant"; +const TAG_RE = /^v(\d+)\.(\d+)\.(\d+)$/; +const DIGEST_RE = /^sha256:[a-f0-9]{64}$/; +const VERSION_RE = /^[A-Za-z0-9-]{32,64}$/; function configuration(env) { const cluster = env.AWS_ECS_CLUSTER; @@ -10,22 +28,38 @@ function configuration(env) { if (!match) throw new Error("AWS_ECS_CLUSTER must be a us-east-1 cluster ARN"); const prefix = env.AWS_ECS_SERVICE_PREFIX || match[2]; if (!/^[\w-]+$/.test(prefix)) throw new Error("Invalid service prefix"); - const definitions = {}; - const counts = {}; - for (const name of ["migration", "backend", "consumer"]) { - const value = env[`AWS_${name.toUpperCase()}_TASK_DEFINITION`]; - const expected = `arn:aws:ecs:us-east-1:${match[1]}:task-definition/${prefix}-${name}:`; - if (!value?.startsWith(expected) || !/^\d+$/.test(value.slice(expected.length))) { - throw new Error(`Provide an explicit ${name} task definition revision`); - } - definitions[name] = value; - if (name !== "migration") { - const raw = env[`AWS_${name.toUpperCase()}_DESIRED_COUNT`] || "preserve"; - if (raw !== "preserve" && !/^[0-5]$/.test(raw)) throw new Error("Desired counts must be preserve or 0 through 5"); - counts[name] = raw === "preserve" ? undefined : Number(raw); + const mode = env.DEPLOY_MODE || "deploy"; + if (!MODES.includes(mode)) throw new Error(`DEPLOY_MODE must be one of ${MODES.join(", ")}`); + const tag = (env.DEPLOY_TAG || "").trim(); + if (tag && !TAG_RE.test(tag)) throw new Error("DEPLOY_TAG must be a stable vX.Y.Z tag; prereleases never deploy"); + if (!tag && (mode === "deploy" || mode === "rollback")) throw new Error(`${mode} needs DEPLOY_TAG`); + const raw = (env.DEPLOY_SERVICES || "all").trim(); + const services = raw === "all" ? [...SERVICES] : raw.split(",").map((s) => s.trim()).filter(Boolean); + if (!services.length || services.some((s) => !SERVICES.includes(s))) throw new Error(`DEPLOY_SERVICES must be all or a list of ${SERVICES.join(", ")}`); + let secretVersions = {}; + if (env.DEPLOY_SECRET_VERSIONS) { + secretVersions = JSON.parse(env.DEPLOY_SECRET_VERSIONS); + for (const [k, v] of Object.entries(secretVersions)) { + if (!SERVICES.includes(k) || !VERSION_RE.test(v)) throw new Error("DEPLOY_SECRET_VERSIONS maps a service to a secret version id"); } + if (mode !== "refresh-secrets") throw new Error("DEPLOY_SECRET_VERSIONS is only accepted with refresh-secrets"); } - return { cluster, prefix, definitions, counts, run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}` }; + const bake = Number(env.DEPLOY_BAKE_SECONDS ?? 300); + if (!Number.isInteger(bake) || bake < 0 || bake > 3600) throw new Error("DEPLOY_BAKE_SECONDS must be 0 through 3600"); + return { + cluster, prefix, mode, tag, services, secretVersions, bake, + fleet: env.DEPLOY_FLEET !== "false", + allowDroppedSecrets: env.DEPLOY_ALLOW_DROPPED_SECRETS === "true", + images: env.DEPLOY_IMAGES_JSON ? JSON.parse(readFileSync(env.DEPLOY_IMAGES_JSON, "utf8")) : null, + run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}`, + }; +} + +function compareTags(a, b) { + const x = TAG_RE.exec(a).slice(1).map(Number); + const y = TAG_RE.exec(b).slice(1).map(Number); + for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] - y[i]; + return 0; } function assertResponse(out) { @@ -33,6 +67,141 @@ function assertResponse(out) { return out; } +const tagValue = (tags, key) => tags?.find((t) => t.key === key)?.value; + +// parseRef splits arn:aws:secretsmanager:::secret::::. +function parseRef(valueFrom) { + const parts = (valueFrom || "").split(":"); + if (parts[2] !== "secretsmanager" || parts.length !== 10) return null; + return { arn: parts.slice(0, 7).join(":"), key: parts[7], stage: parts[8], version: parts[9] }; +} + +// secretService names the service a secret belongs to: /-XXXXXX. +function secretService(arn, prefix) { + const name = arn.split(":")[6] || ""; + const m = new RegExp(`^${prefix}/([a-z]+)-[A-Za-z0-9]{6}$`).exec(name); + return m ? m[1] : null; +} + +const REGISTER_FIELDS = [ + "family", "taskRoleArn", "executionRoleArn", "networkMode", "containerDefinitions", "volumes", + "placementConstraints", "requiresCompatibilities", "cpu", "memory", "pidMode", "ipcMode", + "proxyConfiguration", "inferenceAccelerators", "ephemeralStorage", "runtimePlatform", "enableFaultInjection", +]; + +async function describe(client, taskDefinition) { + const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition, include: ["TAGS"] })); + if (!out.taskDefinition || out.taskDefinition.status !== "ACTIVE") throw new Error(`${taskDefinition} is not an active task definition`); + return { definition: out.taskDefinition, tags: out.tags || [] }; +} + +async function resolveVersions(clients, config, refs) { + const pins = {}; + for (const [arn, seen] of Object.entries(refs)) { + const service = secretService(arn, config.prefix); + if (config.secretVersions[service]) { + pins[arn] = config.secretVersions[service]; + continue; + } + // Explicit versions mean a targeted roll; everyone else keeps the newest pin. + if (config.mode === "refresh-secrets" && !Object.keys(config.secretVersions).length) { + const out = await clients.secrets.send(new DescribeSecretCommand({ SecretId: arn })); + const current = Object.entries(out.VersionIdsToStages || {}).find(([, stages]) => stages.includes("AWSCURRENT")); + if (!current) throw new Error(`${service || "a"} secret has no AWSCURRENT version`); + pins[arn] = current[0]; + continue; + } + // Rotation pins newer versions than Terraform's, so the newest pin anyone uses wins. + const out = await clients.secrets.send(new ListSecretVersionIdsCommand({ SecretId: arn, IncludeDeprecated: true, MaxResults: 100 })); + const created = new Map((out.Versions || []).map((v) => [v.VersionId, new Date(v.CreatedDate).getTime()])); + const known = [...seen].filter((v) => created.has(v)); + if (!known.length) throw new Error(`no pinned version of the ${service || ""} secret still exists`); + pins[arn] = known.sort((a, b) => created.get(b) - created.get(a))[0]; + } + return pins; +} + +function repin(container, pins) { + for (const secret of container.secrets || []) { + const ref = parseRef(secret.valueFrom); + if (ref && ref.version && pins[ref.arn]) { + secret.valueFrom = `${ref.arn}:${ref.key}:${ref.stage}:${pins[ref.arn]}`; + } + } +} + +function same(a, b) { + return JSON.stringify(REGISTER_FIELDS.map((f) => a[f] ?? null)) === JSON.stringify(REGISTER_FIELDS.map((f) => b[f] ?? null)); +} + +async function register(client, source, rendered, tags, release, report) { + const carried = tags.filter((t) => t.key !== RELEASE_TAG && !t.key.startsWith("aws:")); + if (release) carried.push({ key: RELEASE_TAG, value: release }); + if (same(source.definition, rendered) && tagValue(source.tags, RELEASE_TAG) === release) return source.definition.taskDefinitionArn; + const input = {}; + for (const f of REGISTER_FIELDS) if (rendered[f] !== undefined && rendered[f] !== null) input[f] = rendered[f]; + input.tags = carried; + const out = await client.send(new RegisterTaskDefinitionCommand(input)); + const arn = out.taskDefinition?.taskDefinitionArn; + if (!arn) throw new Error(`registering ${rendered.family} returned no revision`); + report(`Registered ${arn.split("/").pop()}`); + return arn; +} + +async function logTail(clients, definition, task, report, quiet) { + const container = definition.containerDefinitions[0]; + const options = container.logConfiguration?.options || {}; + if (!options["awslogs-group"] || !options["awslogs-stream-prefix"]) return []; + try { + const out = await clients.logs.send(new GetLogEventsCommand({ + logGroupName: options["awslogs-group"], + logStreamName: `${options["awslogs-stream-prefix"]}/${container.name}/${task.split("/").pop()}`, + startFromHead: false, limit: 60, + })); + const lines = (out.events || []).map((e) => String(e.message).slice(0, 2000)); + if (!quiet) for (const line of lines) report(` | ${line}`); + return lines; + } catch { + report(" | (no log events could be read for this task)"); + return []; + } +} + +// runTask runs a one-off task to completion; capture may be a predicate the log tail must meet. +async function runTask(clients, config, network, arn, definition, label, sleep, report, options = {}) { + const { command, environment, capture } = Array.isArray(options) ? { command: options } : options; + const token = createHash("sha256").update(`${config.run}/${arn}/${label}/${JSON.stringify(command || [])}/${JSON.stringify(environment || [])}`).digest("hex"); + const name = definition.containerDefinitions[0].name; + const override = { name, ...(command ? { command } : {}), ...(environment ? { environment } : {}) }; + const started = assertResponse(await clients.ecs.send(new RunTaskCommand({ + cluster: config.cluster, taskDefinition: arn, launchType: "FARGATE", platformVersion: "1.4.0", count: 1, + networkConfiguration: network, clientToken: token.slice(0, 64), startedBy: `${label.replace(/\W+/g, "-")}-${token.slice(0, 20)}`.slice(0, 128), + ...(command || environment ? { overrides: { containerOverrides: [override] } } : {}), + }))); + const task = started.tasks?.[0]?.taskArn; + if (!task) throw new Error(`${label} task was not created`); + report(`${label} task: ${task.split("/").pop()}`); + for (let attempt = 0; attempt < 240; attempt++) { + const out = assertResponse(await clients.ecs.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] }))); + const status = out.tasks?.[0]; + if (!status) throw new Error(`${label} task is missing`); + if (status.lastStatus === "STOPPED") { + const container = status.containers?.find((c) => c.name === name); + const ok = status.stopCode === "EssentialContainerExited" && container?.exitCode === 0; + let lines = !ok || command || capture ? await logTail(clients, definition, task, report, ok && capture) : []; + if (!ok) throw new Error(`${label} failed; its log tail is above`); + // Log delivery can trail the task stopping by a few seconds. + for (let retry = 0; typeof capture === "function" && !capture(lines) && retry < 6; retry++) { + await sleep(5000); + lines = await logTail(clients, definition, task, report, true); + } + return lines; + } + await sleep(5000); + } + throw new Error(`${label} is unresolved after 20 minutes; no services were updated. Inspect the task before retrying`); +} + async function waitForService(client, config, name, definition, sleep) { for (let attempt = 0; attempt < 180; attempt++) { const out = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] }))); @@ -47,78 +216,196 @@ async function waitForService(client, config, name, definition, sleep) { throw new Error(`${name} deployment did not become healthy within 15 minutes`); } -async function deploy(client, config, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), report = console.log) { - const definitions = {}; - for (const name of ["migration", "backend", "consumer"]) { - const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition: config.definitions[name] })); - const definition = out.taskDefinition; - const container = definition?.containerDefinitions?.find((c) => c.name === name); - if (!container || definition.status !== "ACTIVE" || !/@sha256:[a-f0-9]{64}$/.test(container.image || "")) throw new Error(`Invalid or unpinned ${name} task definition`); - definitions[name] = { definition, container }; +// bake watches a stable service and its alarms; ECS rolling back counts as failure. +async function bake(clients, config, name, definition, sleep, report) { + for (let waited = 0; waited < config.bake; waited += 15) { + const alarms = await clients.cloudwatch.send(new DescribeAlarmsCommand({ AlarmNamePrefix: `${config.prefix}-${name}-`, StateValue: "ALARM" })); + const firing = (alarms.MetricAlarms || []).map((a) => a.AlarmName); + if (firing.length) throw new Error(`${name} raised ${firing.join(", ")} during the bake`); + const out = assertResponse(await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] }))); + if (out.services?.[0]?.taskDefinition !== definition) throw new Error(`${name} was rolled back during the bake`); + await sleep(15000); } - const migration = definitions.migration.container; - if (migration.image !== definitions.backend.container.image || migration.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image"); - const migrationSecret = migration.secrets?.find((s) => s.name === "PRIMARY_DB"); - const directSecret = definitions.backend.container.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT"); - if (!directSecret || directSecret.valueFrom !== migrationSecret?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version"); - if (!definitions.backend.container.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment"); - const current = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`, `${config.prefix}-consumer`] }))); - if (current.services?.length !== 2 || current.services.some((s) => s.status !== "ACTIVE")) throw new Error("Both ECS services must already exist"); - const backend = current.services.find((s) => s.serviceName === `${config.prefix}-backend`); + if (config.bake) report(`${name} baked for ${config.bake}s with no alarm`); +} + +function imageFor(config, key, fallback) { + if (!config.images) return fallback; + const digest = config.images.images?.[key]; + if (!DIGEST_RE.test(digest || "")) throw new Error(`images.json for ${config.tag} has no ${key} digest`); + return `${config.images.registry}/${key.replace(/-kafka$/, "")}@${digest}`; +} + +async function deploy(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log) { + const ecs = clients.ecs; + const names = SERVICES.map((s) => `${config.prefix}-${s}`); + const current = assertResponse(await ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: names }))); + const live = {}; + for (const s of current.services || []) live[s.serviceName.slice(config.prefix.length + 1)] = s; + for (const name of config.services) { + if (live[name]?.status !== "ACTIVE") throw new Error(`${name} must already exist as an ECS service`); + } + const backend = live.backend; if (!backend?.networkConfiguration?.awsvpcConfiguration) throw new Error("Backend task networking is missing"); - const token = createHash("sha256").update(`${config.run}/${config.definitions.migration}`).digest("hex"); - const started = assertResponse(await client.send(new RunTaskCommand({ - cluster: config.cluster, taskDefinition: config.definitions.migration, - launchType: "FARGATE", platformVersion: "1.4.0", count: 1, - networkConfiguration: backend.networkConfiguration, clientToken: token, - startedBy: `migration-${token.slice(0, 24)}`, - }))); - const task = started.tasks?.[0]?.taskArn; - if (!task) throw new Error("Migration task was not created"); - report(`Migration task: ${task}`); - let migrated = false; - for (let attempt = 0; attempt < 240; attempt++) { - const out = assertResponse(await client.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] }))); - const status = out.tasks?.[0]; - if (!status) throw new Error("Migration task is missing"); - if (status.lastStatus === "STOPPED") { - const container = status.containers?.find((c) => c.name === "migration"); - if (status.stopCode !== "EssentialContainerExited" || container?.exitCode !== 0) throw new Error("Migration failed; inspect its CloudWatch logs before retrying"); - migrated = true; - break; - } - await sleep(5000); + const liveDefs = {}; + for (const [name, s] of Object.entries(live)) liveDefs[name] = await describe(ecs, s.taskDefinition); + + const liveTag = tagValue(liveDefs.backend.tags, RELEASE_TAG); + const tag = config.tag || liveTag; + if (!tag) throw new Error("The live backend carries no release tag yet; pass DEPLOY_TAG"); + if (config.tag && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) < 0 && config.mode !== "rollback") { + throw new Error(`${config.tag} is older than the live ${liveTag}; use mode=rollback to go back`); } - if (!migrated) throw new Error("Migration status is unresolved; no services were updated. Inspect the running task before retrying"); - report("Migration completed successfully"); + if (config.mode === "rollback" && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) > 0) { + throw new Error(`rollback target ${config.tag} is newer than the live ${liveTag}`); + } + if (config.images && config.images.tag !== tag) throw new Error(`images.json describes ${config.images.tag}, not ${tag}`); + if (!config.images && config.mode !== "refresh-secrets") throw new Error("DEPLOY_IMAGES_JSON is required unless refreshing secrets"); + report(`Release ${tag} (${config.mode}); live backend reports ${liveTag || "no release"}`); + + const deployBackend = config.services.includes("backend"); + const families = [...config.services]; + const extra = {}; + for (const name of ["migration", "validation", "ops"]) { + if (name === "migration" && (!deployBackend || config.mode === "rollback")) continue; + try { + extra[name] = await describe(ecs, `${config.prefix}-${name}`); + families.push(name); + } catch (error) { + if (name === "migration") throw error; + report(`No ${name} task definition; skipping it`); + } + } + const sources = {}; + for (const name of config.services) sources[name] = await describe(ecs, `${config.prefix}-${name}`); + Object.assign(sources, extra); + + const refs = {}; + const collect = (def) => { + for (const c of def.containerDefinitions || []) for (const s of c.secrets || []) { + const ref = parseRef(s.valueFrom); + if (ref?.version) (refs[ref.arn] ||= new Set()).add(ref.version); + } + }; + for (const name of families) collect(sources[name].definition); + for (const def of Object.values(liveDefs)) collect(def.definition); + const pins = await resolveVersions(clients, config, refs); + + const variant = (name) => { + const v = tagValue(sources[name].tags, VARIANT_TAG); + if (v !== "kafka" && v !== "default") throw new Error(`${config.prefix}-${name} has no ${VARIANT_TAG} tag; apply the runtime Terraform first`); + return v; + }; + const backendKey = deployBackend || families.includes("ops") ? `backend${variant(deployBackend ? "backend" : "ops") === "kafka" ? "-kafka" : ""}` : null; + const rendered = {}; + for (const name of families) { + const def = structuredClone(sources[name].definition); + const container = def.containerDefinitions?.[0]; + if (!container || def.containerDefinitions.length !== 1) throw new Error(`${name} must have exactly one container`); + repin(container, pins); + if (SERVICES.includes(name)) { + if (container.name !== name) throw new Error(`${name} container is named ${container.name}`); + const key = `${name}${variant(name) === "kafka" ? "-kafka" : ""}`; + if (config.images && !container.image.startsWith(`${config.images.registry}/${name}@`)) { + throw new Error(`${name} does not run a digest from ${config.images.registry}; fix its image in Terraform first`); + } + container.image = imageFor(config, key, liveDefs[name].definition.containerDefinitions[0].image); + } else if (name !== "validation") { + container.image = imageFor(config, backendKey, liveDefs.backend.definition.containerDefinitions[0].image); + } + rendered[name] = def; + } + + for (const name of config.services) { + const before = new Set((liveDefs[name].definition.containerDefinitions[0].secrets || []).map((s) => s.name)); + const after = new Set((rendered[name].containerDefinitions[0].secrets || []).map((s) => s.name)); + const dropped = [...before].filter((k) => !after.has(k)); + if (dropped.length && !config.allowDroppedSecrets) { + throw new Error(`${name} would lose ${dropped.join(", ")}; restore them in Terraform or set DEPLOY_ALLOW_DROPPED_SECRETS=true`); + } + } + if (rendered.migration) { + const m = rendered.migration.containerDefinitions[0]; + const b = rendered.backend.containerDefinitions[0]; + if (m.image !== b.image || m.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image"); + const direct = b.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT"); + if (!direct || direct.valueFrom !== m.secrets?.find((s) => s.name === "PRIMARY_DB")?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version"); + if (!b.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment"); + } + + const arns = {}; + for (const name of families) arns[name] = await register(ecs, sources[name], rendered[name], sources[name].tags, tag, report); + + const network = backend.networkConfiguration; + if (arns.validation) await runTask(clients, config, network, arns.validation, rendered.validation, "Dependency check", sleep, report); + if (arns.migration) { + await runTask(clients, config, network, arns.migration, rendered.migration, "Migration", sleep, report); + report("Migration completed successfully"); + } + const updated = []; try { - for (const name of ["backend", "consumer"]) { - const previous = current.services.find((s) => s.serviceName === `${config.prefix}-${name}`); - updated.push(previous); - await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: config.definitions[name], ...(config.counts[name] === undefined ? {} : { desiredCount: config.counts[name] }) })); - await waitForService(client, config, name, config.definitions[name], sleep); - report(`${name} service is stable on the requested revision`); + for (const wave of WAVES) { + const group = wave.filter((n) => config.services.includes(n)); + // allSettled: a sibling still rolling must finish before anything is restored. + const results = await Promise.allSettled(group.map(async (name) => { + const previous = live[name]; + const unchanged = previous.taskDefinition === arns[name]; + if (unchanged && config.mode !== "redeploy") { + report(`${name} already runs ${arns[name].split("/").pop()}`); + return; + } + updated.push(previous); + await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: arns[name], forceNewDeployment: unchanged })); + await waitForService(ecs, config, name, arns[name], sleep); + report(`${name} is stable on ${arns[name].split("/").pop()}`); + await bake(clients, config, name, arns[name], sleep, report); + })); + const failed = results.find((r) => r.status === "rejected"); + if (failed) throw failed.reason; } } catch (error) { for (const previous of updated.reverse()) { try { - await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount })); - report(`Restored previous deployment configuration for ${previous.serviceName}; verify its health`); + await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount })); + report(`Restored ${previous.serviceName} to ${previous.taskDefinition.split("/").pop()}; verify its health`); } catch { report(`Rollback needs operator attention for ${previous.serviceName}`); } } throw error; } + + if (config.fleet && arns.ops && config.images && (config.mode === "deploy" || config.mode === "rollback")) { + const digests = ["worker", "worker-kafka", "consumer", "consumer-kafka"] + .filter((k) => DIGEST_RE.test(config.images.images?.[k] || "")) + .map((k) => `${k}=${config.images.images[k]}`); + const command = ["fleet", "rollout", "start", tag, "--canary", "1", "--waves", "100", "--bake", "30m"]; + if (digests.length) command.push("--digests", digests.join(",")); + await runTask(clients, config, network, arns.ops, rendered.ops, "Fleet rollout", sleep, report, { command }); + report(`Workers move to ${tag} in canary waves; follow with \`warmblyctl fleet rollout status\``); + } + return { tag, arns, pins: Object.fromEntries(Object.entries(pins).map(([arn, v]) => [secretService(arn, config.prefix) || arn, v])) }; } if (require.main === module) { - const client = new ECSClient({ region: "us-east-1", maxAttempts: 2 }); - Promise.resolve().then(() => deploy(client, configuration(process.env))).catch((error) => { - console.error(error?.$metadata ? `AWS request failed: ${error.name}` : error.message); + const region = { region: "us-east-1", maxAttempts: 3 }; + const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) }; + const lines = []; + const report = (line) => { console.log(line); lines.push(line); }; + Promise.resolve().then(() => deploy(clients, configuration(process.env), undefined, report)).then((out) => { + report(`Deployed ${out.tag}. Secret versions: ${Object.entries(out.pins).map(([k, v]) => `${k}=${v.slice(0, 8)}`).join(", ") || "none"}`); + }).catch((error) => { + const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message; + console.error(message); + lines.push(`**Failed:** ${message}`); process.exitCode = 1; - }).finally(() => client.destroy()); + }).finally(() => { + if (process.env.GITHUB_STEP_SUMMARY) { + require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### AWS deploy\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`); + } + for (const c of Object.values(clients)) c.destroy(); + }); } -module.exports = { configuration, deploy }; +module.exports = { configuration, deploy, runTask, describe, compareTags, parseRef, secretService, SERVICES }; diff --git a/.github/scripts/aws-deploy.test.cjs b/.github/scripts/aws-deploy.test.cjs index 71bb9b284..56ec81113 100644 --- a/.github/scripts/aws-deploy.test.cjs +++ b/.github/scripts/aws-deploy.test.cjs @@ -1,89 +1,305 @@ const { test } = require("node:test"); const assert = require("node:assert/strict"); -const { configuration, deploy } = require("./aws-deploy.cjs"); +const { mkdtempSync, writeFileSync } = require("node:fs"); +const { join } = require("node:path"); +const { tmpdir } = require("node:os"); +const { configuration, deploy, compareTags, parseRef } = require("./aws-deploy.cjs"); const root = "arn:aws:ecs:us-east-1:000000000000:"; -const digest = "sha256:" + "a".repeat(64); -const secret = "arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture:PRIMARY_DB_DIRECT::version"; -function config() { +const registry = "ghcr.io/warmbly/warmbly"; +const d = (c) => "sha256:" + c.repeat(64); +const KAFKA = ["backend", "consumer", "tracking", "migration", "ops"]; +const FAMILIES = ["backend", "consumer", "forms", "tracking", "realtime", "migration", "validation", "ops"]; +const secretArn = (svc) => `arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture/${svc}-AbCdEf`; + +function imagesFile(tag = "v1.3.0") { + const dir = mkdtempSync(join(tmpdir(), "aws-deploy-")); + const images = {}; + for (const svc of ["backend", "consumer", "worker", "forms", "tracking", "realtime"]) images[svc] = d("b"); + for (const svc of ["backend", "consumer", "worker", "tracking"]) images[`${svc}-kafka`] = d("c"); + writeFileSync(join(dir, "images.json"), JSON.stringify({ tag, registry, images })); + return join(dir, "images.json"); +} + +function config(env = {}) { return configuration({ - AWS_ECS_CLUSTER: root + "cluster/fixture", - AWS_MIGRATION_TASK_DEFINITION: root + "task-definition/fixture-migration:2", - AWS_BACKEND_TASK_DEFINITION: root + "task-definition/fixture-backend:2", - AWS_CONSUMER_TASK_DEFINITION: root + "task-definition/fixture-consumer:2", + AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_TAG: "v1.3.0", DEPLOY_IMAGES_JSON: imagesFile(env.DEPLOY_TAG || "v1.3.0"), + DEPLOY_BAKE_SECONDS: "30", ...env, }); } +function definition(family, revision, options) { + const secretSvc = ["migration", "validation", "ops"].includes(family) ? "backend" : family; + const version = family === "backend" && revision === 1 ? "rotated" : options.versions?.[family] || "terraform"; + const secrets = family === "migration" + ? [{ name: "PRIMARY_DB", valueFrom: `${secretArn("backend")}:PRIMARY_DB_DIRECT::${version}` }] + : [ + { name: "PRIMARY_DB_DIRECT", valueFrom: `${secretArn(secretSvc)}:PRIMARY_DB_DIRECT::${version}` }, + { name: "REDIS_PASSWORD", valueFrom: `${secretArn(secretSvc)}:REDIS_PASSWORD::` }, + ]; + if (family === "backend" && (revision === 1 || !options.dropReadKeys)) secrets.push({ name: "CREDENTIALS_ENCRYPTION_READ_KEYS", valueFrom: `${secretArn("backend")}:CREDENTIALS_ENCRYPTION_READ_KEYS::${version}` }); + const image = family === "validation" ? "node:22-alpine" + : `${options.registry || registry}/${["migration", "ops"].includes(family) ? "backend" : family}@${d("a")}`; + return { + taskDefinitionArn: `${root}task-definition/fixture-${family}:${revision}`, family: `fixture-${family}`, revision, status: "ACTIVE", + networkMode: "awsvpc", cpu: "256", memory: "512", requiresCompatibilities: ["FARGATE"], + containerDefinitions: [{ + name: family, image, secrets, + entryPoint: family === "migration" ? ["/app/migrate"] : undefined, + environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }], + logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": "/fixture", "awslogs-stream-prefix": family } }, + }], + }; +} + function fixture(options = {}) { const calls = []; - const services = ["backend", "consumer"].map((name) => ({ - serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 1, runningCount: 1, pendingCount: 0, - taskDefinition: root + `task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }], - networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "ENABLED" } }, - })); - const client = { async send(command) { - const name = command.constructor.name; - calls.push({ name, input: command.input }); - if (name === "DescribeTaskDefinitionCommand") { - const service = /fixture-(\w+):/.exec(command.input.taskDefinition)[1]; - return { taskDefinition: { status: "ACTIVE", containerDefinitions: [{ - name: service, image: `registry/${service === "migration" ? "backend" : service}@${service === "migration" && options.imageMismatch ? "sha256:" + "b".repeat(64) : digest}`, - entryPoint: service === "migration" ? ["/app/migrate"] : undefined, - environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }], - secrets: [{ name: service === "migration" ? "PRIMARY_DB" : "PRIMARY_DB_DIRECT", valueFrom: secret }], - }] } }; + const defs = {}; + const tagsOf = {}; + for (const family of FAMILIES) { + for (const revision of [1, 2]) { + const arn = `${root}task-definition/fixture-${family}:${revision}`; + defs[arn] = definition(family, revision, options); + const tags = [{ key: "Environment", value: "production" }]; + if (!(options.noVariant && family === "backend" && revision === 2)) tags.push({ key: "warmbly-image-variant", value: KAFKA.includes(family) ? "kafka" : "default" }); + if (revision === 1) tags.push({ key: "warmbly-release", value: options.liveTag || "v1.2.0" }); + tagsOf[arn] = tags; } - if (name === "DescribeServicesCommand") return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) }; - if (name === "RunTaskCommand") return options.placementFailure ? { failures: [{ reason: "fixture" }] } : { tasks: [{ taskArn: root + "task/fixture/migration" }] }; - if (name === "DescribeTasksCommand") return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: "migration", exitCode: options.migrationFailure ? 1 : 0 }] }] }; - if (name === "UpdateServiceCommand") { - if (command.input.service === "fixture-consumer" && command.input.taskDefinition.endsWith(":2") && options.consumerFailure) throw new Error("consumer update failed"); + } + const latest = (family) => Object.values(defs).filter((x) => x.family === `fixture-${family}`).sort((a, b) => b.revision - a.revision)[0]; + const services = ["backend", "consumer", "forms", "tracking", "realtime"].map((name) => ({ + serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 2, runningCount: 2, pendingCount: 0, + taskDefinition: `${root}task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }], + networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "DISABLED" } }, + })); + const tasks = {}; + const record = (client) => ({ async send(command) { + const name = command.constructor.name; + calls.push({ client, name, input: structuredClone(command.input) }); + switch (name) { + case "DescribeTaskDefinitionCommand": { + const ref = command.input.taskDefinition; + const def = ref.startsWith("arn:") ? defs[ref] : latest(ref.replace(/^fixture-/, "")); + if (!def) throw new Error(`no task definition ${ref}`); + return { taskDefinition: structuredClone(def), tags: structuredClone(tagsOf[def.taskDefinitionArn]) }; + } + case "RegisterTaskDefinitionCommand": { + const prev = latest(command.input.family.replace(/^fixture-/, "")); + const arn = `${root}task-definition/${command.input.family}:${prev.revision + 1}`; + defs[arn] = { ...structuredClone(command.input), revision: prev.revision + 1, taskDefinitionArn: arn, status: "ACTIVE" }; + delete defs[arn].tags; + tagsOf[arn] = command.input.tags; + return { taskDefinition: { taskDefinitionArn: arn } }; + } + case "DescribeServicesCommand": + return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) }; + case "UpdateServiceCommand": { const service = services.find((s) => s.serviceName === command.input.service); + const svc = service.serviceName.replace("fixture-", ""); + if (options.updateFailure === svc && !command.input.taskDefinition.endsWith(":1")) throw new Error(`${svc} update failed`); service.taskDefinition = command.input.taskDefinition; - if (command.input.desiredCount !== undefined) service.runningCount = service.desiredCount = command.input.desiredCount; - if (options.circuitBreakerRollback && command.input.taskDefinition.endsWith(":2")) service.taskDefinition = root + "task-definition/fixture-backend:1"; + if (command.input.desiredCount !== undefined) service.desiredCount = command.input.desiredCount; + service.deployments = [{ rolloutState: options.rolloutFailure === svc && !command.input.taskDefinition.endsWith(":1") ? "FAILED" : "COMPLETED" }]; + if (options.circuitBreaker === svc) service.taskDefinition = `${root}task-definition/fixture-${svc}:1`; return {}; } - throw new Error(`Unexpected command ${name}`); - } }; - return { client, calls }; + case "RunTaskCommand": { + if (options.placementFailure) return { failures: [{ reason: "fixture" }] }; + const family = /fixture-(\w+):/.exec(command.input.taskDefinition)[1]; + const arn = `${root}task/fixture/${family}-${Object.keys(tasks).length}`; + tasks[arn] = family; + return { tasks: [{ taskArn: arn }] }; + } + case "DescribeTasksCommand": { + const family = tasks[command.input.tasks[0]]; + const exitCode = options.taskFailure === family ? 1 : 0; + return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: family, exitCode }] }] }; + } + case "GetLogEventsCommand": + return { events: [{ message: `log line from ${command.input.logStreamName}` }] }; + case "DescribeAlarmsCommand": { + const firing = options.alarm && command.input.AlarmNamePrefix === `fixture-${options.alarm}-`; + return { MetricAlarms: firing ? [{ AlarmName: `fixture-${options.alarm}-5xx` }] : [] }; + } + case "ListSecretVersionIdsCommand": + return { Versions: [ + { VersionId: "terraform", CreatedDate: new Date("2026-01-01") }, + { VersionId: "rotated", CreatedDate: new Date("2026-02-01") }, + { VersionId: "stale", CreatedDate: new Date("2025-01-01") }, + ] }; + case "DescribeSecretCommand": + return { VersionIdsToStages: { terraform: ["AWSPREVIOUS"], "current-version-id-000000000000000000": ["AWSCURRENT"] } }; + default: + throw new Error(`Unexpected command ${name}`); + } + } }); + const clients = { ecs: record("ecs"), secrets: record("secrets"), cloudwatch: record("cloudwatch"), logs: record("logs") }; + return { clients, calls, defs, tagsOf, services }; } -test("migration success gates both service updates and preserves desired counts", async () => { - const { client, calls } = fixture(); - await deploy(client, config(), async () => {}, () => {}); - const migration = calls.findIndex((c) => c.name === "DescribeTasksCommand"); - const updates = calls.filter((c) => c.name === "UpdateServiceCommand"); - assert.equal(updates.length, 2); - assert.ok(calls.indexOf(updates[0]) > migration); - assert.ok(updates.every((c) => !Object.hasOwn(c.input, "desiredCount"))); - assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1); +const run = (f, cfg, lines = []) => deploy(f.clients, cfg, async () => {}, (l) => lines.push(l)); +const registered = (f) => f.calls.filter((c) => c.name === "RegisterTaskDefinitionCommand").map((c) => c.input); +const updates = (f) => f.calls.filter((c) => c.name === "UpdateServiceCommand").map((c) => c.input); +const runs = (f) => f.calls.filter((c) => c.name === "RunTaskCommand").map((c) => c.input); +const valueFrom = (def, key) => def.containerDefinitions[0].secrets.find((s) => s.name === key).valueFrom; + +test("configuration refuses prereleases, bad modes and secret pins outside refresh-secrets", () => { + assert.throws(() => config({ DEPLOY_TAG: "v1.3.0-rc.1" }), /prereleases never deploy/); + assert.throws(() => config({ DEPLOY_MODE: "yolo" }), /DEPLOY_MODE/); + assert.throws(() => config({ DEPLOY_SERVICES: "backend,worker" }), /DEPLOY_SERVICES/); + assert.throws(() => config({ DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "a".repeat(36) }) }), /refresh-secrets/); + assert.throws(() => config({ DEPLOY_MODE: "refresh-secrets", DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "x" }) }), /secret version id/); + assert.ok(compareTags("v1.10.0", "v1.9.9") > 0); + assert.deepEqual(parseRef(`${secretArn("backend")}:K::v`), { arn: secretArn("backend"), key: "K", stage: "", version: "v" }); }); -for (const failure of ["migrationFailure", "placementFailure", "imageMismatch"]) { - test(`${failure} leaves existing services untouched`, async () => { - const { client, calls } = fixture({ [failure]: true }); - await assert.rejects(deploy(client, config(), async () => {}, () => {})); - assert.ok(calls.every((c) => c.name !== "UpdateServiceCommand")); +test("a release renders from the latest revision, swaps variant digests and carries tags", async () => { + const f = fixture(); + await run(f, config()); + const byFamily = Object.fromEntries(registered(f).map((r) => [r.family.replace("fixture-", ""), r])); + assert.equal(byFamily.backend.containerDefinitions[0].image, `${registry}/backend@${d("c")}`); + assert.equal(byFamily.forms.containerDefinitions[0].image, `${registry}/forms@${d("b")}`); + assert.equal(byFamily.tracking.containerDefinitions[0].image, `${registry}/tracking@${d("c")}`); + assert.equal(byFamily.migration.containerDefinitions[0].image, `${registry}/backend@${d("c")}`); + assert.equal(byFamily.ops.containerDefinitions[0].image, `${registry}/backend@${d("c")}`); + assert.equal(byFamily.validation.containerDefinitions[0].image, "node:22-alpine"); + for (const r of Object.values(byFamily)) { + assert.ok(r.tags.some((t) => t.key === "Environment")); + assert.deepEqual(r.tags.filter((t) => t.key === "warmbly-release"), [{ key: "warmbly-release", value: "v1.3.0" }]); + assert.equal(r.cpu, "256"); + } +}); + +test("the newest pinned secret version wins so a stale Terraform pin cannot undo a rotation", async () => { + const f = fixture(); + await run(f, config()); + const backend = registered(f).find((r) => r.family === "fixture-backend"); + const migration = registered(f).find((r) => r.family === "fixture-migration"); + assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::rotated")); + assert.ok(valueFrom(migration, "PRIMARY_DB").endsWith("::rotated")); + assert.ok(valueFrom(backend, "REDIS_PASSWORD").endsWith("REDIS_PASSWORD::")); + const consumer = registered(f).find((r) => r.family === "fixture-consumer"); + assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::terraform")); +}); + +test("refresh-secrets pins every secret to AWSCURRENT and keeps the live images", async () => { + const f = fixture(); + await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0" })); + const backend = registered(f).find((r) => r.family === "fixture-backend"); + const consumer = registered(f).find((r) => r.family === "fixture-consumer"); + assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000")); + assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000")); + assert.equal(backend.containerDefinitions[0].image, `${registry}/backend@${d("a")}`); + assert.ok(backend.tags.some((t) => t.key === "warmbly-release" && t.value === "v1.2.0")); + assert.ok(!runs(f).some((r) => r.overrides), "refresh-secrets starts no fleet rollout"); +}); + +test("explicit secret versions roll only what they name; other secrets keep their newest pin", async () => { + const f = fixture(); + await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0", + DEPLOY_SERVICES: "consumer", DEPLOY_SECRET_VERSIONS: JSON.stringify({ consumer: "explicit-version-id-0000000000000000" }) })); + const consumer = registered(f).find((r) => r.family === "fixture-consumer"); + const validation = registered(f).find((r) => r.family === "fixture-validation"); + assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::explicit-version-id-0000000000000000")); + assert.ok(valueFrom(validation, "PRIMARY_DB_DIRECT").endsWith("::rotated")); + assert.ok(!f.calls.some((c) => c.name === "DescribeSecretCommand")); + assert.deepEqual(updates(f).map((u) => u.service), ["fixture-consumer"]); +}); + +test("validation and migration gate every service update", async () => { + const f = fixture(); + await run(f, config()); + const firstUpdate = f.calls.findIndex((c) => c.name === "UpdateServiceCommand"); + const taskRuns = f.calls.map((c, i) => [c, i]).filter(([c]) => c.name === "RunTaskCommand"); + assert.deepEqual(taskRuns.slice(0, 2).map(([c]) => /fixture-(\w+):/.exec(c.input.taskDefinition)[1]), ["validation", "migration"]); + assert.ok(taskRuns[1][1] < firstUpdate); + assert.deepEqual(updates(f).map((u) => u.service), ["fixture-backend", "fixture-consumer", "fixture-forms", "fixture-tracking", "fixture-realtime"]); + assert.ok(updates(f).every((u) => !Object.hasOwn(u, "desiredCount"))); +}); + +for (const [failure, message] of [ + [{ taskFailure: "migration" }, /Migration failed/], [{ taskFailure: "validation" }, /Dependency check failed/], + [{ placementFailure: true }, /task-placement failure/], [{ noVariant: true }, /no warmbly-image-variant tag/], + [{ registry: "docker.io/someone" }, /does not run a digest from ghcr/], +]) { + test(`${JSON.stringify(failure)} leaves every service untouched`, async () => { + const f = fixture(failure); + await assert.rejects(run(f, config()), message); + assert.equal(updates(f).length, 0); }); } -test("consumer rollout failure restores previous service revisions without database rollback", async () => { - const { client, calls } = fixture({ consumerFailure: true }); - await assert.rejects(deploy(client, config(), async () => {}, () => {}), /consumer update failed/); - const updates = calls.filter((c) => c.name === "UpdateServiceCommand"); - assert.equal(updates.length, 4); - assert.ok(updates.slice(2).every((c) => c.input.taskDefinition.endsWith(":1") && c.input.desiredCount === 1)); - assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1); +test("a failed migration prints its log tail", async () => { + const f = fixture({ taskFailure: "migration" }); + const lines = []; + await assert.rejects(run(f, config(), lines), /Migration failed/); + assert.ok(lines.some((l) => l.includes("log line from migration/migration/"))); +}); + +test("an older tag is refused unless the mode is rollback, and rollback skips the migration", async () => { + await assert.rejects(run(fixture({ liveTag: "v1.4.0" }), config()), /older than the live v1.4.0/); + await assert.rejects(run(fixture(), config({ DEPLOY_MODE: "rollback" })), /newer than the live/); + const f = fixture({ liveTag: "v1.4.0" }); + await run(f, config({ DEPLOY_MODE: "rollback" })); + assert.ok(!runs(f).some((r) => r.taskDefinition.includes("fixture-migration"))); + assert.ok(runs(f).some((r) => r.overrides), "rollback still moves the fleet"); +}); + +test("a revision that drops a secret the live service reads is refused", async () => { + const f = fixture({ dropReadKeys: true }); + await assert.rejects(run(f, config()), /would lose CREDENTIALS_ENCRYPTION_READ_KEYS/); + assert.equal(registered(f).length, 0); + const g = fixture({ dropReadKeys: true }); + await run(g, config({ DEPLOY_ALLOW_DROPPED_SECRETS: "true" })); +}); + +test("an alarm during the bake restores every updated service in reverse order", async () => { + const f = fixture({ alarm: "consumer" }); + await assert.rejects(run(f, config()), /fixture-consumer-5xx/); + const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")); + assert.deepEqual(restores.map((u) => u.service), ["fixture-consumer", "fixture-backend"]); + assert.ok(restores.every((u) => u.desiredCount === 2)); + assert.ok(!updates(f).some((u) => u.service === "fixture-forms")); +}); + +test("a failure in the parallel wave waits for its siblings and then restores all of them", async () => { + const f = fixture({ rolloutFailure: "forms" }); + await assert.rejects(run(f, config()), /forms deployment failed/); + const forward = updates(f).filter((u) => !u.taskDefinition.endsWith(":1")).map((u) => u.service); + const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")).map((u) => u.service); + assert.equal(forward.length, 5); + assert.deepEqual([...restores].sort(), [...forward].sort()); + assert.deepEqual(restores.slice(-2), ["fixture-consumer", "fixture-backend"]); }); test("a circuit breaker rollback cannot be reported as a successful deployment", async () => { - const { client } = fixture({ circuitBreakerRollback: true }); - await assert.rejects(deploy(client, config(), async () => {}, () => {}), /rolled back/); + const f = fixture({ circuitBreaker: "backend" }); + await assert.rejects(run(f, config()), /backend deployment rolled back/); }); -test("deployment requires explicit revisions in the selected account and region", () => { - assert.throws(() => configuration({ AWS_ECS_CLUSTER: root.replace("us-east-1", "eu-west-1") + "cluster/fixture" })); - const c = config(); - assert.equal(c.counts.backend, undefined); +test("the fleet rollout gets the worker and consumer digests per variant", async () => { + const f = fixture(); + await run(f, config()); + const ops = runs(f).find((r) => r.overrides); + const command = ops.overrides.containerOverrides[0].command; + assert.deepEqual(command.slice(0, 4), ["fleet", "rollout", "start", "v1.3.0"]); + assert.equal(command[command.indexOf("--digests") + 1], `worker=${d("b")},worker-kafka=${d("c")},consumer=${d("b")},consumer-kafka=${d("c")}`); + assert.equal(ops.overrides.containerOverrides[0].name, "ops"); +}); + +test("an unchanged definition reuses its revision and redeploy forces a fresh deployment", async () => { + const f = fixture(); + await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_FLEET: "false" })); + const formsArn = updates(f)[0].taskDefinition; + f.services.find((s) => s.serviceName === "fixture-forms").taskDefinition = formsArn; + const before = registered(f).length; + await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_MODE: "redeploy", DEPLOY_FLEET: "false" })); + assert.equal(registered(f).length, before); + assert.deepEqual(updates(f).at(-1), { cluster: root + "cluster/fixture", service: "fixture-forms", taskDefinition: formsArn, forceNewDeployment: true }); +}); + +test("no secret value or version-free reference is ever read", async () => { + const f = fixture(); + await run(f, config()); + assert.ok(!f.calls.some((c) => c.name === "GetSecretValueCommand")); }); diff --git a/.github/scripts/aws-rotate.cjs b/.github/scripts/aws-rotate.cjs new file mode 100644 index 000000000..1492cbd92 --- /dev/null +++ b/.github/scripts/aws-rotate.cjs @@ -0,0 +1,188 @@ +// Drives a key rotation from GitHub without ever holding a value: the rotator +// task writes new secret versions inside AWS and reports their ids, the deploy +// engine rolls services onto exactly those versions, and warmblyctl in the ops +// task proves each step is safe before the old value is retired. +const { DescribeServicesCommand } = require("@aws-sdk/client-ecs"); +const engine = require("./aws-deploy.cjs"); + +const { configuration, SERVICES } = engine; + +const KINDS = { + CREDENTIALS_ENCRYPTION_KEY: "seal", + TWOFA_SECRET: "seal", + UNSUBSCRIBE_SIGNING_SECRET: "sign", + INTERNAL_API_TOKEN: "token", + NODE_BROKER_TOKEN: "token", +}; +const ACTIONS = ["status", "prepare", "rotate", "retire", "abort"]; + +function rotationConfig(env) { + const action = env.ROTATE_ACTION; + const key = env.ROTATE_KEY; + if (!ACTIONS.includes(action)) throw new Error(`ROTATE_ACTION must be one of ${ACTIONS.join(", ")}`); + if (!KINDS[key]) throw new Error(`ROTATE_KEY must be one of ${Object.keys(KINDS).join(", ")}`); + const wait = Number(env.ROTATE_NODE_WAIT_MINUTES ?? 30); + if (!Number.isInteger(wait) || wait < 0 || wait > 240) throw new Error("ROTATE_NODE_WAIT_MINUTES must be 0 through 240"); + const base = configuration({ ...env, DEPLOY_MODE: "refresh-secrets", DEPLOY_TAG: "", DEPLOY_SERVICES: "all", DEPLOY_SECRET_VERSIONS: "", DEPLOY_IMAGES_JSON: "", DEPLOY_FLEET: "false" }); + return { ...base, action, key, kind: KINDS[key], nodeWaitMs: wait * 60000 }; +} + +async function rotateKeys(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log, deps = engine) { + const { deploy, runTask, describe } = deps; + // ECS run tokens are idempotent, so every task and roll in one run needs its own. + let seq = 0; + const step = () => `${config.run}/${++seq}`; + const services = await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`] })); + const network = services.services?.[0]?.networkConfiguration; + if (!network?.awsvpcConfiguration) throw new Error("Backend task networking is missing"); + + const rotator = async (action) => { + const { definition } = await describe(clients.ecs, `${config.prefix}-rotator`); + const environment = [ + { name: "ROTATE_ACTION", value: action }, { name: "ROTATE_KEY", value: config.key }, { name: "ROTATE_RUN", value: `${step()}/${action}` }, + ]; + const lines = await runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, `Rotator ${action}`, sleep, report, { environment, capture: (ls) => ls.some((l) => l.startsWith("ROTATION_RESULT ")) }); + const line = lines.filter((l) => l.startsWith("ROTATION_RESULT ")).pop(); + if (!line) throw new Error(`the rotator reported no result for ${action}`); + const result = JSON.parse(line.slice("ROTATION_RESULT ".length)); + for (const l of lines) if (!l.startsWith("ROTATION_RESULT ")) report(` rotator: ${l}`); + return result; + }; + const ops = async (label, command) => { + const { definition } = await describe(clients.ecs, `${config.prefix}-ops`); + return runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, label, sleep, report, { command }); + }; + const roll = async (versions) => { + const names = Object.keys(versions || {}).filter((s) => SERVICES.includes(s)); + if (!names.length) return; + report(`Rolling ${names.join(", ")} onto the new secret versions`); + await deploy(clients, { ...config, run: step(), services: names, secretVersions: versions }, sleep, report); + }; + const nodesCurrent = async () => { + const checks = Math.floor(config.nodeWaitMs / 120000) + 1; + for (let i = 1; ; i++) { + try { + await ops("Fleet env status", ["fleet", "env-status"]); + report("Every live node has applied the new env"); + return true; + } catch { + if (i >= checks) return false; + report("Some nodes have not applied the new env yet; checking again in two minutes"); + await sleep(120000); + } + } + }; + const resealed = async () => { + await ops("Reseal", ["keys", "reseal", "--apply"]); + try { + await ops("Key status", ["keys", "status", "--require-primary"]); + return true; + } catch { + return false; + } + }; + const retire = async () => { + const result = await rotator("retire"); + await roll(result.versions); + report(`${config.key} rotated; the previous value is retired`); + }; + // Nodes first: a node still on the old env would seal under the old key after the reseal. + const gate = async () => { + if ((config.kind === "token" || config.key === "CREDENTIALS_ENCRYPTION_KEY") && !(await nodesCurrent())) { + return "some nodes have not applied the new env and still work on the previous value. Run retire once `warmblyctl fleet env-status` is clean"; + } + if (config.kind === "seal" && !(await resealed())) return `some values still open only under the previous ${config.key}; run retire once \`warmblyctl keys status\` is clean`; + return null; + }; + + const status = await rotator("status"); + if (status.key && status.key !== config.key) throw new Error(`${status.key} is mid-rotation; finish or abort it first`); + let phase = status.key === config.key ? status.phase : "none"; + report(`${config.key}: ${phase}`); + // A run that died after writing resumes here: rolling versions already live is a no-op. + const resume = () => roll(status.pinned); + + switch (config.action) { + case "status": + if (config.kind === "seal") await ops("Key status", ["keys", "status"]).catch(() => report("keys status reports unreadable values")); + return { phase }; + case "prepare": { + const result = await rotator("prepare"); + await roll(result.versions); + report(`Overlap keys exist for ${config.key}. Add them to secret_keys for ${result.holders.join(", ")} and apply the runtime Terraform before rotating.`); + return { phase }; + } + case "abort": { + if (phase === "none") throw new Error(`no rotation of ${config.key} is in progress`); + const result = await rotator("abort"); + await roll(result.versions); + report(`${config.key} restored to its previous value`); + return { phase: "none" }; + } + case "retire": { + if (phase !== "staged") throw new Error(`${config.key} has no staged rotation to retire`); + await resume(); + const blocked = await gate(); + if (blocked) throw new Error(blocked); + await retire(); + return { phase: "none" }; + } + case "rotate": { + if (phase !== "none") await resume(); + if (phase === "staged" && config.kind === "sign") { + report(`${config.key} is staged; run retire after ${status.until}`); + return { phase }; + } + if (phase === "none") { + const result = await rotator("stage"); + await roll(result.versions); + phase = result.phase; + } + if (phase === "introduced") { + if (!(await nodesCurrent())) { + report("The new key is readable everywhere except nodes that have not refreshed; run rotate again to make it primary"); + return { phase }; + } + const result = await rotator("stage"); + await roll(result.versions); + phase = result.phase; + } + if (config.kind === "sign") { + report(`${config.key} rotated. Links signed with the previous value keep working for a year, the life of a link; run retire after that`); + return { phase }; + } + const blocked = await gate(); + if (blocked) { + report(`${config.key} is staged: ${blocked}`); + return { phase }; + } + await retire(); + return { phase: "none" }; + } + } + return { phase }; +} + +if (require.main === module) { + const { ECSClient } = require("@aws-sdk/client-ecs"); + const { SecretsManagerClient } = require("@aws-sdk/client-secrets-manager"); + const { CloudWatchClient } = require("@aws-sdk/client-cloudwatch"); + const { CloudWatchLogsClient } = require("@aws-sdk/client-cloudwatch-logs"); + const region = { region: "us-east-1", maxAttempts: 3 }; + const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) }; + const lines = []; + const report = (line) => { console.log(line); lines.push(line); }; + Promise.resolve().then(() => rotateKeys(clients, rotationConfig(process.env), undefined, report)).catch((error) => { + const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message; + console.error(message); + lines.push(`**Failed:** ${message}`); + process.exitCode = 1; + }).finally(() => { + if (process.env.GITHUB_STEP_SUMMARY) { + require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Key rotation\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`); + } + for (const c of Object.values(clients)) c.destroy(); + }); +} + +module.exports = { rotationConfig, rotateKeys }; diff --git a/.github/scripts/aws-rotate.test.cjs b/.github/scripts/aws-rotate.test.cjs new file mode 100644 index 000000000..f1290b2f0 --- /dev/null +++ b/.github/scripts/aws-rotate.test.cjs @@ -0,0 +1,153 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const { rotationConfig, rotateKeys } = require("./aws-rotate.cjs"); + +const CLUSTER = "arn:aws:ecs:us-east-1:000000000000:cluster/fixture"; +const V = (c) => c.repeat(32); + +// world plays the rotator, the ops task and the deploy engine against a phase machine. +function world({ key = "INTERNAL_API_TOKEN", action = "rotate", state = {}, envLagging = 0, unsealed = false, holders } = {}) { + const config = rotationConfig({ AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: action, ROTATE_KEY: key, ROTATE_NODE_WAIT_MINUTES: "4", GITHUB_RUN_ID: "9", GITHUB_RUN_ATTEMPT: "1" }); + const log = []; + const tokens = new Set(); + let lag = envLagging; + const owners = holders || (key.startsWith("CREDENTIALS") ? ["backend", "consumer"] : ["backend", "tracking"]); + const versions = () => Object.fromEntries(owners.map((s, i) => [s, V(String(i + 1))])); + const rotator = (act) => { + log.push(`rotator ${act}`); + if (act === "status") return { phase: state.phase || "none", key: state.key || null, versions: {}, pinned: state.key ? versions() : {}, until: state.until || null }; + if (act === "stage") { + state = key === "CREDENTIALS_ENCRYPTION_KEY" && !state.key ? { key, phase: "introduced" } : { key, phase: "staged" }; + return { phase: state.phase, versions: versions(), holders: owners }; + } + state = {}; + return { phase: "none", versions: act === "prepare" ? { backend: V("p") } : versions(), holders: owners }; + }; + const deps = { + async describe(_, family) { + return { definition: { taskDefinitionArn: `${family}:1`, family, containerDefinitions: [{ name: family.split("-").pop() }] } }; + }, + async runTask(_, cfg, network, arn, definition, label, sleep, report, options) { + assert.ok(network.awsvpcConfiguration); + const token = `${cfg.run}/${label}`; + assert.ok(!tokens.has(token), `run token reused for ${label}`); + tokens.add(token); + if (arn.startsWith("fixture-rotator")) { + const act = options.environment.find((e) => e.name === "ROTATE_ACTION").value; + assert.equal(options.environment.find((e) => e.name === "ROTATE_KEY").value, key); + const lines = ["phase written", `ROTATION_RESULT ${JSON.stringify(rotator(act))}`]; + assert.ok(options.capture(lines) && !options.capture(["phase written"])); + return lines; + } + const command = options.command.join(" "); + log.push(`ops ${command}`); + if (command === "fleet env-status" && lag-- > 0) throw new Error("not current"); + if (command === "keys status --require-primary" && unsealed) throw new Error("on read key"); + return []; + }, + async deploy(_, cfg) { + assert.equal(cfg.mode, "refresh-secrets"); + assert.equal(cfg.fleet, false); + assert.equal(cfg.tag, ""); + log.push(`roll ${cfg.services.join(",")}`); + return {}; + }, + }; + const clients = { ecs: { async send() { + return { services: [{ networkConfiguration: { awsvpcConfiguration: { subnets: ["s"] } } }] }; + } } }; + const report = []; + const run = () => rotateKeys(clients, config, async () => {}, (l) => report.push(l), deps); + return { run, log, report, state: () => state }; +} + +test("configuration refuses unknown keys, actions and waits", () => { + const base = { AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: "rotate", ROTATE_KEY: "INTERNAL_API_TOKEN" }; + assert.throws(() => rotationConfig({ ...base, ROTATE_KEY: "AUTH_SECRET" }), /ROTATE_KEY/); + assert.throws(() => rotationConfig({ ...base, ROTATE_ACTION: "delete" }), /ROTATE_ACTION/); + assert.throws(() => rotationConfig({ ...base, ROTATE_NODE_WAIT_MINUTES: "999" }), /0 through 240/); + assert.equal(rotationConfig(base).kind, "token"); +}); + +test("a token is staged receiver first, waits for nodes, then retires in one run", async () => { + const w = world({ envLagging: 1 }); + assert.deepEqual(await w.run(), { phase: "none" }); + assert.deepEqual(w.log, [ + "rotator status", "rotator stage", "roll backend,tracking", + "ops fleet env-status", "ops fleet env-status", "rotator retire", "roll backend,tracking", + ]); +}); + +test("a token stays staged when nodes never catch up, and says how to finish", async () => { + const w = world({ envLagging: 99 }); + assert.deepEqual(await w.run(), { phase: "staged" }); + assert.ok(!w.log.includes("rotator retire")); + assert.equal(w.log.filter((l) => l === "ops fleet env-status").length, 3); + assert.match(w.report.at(-1), /is staged: .*fleet env-status/); +}); + +test("the credentials key is introduced, made primary once nodes read it, resealed, then retired", async () => { + const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY" }); + assert.deepEqual(await w.run(), { phase: "none" }); + assert.deepEqual(w.log, [ + "rotator status", "rotator stage", "roll backend,consumer", + "ops fleet env-status", "rotator stage", "roll backend,consumer", + "ops fleet env-status", "ops keys reseal --apply", "ops keys status --require-primary", + "rotator retire", "roll backend,consumer", + ]); +}); + +test("the credentials key waits at introduced when a node has not refreshed", async () => { + const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY", envLagging: 99 }); + assert.deepEqual(await w.run(), { phase: "introduced" }); + assert.equal(w.log.filter((l) => l === "rotator stage").length, 1); +}); + +test("a seal key is not retired while values still need the old key", async () => { + const w = world({ key: "TWOFA_SECRET", unsealed: true, holders: ["backend"] }); + assert.deepEqual(await w.run(), { phase: "staged" }); + assert.ok(!w.log.includes("ops fleet env-status")); + assert.ok(!w.log.includes("rotator retire")); + assert.match(w.report.at(-1), /keys status/); +}); + +test("a signing secret stops at staged", async () => { + const w = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"] }); + assert.deepEqual(await w.run(), { phase: "staged" }); + assert.deepEqual(w.log, ["rotator status", "rotator stage", "roll backend"]); + assert.match(w.report.at(-1), /for a year/); +}); + +test("another key mid-rotation refuses before anything is written", async () => { + const w = world({ state: { key: "NODE_BROKER_TOKEN", phase: "staged" } }); + await assert.rejects(w.run(), /NODE_BROKER_TOKEN is mid-rotation/); + assert.deepEqual(w.log, ["rotator status"]); +}); + +test("retire and abort refuse without a rotation and roll when there is one", async () => { + await assert.rejects(world({ action: "retire" }).run(), /no staged rotation/); + await assert.rejects(world({ action: "abort" }).run(), /no rotation/); + const abort = world({ action: "abort", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } }); + await abort.run(); + assert.deepEqual(abort.log, ["rotator status", "rotator abort", "roll backend,tracking"]); + const retire = world({ action: "retire", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } }); + await retire.run(); + assert.deepEqual(retire.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]); +}); + +test("prepare rolls only what changed and says to reference the keys in Terraform", async () => { + const w = world({ action: "prepare" }); + await w.run(); + assert.deepEqual(w.log, ["rotator status", "rotator prepare", "roll backend"]); + assert.match(w.report.at(-1), /secret_keys/); +}); + +test("rotate resumes a run that stopped after staging instead of refusing", async () => { + const w = world({ state: { key: "INTERNAL_API_TOKEN", phase: "staged" } }); + assert.deepEqual(await w.run(), { phase: "none" }); + assert.deepEqual(w.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]); + const sign = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"], state: { key: "UNSUBSCRIBE_SIGNING_SECRET", phase: "staged", until: "2027-10-25T00:00:00.000Z" } }); + assert.deepEqual(await sign.run(), { phase: "staged" }); + assert.deepEqual(sign.log, ["rotator status", "roll backend"]); + assert.match(sign.report.at(-1), /after 2027-10-25/); +}); diff --git a/.github/scripts/terraform-stack.sh b/.github/scripts/terraform-stack.sh new file mode 100755 index 000000000..429976a03 --- /dev/null +++ b/.github/scripts/terraform-stack.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Plans or applies one AWS stack without putting a value in a public log: +# var files come from the private state bucket and are masked first, plan and +# apply output goes to a private file, and only resource addresses and actions +# are printed. +# +# terraform-stack.sh plan +# terraform-stack.sh apply applies the plan saved by plan +# terraform-stack.sh classify +# +# plan writes $WORK/.summary (one "action address" per line) and prints +# safe, gated or none. Safe means only creates, in-place updates, and new +# revisions of task definitions, which keep their old revision. +set -euo pipefail + +classify() { + jq -r ' + [.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"])] as $c | + if ($c | length) == 0 then "none" + elif any($c[]; (.change.actions | index("delete")) and + ((.type == "aws_ecs_task_definition" and (.change.actions | length) == 2) | not)) then "gated" + else "safe" end' "$1" +} + +summary() { + jq -r '.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"]) | + "\(.change.actions | join("/")) \(.address)"' "$1" | sort +} + +main() { + local action=$1 stack=${2:-} + if [ "$action" = classify ]; then classify "$stack"; return 0; fi + case "$stack" in foundation|runtime|workers) ;; *) echo "unknown stack $stack" >&2; return 1 ;; esac + : "${STATE_BUCKET:?}" "${WORK:?}" + local dir="deploy/terraform/aws/$stack" vars="$WORK/$stack.tfvars.json" out="$WORK/$stack.log" + mkdir -p "$WORK" && chmod 700 "$WORK" + + if [ "$action" = plan ]; then + aws s3 cp --only-show-errors "s3://$STATE_BUCKET/warmbly/production/vars/$stack.tfvars.json" "$vars" + chmod 600 "$vars" + # Mask every value before anything can echo one. + jq -r '.. | strings | select(length >= 6)' "$vars" | while IFS= read -r line; do + [ -n "$line" ] && echo "::add-mask::$line" + done + terraform -chdir="$dir" init -input=false -no-color -backend-config="bucket=$STATE_BUCKET" >"$out" 2>&1 || { tail -n 20 "$out" >&2; return 1; } + local code=0 + terraform -chdir="$dir" plan -input=false -no-color -lock-timeout=5m -var-file="$vars" -var="state_bucket=$STATE_BUCKET" \ + -out="$WORK/$stack.tfplan" -detailed-exitcode >>"$out" 2>"$WORK/$stack.err" || code=$? + if [ "$code" = 1 ]; then + echo "terraform plan failed for $stack:" >&2 + grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true + return 1 + fi + terraform -chdir="$dir" show -json "$WORK/$stack.tfplan" >"$WORK/$stack.json" + summary "$WORK/$stack.json" >"$WORK/$stack.summary" + classify "$WORK/$stack.json" + rm -f "$WORK/$stack.json" + return 0 + fi + + if [ "$action" = apply ]; then + terraform -chdir="$dir" apply -input=false -no-color -lock-timeout=5m "$WORK/$stack.tfplan" >"$out" 2>"$WORK/$stack.err" || { + echo "terraform apply failed for $stack:" >&2 + grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true + return 1 + } + grep -E '^Apply complete' "$out" || true + return 0 + fi + echo "unknown action $action" >&2 + return 1 +} + +main "$@" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6e717896..3beae5238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,8 +69,11 @@ jobs: aws-deployment: - '.github/workflows/ci.yml' - '.github/workflows/deploy-aws.yml' + - '.github/workflows/rotate-keys.yml' - '.github/scripts/aws-deploy*.cjs' + - '.github/scripts/aws-rotate*.cjs' - '.github/aws-deploy/**' + - 'deploy/terraform/aws/runtime/rotate/**' tracking: - 'tracking/**' realtime: @@ -903,10 +906,10 @@ jobs: node-version: "22" - name: Install deployment client run: npm ci --prefix .github/aws-deploy --omit=dev --ignore-scripts --no-audit --no-fund - - name: Check migration and rollback gates + - name: Check deploy, rollback and rotation gates env: NODE_PATH: ${{ github.workspace }}/.github/aws-deploy/node_modules - run: node --test .github/scripts/aws-deploy.test.cjs + run: node --test .github/scripts/aws-deploy.test.cjs .github/scripts/aws-rotate.test.cjs deploy/terraform/aws/runtime/rotate/rotate.test.cjs ci-status: name: CI Status diff --git a/.github/workflows/deploy-aws.yml b/.github/workflows/deploy-aws.yml index 1cae35fe5..a6c6f0cec 100644 --- a/.github/workflows/deploy-aws.yml +++ b/.github/workflows/deploy-aws.yml @@ -1,48 +1,122 @@ name: Deploy AWS control plane +# A stable release reaches this on its own from release.yml when the +# AWS_DEPLOY_ENABLED variable is "true". Dispatch it by hand to redeploy, +# roll back, or roll out a changed third-party secret. on: + workflow_call: + inputs: + tag: + description: Release tag to deploy (vX.Y.Z) + required: true + type: string + mode: + description: deploy, rollback, redeploy or refresh-secrets + default: deploy + type: string + services: + description: all, or a comma-separated list of backend,consumer,forms,tracking,realtime + default: all + type: string workflow_dispatch: inputs: - migration_task_definition: - description: Terraform migration task definition ARN with an explicit revision - required: true + tag: + description: Release tag (vX.Y.Z); leave empty with refresh-secrets to keep the live images + required: false type: string - backend_task_definition: - description: Terraform backend task definition ARN with an explicit revision - required: true + mode: + description: What to do + type: choice + default: redeploy + options: [deploy, rollback, redeploy, refresh-secrets] + services: + description: all, or a comma-separated list of backend,consumer,forms,tracking,realtime + default: all type: string - consumer_task_definition: - description: Terraform consumer task definition ARN with an explicit revision - required: true - type: string - backend_desired_count: - description: Backend desired count (preserve or 0 through 5) - default: preserve - type: string - consumer_desired_count: - description: Consumer desired count (preserve or 0 through 5) - default: preserve + bake_seconds: + description: Seconds each service is watched against its alarms after it is stable + default: "300" type: string + fleet: + description: Start a canary rollout of the workers and consumers after the services + default: true + type: boolean + require_attestation: + description: Refuse images without build provenance (only releases before provenance need false) + default: true + type: boolean + allow_dropped_secrets: + description: Allow a revision that stops referencing a secret the live service reads + default: false + type: boolean permissions: contents: read -concurrency: - group: aws-production-control-plane - cancel-in-progress: false - jobs: deploy: + name: ${{ inputs.mode || 'deploy' }} ${{ inputs.tag || 'live release' }} runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 environment: aws-production + # One change to production at a time, queued rather than cancelled. The key + # rotation workflow takes the same group. + concurrency: + group: aws-production + cancel-in-progress: false permissions: contents: read id-token: write + attestations: read steps: - uses: actions/checkout@v4 with: persist-credentials: false + fetch-depth: 0 + + - name: Verify the release + id: release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + MODE: ${{ inputs.mode || 'deploy' }} + REQUIRE_ATTESTATION: ${{ inputs.require_attestation != false }} + run: | + set -euo pipefail + if [ -z "$TAG" ]; then + [ "$MODE" = refresh-secrets ] || { echo "::error::$MODE needs a tag"; exit 1; } + echo "images=" >> "$GITHUB_OUTPUT" + exit 0 + fi + printf '%s\n' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { + echo "::error::$TAG is not a stable vX.Y.Z tag; prereleases never deploy"; exit 1; } + git rev-parse -q --verify "refs/tags/$TAG^{commit}" >/dev/null || { + echo "::error::tag $TAG does not exist"; exit 1; } + git merge-base --is-ancestor "refs/tags/$TAG^{commit}" origin/main || { + echo "::error::$TAG is not on main"; exit 1; } + state=$(gh release view "$TAG" --json isDraft,isPrerelease --jq '"\(.isDraft) \(.isPrerelease)"') + [ "$state" = "false false" ] || { echo "::error::$TAG is a draft or a prerelease"; exit 1; } + mkdir -p /tmp/release + gh release download "$TAG" --pattern images.json --dir /tmp/release --clobber + jq -e --arg tag "$TAG" '.tag == $tag' /tmp/release/images.json >/dev/null || { + echo "::error::images.json does not describe $TAG"; exit 1; } + registry=$(jq -r .registry /tmp/release/images.json) + for key in backend backend-kafka consumer consumer-kafka worker worker-kafka forms tracking tracking-kafka realtime; do + digest=$(jq -r --arg k "$key" '.images[$k] // empty' /tmp/release/images.json) + if [ -z "$digest" ]; then + case "$key" in *-kafka) echo "$key has no digest in $TAG; skipping it"; continue ;; esac + echo "::error::images.json has no $key digest"; exit 1 + fi + if [ "$REQUIRE_ATTESTATION" = true ]; then + gh attestation verify "oci://$registry/${key%-kafka}@$digest" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "refs/tags/$TAG" >/dev/null + echo "verified provenance of $key" + fi + done + echo "images=/tmp/release/images.json" >> "$GITHUB_OUTPUT" + - uses: actions/setup-node@v4 with: node-version: "22" @@ -52,14 +126,16 @@ jobs: with: role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }} aws-region: us-east-1 - - name: Migrate, verify and deploy + - name: Gate, migrate and roll out env: NODE_PATH: ${{ github.workspace }}/.github/aws-deploy/node_modules AWS_ECS_CLUSTER: ${{ vars.AWS_ECS_CLUSTER }} AWS_ECS_SERVICE_PREFIX: ${{ vars.AWS_ECS_SERVICE_PREFIX }} - AWS_MIGRATION_TASK_DEFINITION: ${{ inputs.migration_task_definition }} - AWS_BACKEND_TASK_DEFINITION: ${{ inputs.backend_task_definition }} - AWS_CONSUMER_TASK_DEFINITION: ${{ inputs.consumer_task_definition }} - AWS_BACKEND_DESIRED_COUNT: ${{ inputs.backend_desired_count }} - AWS_CONSUMER_DESIRED_COUNT: ${{ inputs.consumer_desired_count }} + DEPLOY_TAG: ${{ inputs.tag }} + DEPLOY_MODE: ${{ inputs.mode || 'deploy' }} + DEPLOY_SERVICES: ${{ inputs.services || 'all' }} + DEPLOY_BAKE_SECONDS: ${{ inputs.bake_seconds || vars.AWS_DEPLOY_BAKE_SECONDS || '300' }} + DEPLOY_FLEET: ${{ inputs.fleet != false }} + DEPLOY_ALLOW_DROPPED_SECRETS: ${{ inputs.allow_dropped_secrets == true }} + DEPLOY_IMAGES_JSON: ${{ steps.release.outputs.images }} run: node .github/scripts/aws-deploy.cjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df2fd1a29..57363d5cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,6 +54,8 @@ jobs: permissions: contents: read packages: write + id-token: write + attestations: write steps: - uses: actions/checkout@v4 @@ -68,6 +70,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push + id: build uses: docker/build-push-action@v6 with: context: . @@ -92,6 +95,13 @@ jobs: cache-from: type=gha,scope=${{ matrix.service }} cache-to: type=gha,mode=max,scope=${{ matrix.service }} + # The production deploy refuses a digest this workflow did not attest. + - name: Attest build provenance + uses: actions/attest-build-provenance@v2 + with: + subject-name: ${{ env.IMAGE_PREFIX }}/${{ matrix.service }} + subject-digest: ${{ steps.build.outputs.digest }} + # Frontends (web dashboard, admin panel): the pnpm build runs once on the # build platform and its static output is served from nginx, so a single # buildx build produces both arches without a per-arch rebuild. @@ -247,6 +257,8 @@ jobs: permissions: contents: read packages: write + id-token: write + attestations: write steps: - name: Download digests uses: actions/download-artifact@v4 @@ -275,6 +287,20 @@ jobs: -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:prod${{ matrix.target.suffix }} \ $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}@sha256:%s ' *) + - name: Read the manifest digest + id: manifest + run: | + digest=$(docker buildx imagetools inspect \ + "${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:${{ github.ref_name }}${{ matrix.target.suffix }}" \ + --format '{{json .Manifest}}' | jq -r .digest) + echo "digest=$digest" >> "$GITHUB_OUTPUT" + + - name: Attest build provenance + uses: actions/attest-build-provenance@v2 + with: + subject-name: ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }} + subject-digest: ${{ steps.manifest.outputs.digest }} + # The `warmbly` CLI is a plain static binary, so it cross-compiles for every # platform on one runner. # @@ -371,6 +397,11 @@ jobs: ./scripts/check-images-public.sh --prefix "$IMAGE_PREFIX" --tag "$TAG" \ backend consumer worker forms updater web admin tracking realtime \ | grep -E '^[a-z]+[[:space:]]+sha256:' > /tmp/digests.tsv + # The Kafka builds the hosted deploy and its fleet run, keyed -kafka. + ./scripts/check-images-public.sh --prefix "$IMAGE_PREFIX" --tag "$TAG-kafka" \ + backend consumer worker tracking \ + | grep -E '^[a-z]+[[:space:]]+sha256:' \ + | sed -E 's/^([a-z]+)/\1-kafka/' > /tmp/digests-kafka.tsv { printf '{\n' printf ' "tag": "%s",\n' "$TAG" @@ -381,7 +412,7 @@ jobs: [ "$first" = 1 ] || printf ',\n' first=0 printf ' "%s": "%s"' "$svc" "$digest" - done < /tmp/digests.tsv + done < <(cat /tmp/digests.tsv /tmp/digests-kafka.tsv) printf '\n }\n}\n' } > /tmp/images.json cat /tmp/images.json @@ -389,6 +420,8 @@ jobs: # and the installer would silently skip whatever is missing. [ "$(wc -l < /tmp/digests.tsv)" -eq 9 ] || { echo "::error::expected 9 image digests, got $(wc -l < /tmp/digests.tsv)"; exit 1; } + [ "$(wc -l < /tmp/digests-kafka.tsv)" -eq 4 ] || { + echo "::error::expected 4 Kafka image digests, got $(wc -l < /tmp/digests-kafka.tsv)"; exit 1; } - name: Generate release notes env: @@ -451,9 +484,7 @@ jobs: | Updater | `${{ env.IMAGE_PREFIX }}/updater:${{ github.ref_name }}` | | CLI | `${{ env.IMAGE_PREFIX }}/cli:${{ github.ref_name }}` | - ## Deployment - - This release will be automatically deployed to production. + The `-kafka` entries are the same release built for a Kafka event bus. EOF } > /tmp/release-body.md @@ -505,6 +536,22 @@ jobs: draft: false prerelease: ${{ contains(github.ref_name, '-') }} + # A stable tag on main deploys itself once the repository opts in with the + # AWS_DEPLOY_ENABLED variable; setting it to anything else is the kill switch. + deploy-production: + name: Deploy to production + needs: create-release + if: ${{ !contains(github.ref_name, '-') && vars.AWS_DEPLOY_ENABLED == 'true' }} + permissions: + contents: read + id-token: write + attestations: read + uses: ./.github/workflows/deploy-aws.yml + with: + tag: ${{ github.ref_name }} + mode: deploy + secrets: inherit + pages-targets: name: Configure Pages release deployments needs: create-release diff --git a/.github/workflows/rotate-keys.yml b/.github/workflows/rotate-keys.yml new file mode 100644 index 000000000..3c9bf571d --- /dev/null +++ b/.github/workflows/rotate-keys.yml @@ -0,0 +1,70 @@ +name: Rotate instance keys + +# New values are generated inside AWS by the rotator task and never reach this +# runner or its logs; it only sees secret version ids. See the AWS Fargate +# runbook in the self-hosting docs for what each step proves. +on: + workflow_dispatch: + inputs: + key: + description: Key to rotate + type: choice + options: + - INTERNAL_API_TOKEN + - NODE_BROKER_TOKEN + - CREDENTIALS_ENCRYPTION_KEY + - TWOFA_SECRET + - UNSUBSCRIBE_SIGNING_SECRET + action: + description: rotate runs every step it can prove safe; retire and abort finish or undo a staged rotation + type: choice + default: status + options: [status, prepare, rotate, retire, abort] + node_wait_minutes: + description: How long to wait for every node to apply the new env + default: "30" + type: string + bake_seconds: + description: Seconds each rolled service is watched against its alarms + default: "300" + type: string + +permissions: + contents: read + +jobs: + rotate: + name: ${{ inputs.action }} ${{ inputs.key }} + runs-on: ubuntu-latest + timeout-minutes: 240 + environment: aws-production + # Same group as deploy-aws.yml: a rotation and a deploy never overlap. + concurrency: + group: aws-production + cancel-in-progress: false + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install deployment client + run: npm ci --prefix .github/aws-deploy --omit=dev --ignore-scripts --no-audit --no-fund + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }} + aws-region: us-east-1 + - name: Rotate + env: + NODE_PATH: ${{ github.workspace }}/.github/aws-deploy/node_modules + AWS_ECS_CLUSTER: ${{ vars.AWS_ECS_CLUSTER }} + AWS_ECS_SERVICE_PREFIX: ${{ vars.AWS_ECS_SERVICE_PREFIX }} + ROTATE_KEY: ${{ inputs.key }} + ROTATE_ACTION: ${{ inputs.action }} + ROTATE_NODE_WAIT_MINUTES: ${{ inputs.node_wait_minutes }} + DEPLOY_BAKE_SECONDS: ${{ inputs.bake_seconds }} + run: node .github/scripts/aws-rotate.cjs diff --git a/.github/workflows/terraform-aws.yml b/.github/workflows/terraform-aws.yml new file mode 100644 index 000000000..58c31638c --- /dev/null +++ b/.github/workflows/terraform-aws.yml @@ -0,0 +1,274 @@ +name: Terraform AWS + +# Pull requests get a plan summary comment. On main, a stack whose plan only +# creates or updates applies on its own; one that replaces or destroys waits in +# the aws-infrastructure-approval environment. Var files live in the private +# state bucket, and no plan body ever reaches a log, a comment or an artifact. +on: + pull_request: + paths: + - 'deploy/terraform/aws/**' + - '.github/workflows/terraform-aws.yml' + - '.github/scripts/terraform-stack.sh' + push: + branches: [main] + paths: + - 'deploy/terraform/aws/**' + - '.github/workflows/terraform-aws.yml' + - '.github/scripts/terraform-stack.sh' + workflow_dispatch: + +permissions: + contents: read + +env: + TF_IN_AUTOMATION: "1" + TF_INPUT: "0" + TERRAFORM_VERSION: "1.16.5" + +jobs: + check: + name: Format and validate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + - run: terraform fmt -check -recursive deploy/terraform/aws + - name: Validate + run: | + set -euo pipefail + for stack in bootstrap foundation runtime workers; do + terraform -chdir="deploy/terraform/aws/$stack" init -backend=false -input=false >/dev/null + terraform -chdir="deploy/terraform/aws/$stack" validate -no-color + done + - run: shellcheck .github/scripts/terraform-stack.sh + + plan: + name: Plan summary + needs: check + # Same-repository pull requests only: a fork never reaches the plan role. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && vars.AWS_TERRAFORM_ENABLED == 'true' + runs-on: ubuntu-latest + environment: aws-infrastructure-plan + permissions: + contents: read + id-token: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_TERRAFORM_PLAN_ROLE_ARN }} + aws-region: us-east-1 + - name: Plan + env: + STATE_BUCKET: ${{ vars.AWS_TERRAFORM_STATE_BUCKET }} + WORK: ${{ runner.temp }}/terraform + run: | + set -euo pipefail + { + echo "" + echo "### Terraform plan" + echo + # The foundation reads the cache password while planning, which the plan role may not. + echo "**foundation**: planned on merge, before it applies." + for stack in runtime workers; do + class=$(.github/scripts/terraform-stack.sh plan "$stack") + echo + case "$class" in + none) echo "**$stack**: no changes" ;; + safe) echo "**$stack**: applies on merge" ;; + gated) echo "**$stack**: replaces or destroys, so it waits for approval in aws-infrastructure-approval" ;; + esac + if [ "$class" != none ]; then + echo '```' + cat "$WORK/$stack.summary" + echo '```' + fi + done + } > "$RUNNER_TEMP/comment.md" + cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" + - name: Comment + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" --paginate \ + --jq '[.[] | select(.user.login == "github-actions[bot]" and (.body | startswith("")))][0].id // empty') + if [ -n "$id" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -F "body=@$RUNNER_TEMP/comment.md" >/dev/null + else + gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F "body=@$RUNNER_TEMP/comment.md" >/dev/null + fi + + apply: + name: Apply safe changes + needs: check + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && vars.AWS_TERRAFORM_ENABLED == 'true' + runs-on: ubuntu-latest + environment: aws-infrastructure + # Shares the deploy queue, so an apply never lands in the middle of a rollout. + concurrency: + group: aws-production + cancel-in-progress: false + permissions: + contents: read + id-token: write + outputs: + gated: ${{ steps.apply.outputs.gated }} + gated_summary: ${{ steps.apply.outputs.gated_summary }} + redeploy: ${{ steps.apply.outputs.redeploy }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_TERRAFORM_APPLY_ROLE_ARN }} + aws-region: us-east-1 + - name: Plan and apply, in order + id: apply + env: + STATE_BUCKET: ${{ vars.AWS_TERRAFORM_STATE_BUCKET }} + WORK: ${{ runner.temp }}/terraform + run: | + set -euo pipefail + redeploy=false + for stack in foundation runtime workers; do + class=$(.github/scripts/terraform-stack.sh plan "$stack") + echo "### $stack: $class" >> "$GITHUB_STEP_SUMMARY" + if [ "$class" != none ]; then + { echo '```'; cat "$WORK/$stack.summary"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + if [ "$class" = gated ]; then + # Later stacks read this one's outputs, so they wait with it. + echo "gated=$stack" >> "$GITHUB_OUTPUT" + echo "gated_summary=$(sha256sum "$WORK/$stack.summary" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "$stack needs approval in aws-infrastructure-approval" >> "$GITHUB_STEP_SUMMARY" + break + fi + if [ "$class" = safe ]; then + .github/scripts/terraform-stack.sh apply "$stack" + if [ "$stack" = runtime ] && grep -q 'aws_ecs_task_definition\.service\[' "$WORK/$stack.summary"; then redeploy=true; fi + fi + done + echo "redeploy=$redeploy" >> "$GITHUB_OUTPUT" + + apply-gated: + name: Apply after approval + needs: apply + if: needs.apply.outputs.gated != '' + runs-on: ubuntu-latest + environment: aws-infrastructure-approval + concurrency: + group: aws-production + cancel-in-progress: false + permissions: + contents: read + id-token: write + outputs: + redeploy: ${{ steps.apply.outputs.redeploy }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_TERRAFORM_APPLY_ROLE_ARN }} + aws-region: us-east-1 + - name: Re-plan, confirm, apply + id: apply + env: + STATE_BUCKET: ${{ vars.AWS_TERRAFORM_STATE_BUCKET }} + WORK: ${{ runner.temp }}/terraform + GATED: ${{ needs.apply.outputs.gated }} + APPROVED: ${{ needs.apply.outputs.gated_summary }} + run: | + set -euo pipefail + redeploy=false + started=false + for stack in foundation runtime workers; do + [ "$stack" = "$GATED" ] && started=true + [ "$started" = true ] || continue + class=$(.github/scripts/terraform-stack.sh plan "$stack") + echo "### $stack: $class" >> "$GITHUB_STEP_SUMMARY" + if [ "$stack" = "$GATED" ]; then + # What was approved is what applies: a plan that moved since asks again. + [ "$(sha256sum "$WORK/$stack.summary" | cut -d' ' -f1)" = "$APPROVED" ] || { + echo "::error::the $stack plan changed after approval was requested; run this workflow again"; exit 1; } + elif [ "$class" = gated ]; then + echo "::error::$stack also replaces or destroys; run this workflow again to approve it" + exit 1 + fi + [ "$class" = none ] && continue + { echo '```'; cat "$WORK/$stack.summary"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + .github/scripts/terraform-stack.sh apply "$stack" + if [ "$stack" = runtime ] && grep -q 'aws_ecs_task_definition\.service\[' "$WORK/$stack.summary"; then redeploy=true; fi + done + echo "redeploy=$redeploy" >> "$GITHUB_OUTPUT" + + live-release: + name: Find the live release + needs: [apply, apply-gated] + if: always() && (needs.apply.outputs.redeploy == 'true' || needs.apply-gated.outputs.redeploy == 'true') && vars.AWS_DEPLOY_ENABLED == 'true' + runs-on: ubuntu-latest + environment: aws-production + permissions: + contents: read + id-token: write + outputs: + tag: ${{ steps.live.outputs.tag }} + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }} + aws-region: us-east-1 + - id: live + env: + CLUSTER: ${{ vars.AWS_ECS_CLUSTER }} + PREFIX: ${{ vars.AWS_ECS_SERVICE_PREFIX }} + run: | + set -euo pipefail + prefix=${PREFIX:-${CLUSTER##*/}} + def=$(aws ecs describe-services --cluster "$CLUSTER" --services "$prefix-backend" --query 'services[0].taskDefinition' --output text) + tag=$(aws ecs describe-task-definition --task-definition "$def" --include TAGS --query "tags[?key=='warmbly-release'].value | [0]" --output text) + if ! printf '%s\n' "$tag" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::warning::the live backend carries no release tag; redeploy a release by hand to roll out the new task definitions" + tag="" + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + # Services ignore task_definition in Terraform, so a new revision reaches them through a redeploy. + redeploy: + name: Roll the new task definitions out + needs: live-release + if: always() && needs.live-release.outputs.tag != '' + permissions: + contents: read + id-token: write + attestations: read + uses: ./.github/workflows/deploy-aws.yml + with: + tag: ${{ needs.live-release.outputs.tag }} + mode: redeploy + secrets: inherit diff --git a/admin/src/app/dashboard/ConfigurationPage.tsx b/admin/src/app/dashboard/ConfigurationPage.tsx index 3f0c9ae98..39a3c2cce 100644 --- a/admin/src/app/dashboard/ConfigurationPage.tsx +++ b/admin/src/app/dashboard/ConfigurationPage.tsx @@ -1,21 +1,22 @@ // Configuration: everything an operator can read or change about this -// deployment, in four tabs. Settings and notifications write to the instance -// settings document; environment and limits are read only. ?tab= deep-links +// deployment, in five tabs. Settings and notifications write to the instance +// settings document; environment, limits and keys are read only. ?tab= deep-links // a tab, and a dirty form is confirmed before a tab switch or a navigation // discards it. import { useCallback, useEffect, useRef, useState } from "react"; import { useBlocker, useSearchParams } from "react-router-dom"; -import { Bell, Gauge, Settings2, Terminal } from "lucide-react"; +import { Bell, Gauge, KeyRound, Settings2, Terminal } from "lucide-react"; import { PageHeader } from "@/components/layout/PageHeader"; import { PageTabs } from "@/components/layout/PageTabs"; import { useConfirm } from "@/components/ConfirmDialog"; import { EnvironmentTab } from "./configuration/EnvironmentTab"; +import { KeysTab } from "./configuration/KeysTab"; import { LimitsTab } from "./configuration/LimitsTab"; import { NotificationsTab } from "./configuration/NotificationsTab"; import { SettingsTab } from "./configuration/SettingsTab"; -const TAB_IDS = ["settings", "notifications", "environment", "limits"] as const; +const TAB_IDS = ["settings", "notifications", "environment", "limits", "keys"] as const; type TabId = (typeof TAB_IDS)[number]; function isTabId(v: string | null): v is TabId { @@ -117,6 +118,7 @@ export default function ConfigurationPage() { { id: "notifications", label: "Notifications", icon: Bell }, { id: "environment", label: "Environment", icon: Terminal }, { id: "limits", label: "Limits", icon: Gauge }, + { id: "keys", label: "Keys", icon: KeyRound }, ]} value={tab} onChange={(id) => void setTab(id)} @@ -129,6 +131,7 @@ export default function ConfigurationPage() { {tab === "notifications" && } {tab === "environment" && } {tab === "limits" && } + {tab === "keys" && } ); diff --git a/admin/src/app/dashboard/configuration/KeysTab.tsx b/admin/src/app/dashboard/configuration/KeysTab.tsx new file mode 100644 index 000000000..1327e06e7 --- /dev/null +++ b/admin/src/app/dashboard/configuration/KeysTab.tsx @@ -0,0 +1,221 @@ +// Configuration, keys: whether a key rotation can be finished. Shows which +// sealed values still need a read key, which credential windows are open and +// which nodes have not picked up the current env. Read only: rotation runs +// where the keys are stored, never from the browser. + +import { useQueryClient, useQuery } from "@tanstack/react-query"; +import { useState } from "react"; +import { ExternalLink, KeyRound, RefreshCw, ShieldAlert, ShieldCheck } from "lucide-react"; +import { ErrorState } from "@/components/ErrorState"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Callout, EmptyState, StatusBadge } from "@/components/ui/kit"; +import { SettingsGroup, SettingsRow } from "./SettingsLayout"; +import { getKeyStatus, type KeyColumnCount, type KeyStatus } from "@/lib/api/client/admin/keys"; +import { docsUrl } from "@/lib/docs"; +import { TONE_TEXT } from "@/lib/tones"; +import { cn } from "@/lib/utils"; + +const QUERY_KEY = ["admin", "keys", "status"] as const; + +export function KeysTab() { + const queryClient = useQueryClient(); + const [refreshing, setRefreshing] = useState(false); + const statusQ = useQuery({ + queryKey: QUERY_KEY, + queryFn: () => getKeyStatus(), + retry: false, + }); + + async function refresh() { + setRefreshing(true); + try { + queryClient.setQueryData(QUERY_KEY, await getKeyStatus(true)); + } catch { + await statusQ.refetch(); + } finally { + setRefreshing(false); + } + } + + return ( +
+
+

+ Whether a key rotation can be finished. A rotation keeps the old key as a read + key while stored values are resealed; it is safe to retire once nothing below + still needs it. Rotation itself runs where the keys are stored. +

+
+ + +
+
+ + {statusQ.isLoading && ( +
+ {[0, 1].map((i) => ( +
+ + +
+ ))} +
+ )} + + {statusQ.isError && ( + statusQ.refetch()} + /> + )} + + {statusQ.data && } +
+ ); +} + +function KeyStatusView({ status }: { status: KeyStatus }) { + const { keys, fleet, ready } = status; + const columns = (keys.columns ?? []).filter((c) => !c.missing); + const onReadKey = columns.reduce((n, c) => n + c.read_key, 0); + const unreadable = columns.reduce((n, c) => n + c.unreadable, 0); + const windows = keys.grace_windows ?? []; + const nodes = fleet?.nodes ?? []; + const readKeys = keys.instance_read_keys + keys.mfa_read_keys; + + return ( + <> + {unreadable > 0 ? ( + + Retire nothing. Restore the key that sealed them as a read key first. + + ) : onReadKey > 0 ? ( + + Run the reseal step of the rotation, then check again. + + ) : ( + + {readKeys > 0 + ? "The read keys can be retired once no rollback target or retained backup still needs them." + : "No read keys are configured, so there is nothing to retire."} + + )} + + + + 0 ? "warning" : "neutral"}> + {readKeyLabel(keys.instance_read_keys)} + + + + {keys.mfa_configured ? ( + 0 ? "warning" : "neutral"}>{readKeyLabel(keys.mfa_read_keys)} + ) : ( + Not configured + )} + + + + {ready.retire_read_keys ? "Yes" : readKeys > 0 ? "Not yet" : "Nothing to retire"} + + + + + + {columns.length === 0 ? ( + + ) : ( +
+ + + + + + + + + + + + + {columns.map((c) => ( + + ))} + +
ColumnPrimaryRead keyWorkspaceUnsealedUnreadable
+
+ )} +
+ + {windows.length > 0 && ( + + {windows.map((w) => ( + {w.name}} + description={`${w.entries} previous ${w.entries === 1 ? "value" : "values"}`} + > + + {w.open ? `Open until ${new Date(w.until).toLocaleString()}` : "Closed"} + + + ))} + + )} + + {fleet && ( + + {nodes.length === 0 ? ( + + ) : ( + nodes.map((n) => ( + + + {n.current ? "Current" : n.tracked ? "Updating" : "Manual"} + + + )) + )} + + )} + + ); +} + +function ColumnRow({ c }: { c: KeyColumnCount }) { + return ( + + + {c.table}.{c.column} + {c.domain === "mfa" && (MFA)} + + + + + + + + ); +} + +function Cell({ n, tone, last }: { n: number; tone?: "warning" | "danger"; last?: boolean }) { + const color = n === 0 ? TONE_TEXT.neutral : TONE_TEXT[tone ?? "strong"]; + return {n.toLocaleString()}; +} + +function readKeyLabel(n: number) { + return n === 0 ? "Primary only" : `Primary + ${n} read ${n === 1 ? "key" : "keys"}`; +} diff --git a/admin/src/lib/api/client/admin/keys.ts b/admin/src/lib/api/client/admin/keys.ts new file mode 100644 index 000000000..808fad9b5 --- /dev/null +++ b/admin/src/lib/api/client/admin/keys.ts @@ -0,0 +1,56 @@ +// /admin/keys/status: where every value the instance keys seal stands, and +// whether every fleet node runs the env the backend renders now. Read only. + +import { Request } from "@/lib/api/client"; + +export interface KeyColumnCount { + table: string; + column: string; + domain: "instance" | "mfa"; + primary: number; + read_key: number; + workspace: number; + unsealed: number; + unreadable: number; + missing?: boolean; +} + +export interface KeyGraceWindow { + name: string; + until: string; + open: boolean; + entries: number; +} + +export interface KeyReport { + applied: boolean; + instance_read_keys: number; + mfa_read_keys: number; + mfa_configured: boolean; + columns: KeyColumnCount[] | null; + grace_windows: KeyGraceWindow[] | null; + checked_at: string; +} + +export interface NodeEnvState { + id: string; + name: string; + role: string; + applied?: string; + tracked: boolean; + current: boolean; +} + +export interface KeyStatus { + keys: KeyReport; + fleet?: { all_current: boolean; nodes: NodeEnvState[] | null }; + ready: { retire_read_keys: boolean; retire_tokens: boolean }; +} + +export function getKeyStatus(refresh = false): Promise { + return Request({ + method: "GET", + url: refresh ? "/admin/keys/status?refresh=true" : "/admin/keys/status", + authorization: true, + }); +} diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 63cd23891..c9cc1446d 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -77,6 +77,7 @@ import ( "github.com/warmbly/warmbly/internal/app/instanceconfig" "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/app/integration" + "github.com/warmbly/warmbly/internal/app/keystatus" "github.com/warmbly/warmbly/internal/app/leadsync" "github.com/warmbly/warmbly/internal/app/mailboxavatar" "github.com/warmbly/warmbly/internal/app/mailboximport" @@ -222,6 +223,7 @@ func main() { var fleetNodeRepo repository.FleetNodeRepository var fleetSettingsRepo repository.FleetSettingsRepository var fleetNodeService *fleetnode.Service + var keyStatusDB keystatus.DB var tasksService tasks.TasksService var advancedService advanced.Service var unsubSigner *unsublink.Signer @@ -496,6 +498,9 @@ func main() { log.Println("Database migrations completed") } primaryDB, err := db.NewWithDirect(ctx, primaryDBEndpoint, directDBEndpoint) + if err == nil { + keyStatusDB = primaryDB.Pool + } if err != nil { errs.CaptureFatal(err) log.Fatal(err) @@ -1196,6 +1201,8 @@ func main() { fleetNodeRepo = repository.NewFleetNodeRepository(primaryDB) fleetSettingsRepo = repository.NewFleetSettingsRepository(primaryDB) fleetNodeService = fleetnode.New(fleetNodeRepo, workerRepository, fleetSettingsRepo) + // Idle unless a staged rollout was started; every replica may run it. + go fleetNodeService.RunRolloutLoop(ctx, 30*time.Second) settingsRegistrar := settings.NewRegistrar(storageBackendRepo) if err := settingsRegistrar.RegisterAll(ctx, []settings.Backend{ {Kind: "kms", Provider: kms.Name(), Display: kms.Name(), ReadOnly: true}, @@ -2572,6 +2579,7 @@ func main() { FleetNodeRepo: fleetNodeRepo, FleetSettingsRepo: fleetSettingsRepo, FleetNodes: fleetNodeService, + KeyStatusDB: keyStatusDB, // Danger zone DangerZoneService: dangerZoneService, diff --git a/cmd/warmblyctl/fleet.go b/cmd/warmblyctl/fleet.go index 09c2abb07..e80962c01 100644 --- a/cmd/warmblyctl/fleet.go +++ b/cmd/warmblyctl/fleet.go @@ -28,6 +28,14 @@ func fleetUsage(w *os.File) { warmblyctl fleet version What version the fleet should be running. warmblyctl fleet version Pin the whole fleet to a tag. warmblyctl fleet channel Follow stable, dev, or pinned. + warmblyctl fleet rollout start Move the fleet in stages: one canary, then waves. + --digests key=sha256:...,... --canary 1 --waves 100 + --bake 30m --deadline 15m + warmblyctl fleet rollout status Stage, admitted nodes, and why it halted. + warmblyctl fleet rollout advance Admit the next stage now, skipping the bake. + warmblyctl fleet rollout halt Stop admitting; admitted nodes return to From. + warmblyctl fleet rollout abort Discard the rollout, leaving the fleet on From. + warmblyctl fleet env-status Whether every node has applied the current env. Adding a machine is two commands: issue a token here, then on that machine run @@ -61,6 +69,10 @@ func runFleet(ctx context.Context, args []string) error { return runFleetVersion(ctx, args[1:]) case "channel": return runFleetChannel(ctx, args[1:]) + case "rollout": + return runFleetRollout(ctx, args[1:]) + case "env-status": + return runFleetEnvStatus(ctx, args[1:]) } fleetUsage(os.Stderr) return fmt.Errorf("unknown fleet subcommand %q", args[0]) diff --git a/cmd/warmblyctl/fleet_rollout.go b/cmd/warmblyctl/fleet_rollout.go new file mode 100644 index 000000000..35d40b677 --- /dev/null +++ b/cmd/warmblyctl/fleet_rollout.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/warmbly/warmbly/internal/app/fleetnode" + "github.com/warmbly/warmbly/internal/models" +) + +func runFleetRollout(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("usage: warmblyctl fleet rollout ") + } + switch args[0] { + case "start": + return runFleetRolloutStart(ctx, args[1:]) + case "status": + return runFleetRolloutStatus(ctx, args[1:]) + case "advance", "halt", "abort": + return runFleetRolloutAction(ctx, args[0], args[1:]) + } + return fmt.Errorf("unknown rollout subcommand %q", args[0]) +} + +func runFleetRolloutStart(ctx context.Context, args []string) error { + fs := newFlagSet("fleet rollout start") + digests := fs.String("digests", "", "comma-separated image=sha256:... pairs (worker, worker-kafka, consumer, consumer-kafka)") + canary := fs.Int("canary", 1, "nodes admitted first, alone") + waves := fs.String("waves", "100", "comma-separated cumulative percentages admitted after the canary") + bake := fs.Duration("bake", 30*time.Minute, "healthy time on the target before the next stage") + deadline := fs.Duration("deadline", 15*time.Minute, "time an admitted node has to report the target") + asJSON := fs.Bool("json", false, "machine-readable output") + tag, err := parseWithPositional(fs, args) + if err != nil { + return err + } + policy := models.FleetRolloutPolicy{Canary: *canary, BakeSeconds: int(bake.Seconds()), DeadlineSeconds: int(deadline.Seconds())} + for _, w := range strings.Split(*waves, ",") { + if w = strings.TrimSpace(w); w == "" { + continue + } + pct, err := strconv.Atoi(w) + if err != nil || pct <= 0 || pct > 100 { + return fmt.Errorf("--waves takes percentages between 1 and 100, got %q", w) + } + policy.Waves = append(policy.Waves, pct) + } + pins := map[string]string{} + for _, pair := range strings.Split(*digests, ",") { + if pair = strings.TrimSpace(pair); pair == "" { + continue + } + key, value, ok := strings.Cut(pair, "=") + if !ok { + return fmt.Errorf("--digests entries are image=sha256:..., got %q", pair) + } + pins[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + + c, _, _, svc, err := fleetDeps(ctx) + if err != nil { + return err + } + defer c.close() + state, err := svc.StartRollout(ctx, tag, pins, policy) + if err != nil { + return err + } + return printRollout(state, *asJSON) +} + +// parseWithPositional accepts the one positional argument before or after flags. +func parseWithPositional(fs interface { + Parse([]string) error + NArg() int + Arg(int) string +}, args []string) (string, error) { + var positional string + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + positional, args = args[0], args[1:] + } + if err := fs.Parse(args); err != nil { + return "", err + } + if positional == "" && fs.NArg() == 1 { + positional = fs.Arg(0) + } else if fs.NArg() != 0 { + return "", errors.New("expected exactly one tag") + } + if positional == "" { + return "", errors.New("a tag is required") + } + return positional, nil +} + +func runFleetRolloutStatus(ctx context.Context, args []string) error { + fs := newFlagSet("fleet rollout status") + asJSON := fs.Bool("json", false, "machine-readable output") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + c, _, settings, _, err := fleetDeps(ctx) + if err != nil { + return err + } + defer c.close() + state, err := settings.GetRelease(ctx) + if err != nil { + return err + } + return printRollout(state, *asJSON) +} + +func runFleetRolloutAction(ctx context.Context, action string, args []string) error { + fs := newFlagSet("fleet rollout " + action) + reason := fs.String("reason", "", "why, recorded with a halt") + asJSON := fs.Bool("json", false, "machine-readable output") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + c, _, _, svc, err := fleetDeps(ctx) + if err != nil { + return err + } + defer c.close() + var state *models.FleetReleaseState + switch action { + case "advance": + state, err = svc.AdvanceRollout(ctx, true) + case "halt": + state, err = svc.HaltRollout(ctx, *reason) + default: + state, err = svc.AbortRollout(ctx) + } + if err != nil { + return err + } + return printRollout(state, *asJSON) +} + +func printRollout(state *models.FleetReleaseState, asJSON bool) error { + if state == nil { + state = &models.FleetReleaseState{} + } + if asJSON { + return json.NewEncoder(os.Stdout).Encode(state) + } + r := state.Rollout + if r == nil { + fmt.Printf("No rollout in progress. Fleet target: %s\n", orDash(state.Tag)) + return nil + } + fmt.Printf("Rollout %s -> %s: %s", orDash(r.From), r.Target, r.Stage) + if r.Stage == models.FleetRolloutWave { + fmt.Printf(" %d of %d", r.Wave+1, len(r.Policy.Waves)) + } + fmt.Printf(", %d admitted, stage started %s\n", len(r.Admitted), r.StageStartedAt.Format(time.RFC3339)) + if r.HaltReason != "" { + fmt.Println("Halted:", r.HaltReason) + fmt.Println("Admitted nodes return to", orDash(r.From)+". Fix the cause, then `fleet rollout start` again or `fleet rollout abort`.") + } + return nil +} + +// runFleetEnvStatus compares each live node's applied env revision with the +// env this process would render, which matches the backend's when it runs +// from the backend's own task configuration. +func runFleetEnvStatus(ctx context.Context, args []string) error { + fs := newFlagSet("fleet env-status") + asJSON := fs.Bool("json", false, "machine-readable output") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + c, nodes, _, _, err := fleetDeps(ctx) + if err != nil { + return err + } + defer c.close() + list, err := nodes.List(ctx, "") + if err != nil { + return err + } + report := fleetnode.EnvStatusOf(list) + if *asJSON { + if err := json.NewEncoder(os.Stdout).Encode(report); err != nil { + return err + } + } else { + for _, n := range report.Nodes { + state := "stale" + switch { + case n.Current: + state = "current" + case !n.Tracked: + state = "manual" + } + fmt.Printf("%-8s %-24s %-8s %s\n", n.Role, n.Name, state, n.ID) + } + } + if !report.AllCurrent { + return errors.New("some live nodes have not applied the current env") + } + return nil +} diff --git a/cmd/warmblyctl/keys.go b/cmd/warmblyctl/keys.go index 4a4982e6a..ecff25f76 100644 --- a/cmd/warmblyctl/keys.go +++ b/cmd/warmblyctl/keys.go @@ -12,95 +12,9 @@ import ( "strings" "time" - "github.com/jackc/pgx/v5" - - "github.com/warmbly/warmbly/internal/app/mailcred" - "github.com/warmbly/warmbly/internal/app/orgtransfer" - "github.com/warmbly/warmbly/internal/app/twofa" - "github.com/warmbly/warmbly/internal/config" - "github.com/warmbly/warmbly/internal/pkg/encrypt" - "github.com/warmbly/warmbly/internal/pkg/secretset" + "github.com/warmbly/warmbly/internal/app/keystatus" ) -// A rotation ships the new primary with the old key as a read key, reseals, and only then retires the old key. - -// keyDomain names which configured key seals a column. -type keyDomain string - -const ( - domainInstance keyDomain = "instance" - domainMFA keyDomain = "mfa" -) - -// sealedColumn is one column holding values sealed under an instance-wide key. -type sealedColumn struct { - Table string `json:"table"` - Column string `json:"column"` - Domain keyDomain `json:"domain"` - guard string -} - -// instanceOnlyColumns are sealed under an instance-wide key on tables that never travel in a workspace archive. -var instanceOnlyColumns = []sealedColumn{ - {Table: "cloud_link", Column: "token", Domain: domainInstance}, - {Table: "mailbox_erasures", Column: "refresh_token", Domain: domainInstance}, - {Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: domainMFA}, -} - -// sealedColumns is every column the instance keys seal: the archive registry's instance-key and mailbox secrets plus the instance-only ones. -func sealedColumns() []sealedColumn { - var out []sealedColumn - for _, t := range orgtransfer.Tables { - for _, s := range t.Secrets { - if s.Domain == orgtransfer.KeyDomainInstance || s.Domain == orgtransfer.KeyDomainMailbox { - out = append(out, sealedColumn{Table: t.Name, Column: s.Column, Domain: domainInstance, guard: s.Guard}) - } - } - } - return append(out, instanceOnlyColumns...) -} - -// columnCount is where one column's values stand against the configured keys. -type columnCount struct { - sealedColumn - Primary int `json:"primary"` - ReadKey int `json:"read_key"` - Workspace int `json:"workspace"` - Unsealed int `json:"unsealed"` - Unreadable int `json:"unreadable"` - Resealed int `json:"resealed,omitempty"` - Raced int `json:"changed_meanwhile,omitempty"` - Missing bool `json:"missing,omitempty"` -} - -// keyReport is the whole answer, shaped for --json as much as for a terminal. -type keyReport struct { - Applied bool `json:"applied"` - InstanceReadKeys int `json:"instance_read_keys"` - MFAReadKeys int `json:"mfa_read_keys"` - MFAConfigured bool `json:"mfa_configured"` - Columns []columnCount `json:"columns"` - Windows []graceWindow `json:"grace_windows"` -} - -// graceWindow is one previous-credential acceptance window and whether it is still open. -type graceWindow struct { - Name string `json:"name"` - Until time.Time `json:"until"` - Open bool `json:"open"` - Entries int `json:"entries"` -} - -// keyring is the configured keys the backend opens and seals with. -type keyring struct { - instance *encrypt.Encrypter - instanceRead int - mfa *[32]byte - mfaRead [][32]byte -} - -var errUnreadableValues = errors.New("some stored values do not open under any configured key") - func runKeys(ctx context.Context, args []string) error { if len(args) == 0 { keysUsage(os.Stderr) @@ -134,8 +48,11 @@ func keysUsage(w *os.File) { func runKeysScan(ctx context.Context, name string, args []string, reseal bool) error { fs := newFlagSet(name) asJSON := fs.Bool("json", false, "print the report as JSON") - var apply *bool + var apply, requirePrimary *bool var batch *int + if !reseal { + requirePrimary = fs.Bool("require-primary", false, "also exit non-zero while any value opens only under a read key") + } if reseal { apply = fs.Bool("apply", false, "write the resealed values; without it nothing is written") batch = fs.Int("batch", 500, "rows read per query") @@ -155,7 +72,7 @@ func runKeysScan(ctx context.Context, name string, args []string, reseal bool) e } write := apply != nil && *apply - keys, err := loadKeyring(ctx) + keys, err := keystatus.LoadKeyring(ctx) if err != nil { return err } @@ -165,18 +82,10 @@ func runKeysScan(ctx context.Context, name string, args []string, reseal bool) e } defer c.close() - report := keyReport{Applied: write, InstanceReadKeys: keys.instanceRead, MFAReadKeys: len(keys.mfaRead), MFAConfigured: keys.mfa != nil, Windows: graceWindows(time.Now())} - for _, col := range sealedColumns() { - if col.Domain == domainMFA && keys.mfa == nil { - continue - } - count, err := scanColumn(ctx, c, keys, col, size, write) - if err != nil { - return err - } - report.Columns = append(report.Columns, count) + report, err := keystatus.Scan(ctx, c.db.Pool, keys, keystatus.Options{Batch: size, Write: write}) + if err != nil { + return err } - if *asJSON { if err := json.NewEncoder(os.Stdout).Encode(report); err != nil { return err @@ -184,182 +93,16 @@ func runKeysScan(ctx context.Context, name string, args []string, reseal bool) e } else { printKeyReport(report, reseal) } - for _, col := range report.Columns { - if col.Unreadable > 0 { - return errUnreadableValues - } + if report.Unreadable() > 0 { + return keystatus.ErrUnreadable + } + if requirePrimary != nil && *requirePrimary && report.OnReadKey() > 0 { + return keystatus.ErrOnReadKey } return nil } -// loadKeyring resolves the keys exactly as the backend does at boot. -func loadKeyring(ctx context.Context) (*keyring, error) { - instance, err := encrypt.FromEnv() - if err != nil { - return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY or CREDENTIALS_ENCRYPTION_READ_KEYS is invalid: each must be 64 hex characters") - } - if instance == nil { - return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY is not set. Run this inside the backend container, where the keys are configured.") - } - reads, _ := secretset.Parse(os.Getenv("CREDENTIALS_ENCRYPTION_READ_KEYS")) - k := &keyring{instance: instance} - for _, r := range reads { - if r != os.Getenv("CREDENTIALS_ENCRYPTION_KEY") { - k.instanceRead++ - } - } - - var primary string - var mfaReads []string - if cfg, err := config.NewConfig(ctx); err == nil { - if auth, aerr := cfg.LoadAuthConfig(ctx); aerr == nil { - primary, mfaReads = auth.TwoFASecret, auth.TwoFAReadSecrets - } - } - if primary == "" { - primary = strings.TrimSpace(os.Getenv("TWOFA_SECRET")) - if primary == "" { - primary = strings.TrimSpace(os.Getenv("AUTH_SECRET")) - } - if mfaReads, err = secretset.Parse(os.Getenv("TWOFA_READ_SECRETS")); err != nil { - return nil, fmt.Errorf("TWOFA_READ_SECRETS: %w", err) - } - } - if primary != "" { - key := twofa.DeriveKey(primary) - k.mfa = &key - for _, s := range mfaReads { - if s != primary { - k.mfaRead = append(k.mfaRead, twofa.DeriveKey(s)) - } - } - } - return k, nil -} - -// scanColumn classifies every value in one column and, with write, reseals the read-key ones by compare-and-swap. -func scanColumn(ctx context.Context, c *conn, keys *keyring, col sealedColumn, size int, write bool) (columnCount, error) { - out := columnCount{sealedColumn: col} - var exists bool - if err := c.db.Pool.QueryRow(ctx, `SELECT to_regclass($1) IS NOT NULL`, col.Table).Scan(&exists); err != nil { - return out, fmt.Errorf("reading %s: %w", col.Table, err) - } - if !exists { - out.Missing = true - return out, nil - } - table, column := pgx.Identifier{col.Table}.Sanitize(), pgx.Identifier{col.Column}.Sanitize() - filter := "" - if col.guard != "" { - filter = " AND " + pgx.Identifier{col.guard}.Sanitize() - } - query := fmt.Sprintf(`SELECT ctid::text AS row_tid, %[2]s::text FROM %[1]s WHERE %[2]s IS NOT NULL AND %[2]s::text <> ''%[3]s AND ctid > $1::tid ORDER BY ctid LIMIT $2`, table, column, filter) - // The ctid and the old value together are the compare: a row rewritten meanwhile matches neither. - update := fmt.Sprintf(`UPDATE %[1]s SET %[2]s = $1 WHERE ctid = $2::tid AND %[2]s::text = $3`, table, column) - - // One snapshot for the reads, so a tuple this run rewrites is never read back. - snap, err := c.db.Pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) - if err != nil { - return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) - } - defer func() { _ = snap.Rollback(ctx) }() - - after := "(0,0)" - for { - rows, err := snap.Query(ctx, query, after, size) - if err != nil { - return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) - } - type row struct{ ctid, value string } - var page []row - for rows.Next() { - var r row - if err := rows.Scan(&r.ctid, &r.value); err != nil { - rows.Close() - return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) - } - page = append(page, r) - } - rows.Close() - if err := rows.Err(); err != nil { - return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) - } - for _, r := range page { - resealed, changed, err := resealValue(keys, col, r.value) - switch { - case errors.Is(err, errUnsealed): - out.Unsealed++ - continue - case errors.Is(err, errWorkspaceSealed): - out.Workspace++ - continue - case err != nil: - out.Unreadable++ - continue - case !changed: - out.Primary++ - continue - } - out.ReadKey++ - if !write { - continue - } - tag, err := c.db.Pool.Exec(ctx, update, resealed, r.ctid, r.value) - if err != nil { - return out, fmt.Errorf("resealing %s.%s: %w", col.Table, col.Column, err) - } - if tag.RowsAffected() == 1 { - out.Resealed++ - } else { - out.Raced++ - } - } - if len(page) < size { - return out, nil - } - after = page[len(page)-1].ctid - } -} - -var ( - errUnsealed = errors.New("value is stored outside the sealed format") - errWorkspaceSealed = errors.New("value is sealed under its workspace key") -) - -func resealValue(keys *keyring, col sealedColumn, value string) (string, bool, error) { - if col.Domain == domainMFA { - return twofa.Reseal(*keys.mfa, keys.mfaRead, value) - } - // A workspace-sealed value needs no instance key, so a rotation leaves it alone. - if mailcred.IsOrgSealed(value) { - return "", false, errWorkspaceSealed - } - if !encrypt.LooksSealed(value) { - return "", false, errUnsealed - } - return keys.instance.Reseal(value) -} - -// graceWindows reports every previous-credential window the instance is configured with. -func graceWindows(now time.Time) []graceWindow { - var out []graceWindow - for _, w := range []struct{ name, list string }{ - {"INTERNAL_API_PREVIOUS_UNTIL", "INTERNAL_API_PREVIOUS_TOKENS"}, - {"NODE_BROKER_PREVIOUS_UNTIL", "NODE_BROKER_PREVIOUS_TOKENS"}, - {"UNSUBSCRIBE_SIGNING_READ_UNTIL", "UNSUBSCRIBE_SIGNING_READ_SECRETS"}, - } { - raw := strings.TrimSpace(os.Getenv(w.name)) - entries, _ := secretset.Parse(os.Getenv(w.list)) - if raw == "" && len(entries) == 0 { - continue - } - until, _ := time.Parse(time.RFC3339, raw) - out = append(out, graceWindow{Name: w.name, Until: until, Open: now.Before(until), Entries: len(entries)}) - } - return out -} - -func printKeyReport(r keyReport, reseal bool) { +func printKeyReport(r keystatus.Report, reseal bool) { if reseal && !r.Applied { fmt.Print("Dry run: nothing will be written. Pass --apply to reseal.\n\n") } @@ -371,7 +114,7 @@ func printKeyReport(r keyReport, reseal bool) { } fmt.Printf("\n %-44s %8s %9s %10s %9s %11s\n", "column", "primary", "read key", "workspace", "unsealed", "unreadable") var onRead, unreadable, resealed, raced int - cols := append([]columnCount(nil), r.Columns...) + cols := append([]keystatus.Count(nil), r.Columns...) sort.SliceStable(cols, func(i, j int) bool { return cols[i].Domain < cols[j].Domain }) for _, c := range cols { label := c.Table + "." + c.Column diff --git a/cmd/warmblyctl/keys_test.go b/cmd/warmblyctl/keys_test.go index 1ef26c6b9..dc70a3c20 100644 --- a/cmd/warmblyctl/keys_test.go +++ b/cmd/warmblyctl/keys_test.go @@ -1,97 +1,6 @@ package main -import ( - "encoding/hex" - "errors" - "strings" - "testing" - - "github.com/warmbly/warmbly/internal/app/mailcred" - "github.com/warmbly/warmbly/internal/app/orgtransfer" - "github.com/warmbly/warmbly/internal/app/twofa" - "github.com/warmbly/warmbly/internal/pkg/encrypt" -) - -func TestSealedColumnsCoverEveryInstanceKeySecret(t *testing.T) { - got := map[string]keyDomain{} - for _, c := range sealedColumns() { - got[c.Table+"."+c.Column] = c.Domain - } - for _, want := range []string{ - "email_accounts_smtp_imap.smtp_password", - "email_accounts_smtp_imap.imap_password", - "email_accounts_oauth.access_token", - "email_accounts_oauth.refresh_token", - "webhook_endpoints.secret", - "oauth_applications.webhook_secret", - "cloud_link.token", - "mailbox_erasures.refresh_token", - } { - if got[want] != domainInstance { - t.Fatalf("%s must be scanned under the instance credential key", want) - } - } - if got["user_totp_settings.totp_secret_sealed"] != domainMFA { - t.Fatal("TOTP enrollments must be scanned under the MFA key") - } - - for _, c := range instanceOnlyColumns { - if _, ok := orgtransfer.ExcludedTables[c.Table]; c.Domain == domainInstance && !ok { - t.Fatalf("%s is listed as instance-only but is not in orgtransfer.ExcludedTables", c.Table) - } - } -} - -func TestResealValueClassifiesEveryShape(t *testing.T) { - oldKey, newKey := strings.Repeat("11", 32), strings.Repeat("22", 32) - t.Setenv("CREDENTIALS_ENCRYPTION_KEY", oldKey) - t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", "") - old, err := encrypt.FromEnv() - if err != nil { - t.Fatal(err) - } - legacy, err := old.Encrypt("smtp-password") - if err != nil { - t.Fatal(err) - } - t.Setenv("CREDENTIALS_ENCRYPTION_KEY", newKey) - t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", `["`+oldKey+`"]`) - current, err := encrypt.FromEnv() - if err != nil { - t.Fatal(err) - } - keys := &keyring{instance: current, instanceRead: 1} - col := sealedColumn{Table: "email_accounts_smtp_imap", Column: "smtp_password", Domain: domainInstance} - - resealed, changed, err := resealValue(keys, col, legacy) - if err != nil || !changed { - t.Fatalf("a read-key value must be resealed, got changed=%v err=%v", changed, err) - } - if _, changed, err := resealValue(keys, col, resealed); err != nil || changed { - t.Fatal("a resealed value must open under the primary") - } - if _, _, err := resealValue(keys, col, mailcred.OrgPrefix+"c2VhbGVk"); !errors.Is(err, errWorkspaceSealed) { - t.Fatal("a workspace-sealed value must be counted apart and never touched by an instance rotation") - } - if _, _, err := resealValue(keys, col, "ya29.a0Af-legacy-token"); !errors.Is(err, errUnsealed) { - t.Fatal("a value outside the sealed format must be reported unsealed, never rewritten") - } - foreign := hex.EncodeToString(make([]byte, 64)) - if _, _, err := resealValue(keys, col, foreign); err == nil || errors.Is(err, errUnsealed) { - t.Fatal("a sealed value no key opens must be reported unreadable") - } - - mfaOld, mfaNew := twofa.DeriveKey("old-mfa-secret"), twofa.DeriveKey("new-mfa-secret") - sealed, err := twofa.Seal(mfaOld, "JBSWY3DPEHPK3PXP") - if err != nil { - t.Fatal(err) - } - keys.mfa, keys.mfaRead = &mfaNew, [][32]byte{mfaOld} - mfa := sealedColumn{Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: domainMFA} - if _, changed, err := resealValue(keys, mfa, sealed); err != nil || !changed { - t.Fatal("an enrollment sealed under the MFA read key must be resealed") - } -} +import "testing" func TestGenerateRefusesUnknownNames(t *testing.T) { if err := runKeysGenerate([]string{"NOT_A_KEY"}); err == nil { diff --git a/cmd/warmblyctl/main.go b/cmd/warmblyctl/main.go index 88c69eed1..10df8b64c 100644 --- a/cmd/warmblyctl/main.go +++ b/cmd/warmblyctl/main.go @@ -135,6 +135,9 @@ var commands = []command{ {"fleet channel", "Follow stable or dev releases, or hold the fleet where it is", composeExec + "fleet channel stable"}, {"fleet pin", "Hold one node at a version, to canary or to hold it back", composeExec + "fleet pin v1.4.1"}, {"fleet remove", "Forget a node. Its mailboxes re-place themselves", composeExec + "fleet remove "}, + {"fleet rollout start", "Move the fleet to a tag in stages: one canary, then waves", composeExec + "fleet rollout start v1.4.2 --bake 30m"}, + {"fleet rollout status", "Show a staged rollout's stage, admitted nodes and halt reason", composeExec + "fleet rollout status"}, + {"fleet env-status", "Whether every live node has applied the current rendered env", composeExec + "fleet env-status"}, } func usage(w *os.File) { @@ -212,7 +215,8 @@ Exit status: and nothing at all when none fire. status --json always exits 0, because it doubles as a liveness probe; read .summary.error there instead. keys status and keys reseal exit non-zero when a stored value opens under no - configured key, which is the signal not to retire one. + configured key, which is the signal not to retire one. keys status + --require-primary also fails while a value still needs a read key. Run `+"`warmblyctl --help`"+` for one command's flags. `) diff --git a/cmd/warmblyctl/remote_catalog.go b/cmd/warmblyctl/remote_catalog.go index 32694af2f..a54ceb08c 100644 --- a/cmd/warmblyctl/remote_catalog.go +++ b/cmd/warmblyctl/remote_catalog.go @@ -101,6 +101,7 @@ var remoteAdminRoutes = []remoteAdminRoute{ {"POST", "/admin/campaigns/:id/stop", []string{"StopCampaigns"}, false, ""}, {"GET", "/admin/system/status", []string{"ViewAnalytics"}, false, ""}, {"GET", "/admin/instance/config", []string{"ManageSettings"}, false, ""}, + {"GET", "/admin/keys/status", []string{"ManageSettings"}, false, ""}, {"GET", "/admin/instance/health", []string{"ViewAnalytics"}, false, ""}, {"GET", "/admin/instance/monitoring", []string{"ViewAnalytics"}, false, ""}, {"DELETE", "/admin/instance/invitations/expired", []string{"ManageOrganizations"}, false, ""}, diff --git a/deploy/README.md b/deploy/README.md index c1a75c534..7775eba03 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -77,7 +77,7 @@ cloud region. Its README lists what is in it. ## Deploying the control plane -The Dockerfiles in `deploy/docker/` are the deployment unit. Production runs on Railway. Other valid targets: Fly.io, ECS Fargate, single-VPS systemd. Migrations run automatically on backend boot. +The Dockerfiles in `deploy/docker/` are the deployment unit. The hosted service runs on ECS Fargate (`terraform/aws/`), where a stable release tag deploys itself through `.github/workflows/deploy-aws.yml` once the `AWS_DEPLOY_ENABLED` repository variable is `true`: the migration runs as its own task first and backend startup migrations are off. Other valid targets: Fly.io, Coolify, single-VPS systemd. A self-host runs migrations automatically on backend boot. Configuration is env-driven — see `deploy/config/env.example` for the full env reference, or [the self-hosting guide](https://docs.warmbly.com/development/deployment-guide/) for a step-by-step. @@ -142,7 +142,7 @@ Two settings do not survive a fleet on their defaults. `BLOB_PROVIDER=filesystem ## Auto-update -`internal/app/releases` resolves the head of the configured channel and writes the tag to `admin_settings`; it updates nothing itself. The heartbeat reply carries `desired_version`, the node writes it to a file, and a systemd timer pulls and restarts, so the process being replaced is never the process doing the replacing. A per-node `pinned_version` holds a machine back for canarying. +`internal/app/releases` resolves the head of the configured channel and writes the tag to `admin_settings`; it updates nothing itself. The heartbeat reply carries `desired_version`, the node writes it to a file, and a systemd timer pulls and restarts, so the process being replaced is never the process doing the replacing. A per-node `pinned_version` holds a machine back for canarying. `warmblyctl fleet rollout` moves the fleet in a canary and waves instead of all at once and halts on the first unhealthy node; the updater puts a node back on its previous image when an update does not stay up, and, with `WARMBLY_ENV_REFRESH=true`, replaces `node.env` when the control plane's rendering changes. The backend is deliberately excluded: it is what tells everyone else their version. See [the self-hosting guide](https://docs.warmbly.com/development/deployment-guide/). diff --git a/deploy/terraform/aws/runtime/README.md b/deploy/terraform/aws/runtime/README.md index 996e3b0c5..37e8d1e8d 100644 --- a/deploy/terraform/aws/runtime/README.md +++ b/deploy/terraform/aws/runtime/README.md @@ -18,21 +18,35 @@ Start at desired count zero. Separate execution roles retrieve each service's configuration and write its logs. Application roles receive reviewed existing permissions or IAM Policy Autopilot output through `application_policies`. -## Migrate before deployment +## Deploy releases -Terraform registers the application and migration task definitions. Service -revision and desired-count changes are deployment operations; Terraform ignores -those two fields after service creation. The workflow handles backend and consumer -rollouts. Forms, realtime and tracking cutovers use separate explicit ECS service -updates after their own checks. Start new services at zero and use explicit -counts when activating them. Terraform output task definition ARNs describe -prepared revisions, not necessarily running revisions. +Terraform owns each task definition's shape: environment, secret references, +sizes, health checks and the `warmbly-image-variant` tag that says whether a +service runs the default or the `-kafka` image. Service revision and desired +count are deployment operations; Terraform ignores those two fields after +service creation, and its task definition outputs describe prepared revisions, +not necessarily running ones. Start new services at zero and use explicit counts +when activating them. -Set `github_repository` to the repository running `.github/workflows/deploy-aws.yml`. -The root creates an OIDC provider and a deployment role scoped to the migration -task and backend/consumer services. Import an existing account-wide GitHub OIDC -provider instead of creating a second one. Restrict the `aws-production` GitHub -environment to trusted deployment branches. Set its variables: +`.github/workflows/deploy-aws.yml` owns only the image. A stable release tag on +`main` reaches it from `release.yml` when the repository variable +`AWS_DEPLOY_ENABLED` is `true`; dispatch it by hand to redeploy, roll back or +roll out changed secret values. Each run verifies the release's build +provenance, renders every service from its latest Terraform revision with the +release's digests, runs the dependency validation and migration tasks, then +rolls backend, consumer, and forms, tracking and realtime in waves. Each service +is watched against its CloudWatch alarms for a bake after it is stable, and any +failure restores every service already moved, in reverse. Workers and consumers +on hosts follow through a canary rollout started from the `ops` task. The +procedure, modes and one-time GitHub setup are in +`docs/content/docs/development/aws-fargate.mdx`. + +Set `github_repository` to the repository running the workflows. The root +creates an OIDC provider and a deployment role that can register task +definition revisions, run the validation, migration, `ops` and rotator tasks, +update the five services and read secret metadata, never a secret value. Import +an existing account-wide GitHub OIDC provider instead of creating a second one. +Set the `aws-production` environment's variables: | Variable | Value | | --- | --- | @@ -46,16 +60,32 @@ workflow never retrieves database credentials. Its backend and migration tasks must use the same immutable image digest and direct secret reference. Keep the default `migration_secret_key=PRIMARY_DB_DIRECT` for workflow deployments. -Dispatch the workflow with explicit Terraform task definition revisions for -migration, backend and consumer. It runs `/app/migrate` inside the VPC over the -existing PrivateLink route. An unsuccessful or unresolved migration blocks all -service updates. A successful migration permits sequential backend and consumer -rollouts, verifying that the requested revision becomes stable. A rollout failure -restores the previous service revisions and counts, retaining the current schema. -Inspect rollback health separately. The workflow never runs down migrations. -Task definitions use `skip_destroy` so earlier revisions remain active for -rollback. Retain their referenced images and secret versions throughout the -rollback window; retire them separately after it closes. +The migration runs `/app/migrate` inside the VPC over the existing PrivateLink +route, and an unsuccessful or unresolved migration blocks every service update. +A rollback skips it, because schemas stay compatible with the previous release. +The workflow never runs down migrations. Task definitions use `skip_destroy` so +earlier revisions remain active for rollback. Retain their referenced images and +secret versions throughout the rollback window; retire them separately after it +closes. + +## Rotate keys + +`rotation.tf` registers a rotator task that generates replacement values with +its own narrow role, writes them as new secret versions and reports only +version ids. `.github/workflows/rotate-keys.yml` drives it, rolls services onto +those versions and runs `warmblyctl` in the `ops` task to prove each step before +the old value is retired. No value passes through GitHub or a log. The runbook +is in `aws-fargate.mdx`. + +## Terraform in CI + +Set `terraform_ci = true` to create the `-terraform-plan` and +`-terraform-apply` roles `.github/workflows/terraform-aws.yml` uses. The +plan role reads configuration and is denied secret values, decryption, logs and +data reads; the apply role administers resources named for this deployment and +is reachable only from `main` through protected environments. Var files stay in +the state bucket under `warmbly/production/vars/`; only resource addresses and +actions are ever printed. The runtime sets `DB_MIGRATIONS_ON_START=false` for the backend. New application images must support that setting and schema readiness before activation. Keep diff --git a/deploy/terraform/aws/runtime/ci.tf b/deploy/terraform/aws/runtime/ci.tf index 519cda795..3dfca5903 100644 --- a/deploy/terraform/aws/runtime/ci.tf +++ b/deploy/terraform/aws/runtime/ci.tf @@ -43,23 +43,51 @@ resource "aws_iam_role" "github_deploy" { max_session_duration = 3600 } +locals { + ecs_arn = "arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}" + # One-off tasks a deploy or a rotation runs, and the roles they pass. + github_tasks = { + migration = [aws_iam_role.migration_execution.arn] + validation = [aws_iam_role.validation_execution.arn, aws_iam_role.application["backend"].arn] + ops = [aws_iam_role.execution["backend"].arn, aws_iam_role.application["backend"].arn] + rotator = [aws_iam_role.rotator_execution.arn, aws_iam_role.rotator.arn] + } + github_log_groups = concat( + [aws_cloudwatch_log_group.migration.arn, aws_cloudwatch_log_group.validation.arn, aws_cloudwatch_log_group.rotator.arn], + ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:log-group:/ecs/${var.name}/backend"], + ) +} + data "aws_iam_policy_document" "github_deploy" { count = var.github_repository == null ? 0 : 1 statement { - actions = ["ecs:DescribeTaskDefinition"] + actions = ["ecs:DescribeTaskDefinition", "ecs:RegisterTaskDefinition"] resources = ["*"] } + statement { + actions = ["ecs:TagResource"] + resources = ["${local.ecs_arn}:task-definition/${var.name}-*:*"] + condition { + test = "StringEquals" + variable = "ecs:CreateAction" + values = ["RegisterTaskDefinition"] + } + } + statement { + actions = ["ecs:ListTagsForResource"] + resources = ["${local.ecs_arn}:task-definition/${var.name}-*:*"] + } statement { actions = ["ecs:DescribeServices", "ecs:UpdateService"] - resources = [for service in ["backend", "consumer"] : "arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:service/${var.name}/${var.name}-${service}"] + resources = [for service in keys(var.services) : "${local.ecs_arn}:service/${var.name}/${var.name}-${service}"] } statement { actions = ["ecs:DescribeTasks"] - resources = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:task/${var.name}/*"] + resources = ["${local.ecs_arn}:task/${var.name}/*"] } statement { actions = ["ecs:RunTask"] - resources = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:task-definition/${var.name}-migration:*"] + resources = [for task in keys(local.github_tasks) : "${local.ecs_arn}:task-definition/${var.name}-${task}:*"] condition { test = "ArnEquals" variable = "ecs:cluster" @@ -68,15 +96,28 @@ data "aws_iam_policy_document" "github_deploy" { } statement { actions = ["iam:PassRole"] - resources = concat([aws_iam_role.migration_execution.arn], flatten([ - for name in ["backend", "consumer"] : [aws_iam_role.execution[name].arn, aws_iam_role.application[name].arn] - ])) + resources = distinct(concat(flatten(values(local.github_tasks)), flatten([ + for name in keys(var.services) : [aws_iam_role.execution[name].arn, aws_iam_role.application[name].arn] + ]))) condition { test = "StringEquals" variable = "iam:PassedToService" values = ["ecs-tasks.amazonaws.com"] } } + statement { + actions = ["logs:GetLogEvents"] + resources = [for group in local.github_log_groups : "${group}:log-stream:*"] + } + statement { + actions = ["cloudwatch:DescribeAlarms"] + resources = ["*"] + } + # Version metadata only: the deploy repins references and never reads a value. + statement { + actions = ["secretsmanager:DescribeSecret", "secretsmanager:ListSecretVersionIds"] + resources = [for name in keys(var.services) : local.foundation.secret_arns[name]] + } } resource "aws_iam_role_policy" "github_deploy" { diff --git a/deploy/terraform/aws/runtime/main.tf b/deploy/terraform/aws/runtime/main.tf index afae9ec2f..314c1be07 100644 --- a/deploy/terraform/aws/runtime/main.tf +++ b/deploy/terraform/aws/runtime/main.tf @@ -60,10 +60,13 @@ variable "services" { secret_version_id = string secret_keys = set(string) environment = optional(map(string), {}) + # kafka when the service runs the -kafka build; the deploy reads it off the task definition tag. + image_variant = optional(string, "default") })) validation { condition = (var.github_repository == null || alltrue([for name in ["backend", "consumer"] : contains(keys(var.services), name)])) && alltrue([for name, service in var.services : contains(["backend", "consumer", "forms", "realtime", "tracking"], name) && + contains(["default", "kafka"], service.image_variant) && can(regex("@sha256:[a-f0-9]{64}$", service.image)) && ((service.cpu == 256 && contains([512, 1024, 2048], service.memory)) || (service.cpu == 512 && contains([1024, 2048, 3072, 4096], service.memory)) || @@ -343,6 +346,7 @@ resource "aws_ecs_task_definition" "service" { operating_system_family = "LINUX" } container_definitions = jsonencode([local.service_containers[each.key]]) + tags = { "warmbly-image-variant" = each.value.image_variant } depends_on = [aws_iam_role_policy.execution, aws_iam_role_policy.application] } resource "aws_ecs_service" "service" { @@ -365,6 +369,14 @@ resource "aws_ecs_service" "service" { enable = true rollback = true } + dynamic "alarms" { + for_each = contains(keys(local.public_services), each.key) ? [1] : [] + content { + alarm_names = [aws_cloudwatch_metric_alarm.target_5xx[each.key].alarm_name, aws_cloudwatch_metric_alarm.unhealthy_hosts[each.key].alarm_name] + enable = true + rollback = true + } + } network_configuration { subnets = local.foundation.public_subnet_ids security_groups = [local.foundation.tasks_sg_id] @@ -385,6 +397,58 @@ resource "aws_ecs_service" "service" { depends_on = [aws_lb_listener_rule.service] } +# Named ${var.name}--*: the deploy bakes each service against every alarm with that prefix. +resource "aws_cloudwatch_metric_alarm" "target_5xx" { + for_each = local.public_services + alarm_name = "${var.name}-${each.key}-target-5xx-rate" + alarm_description = "More than 5% of requests to ${each.key} answered 5xx" + comparison_operator = "GreaterThanThreshold" + threshold = 5 + evaluation_periods = 3 + datapoints_to_alarm = 2 + treat_missing_data = "notBreaching" + metric_query { + id = "rate" + expression = "IF(requests >= 20, 100 * errors / requests, 0)" + label = "5xx percent" + return_data = true + } + metric_query { + id = "errors" + metric { + namespace = "AWS/ApplicationELB" + metric_name = "HTTPCode_Target_5XX_Count" + period = 60 + stat = "Sum" + dimensions = { LoadBalancer = aws_lb.main.arn_suffix, TargetGroup = aws_lb_target_group.service[each.key].arn_suffix } + } + } + metric_query { + id = "requests" + metric { + namespace = "AWS/ApplicationELB" + metric_name = "RequestCount" + period = 60 + stat = "Sum" + dimensions = { LoadBalancer = aws_lb.main.arn_suffix, TargetGroup = aws_lb_target_group.service[each.key].arn_suffix } + } + } +} +resource "aws_cloudwatch_metric_alarm" "unhealthy_hosts" { + for_each = local.public_services + alarm_name = "${var.name}-${each.key}-unhealthy-hosts" + alarm_description = "A ${each.key} target has failed its load balancer health check for three minutes" + namespace = "AWS/ApplicationELB" + metric_name = "UnHealthyHostCount" + dimensions = { LoadBalancer = aws_lb.main.arn_suffix, TargetGroup = aws_lb_target_group.service[each.key].arn_suffix } + statistic = "Maximum" + period = 60 + evaluation_periods = 3 + comparison_operator = "GreaterThanOrEqualToThreshold" + threshold = 1 + treat_missing_data = "notBreaching" +} + output "runtime" { value = { cluster_arn = local.foundation.cluster_arn diff --git a/deploy/terraform/aws/runtime/migrations.tf b/deploy/terraform/aws/runtime/migrations.tf index 3b085e78c..b5d852b75 100644 --- a/deploy/terraform/aws/runtime/migrations.tf +++ b/deploy/terraform/aws/runtime/migrations.tf @@ -77,6 +77,7 @@ resource "aws_ecs_task_definition" "migration" { } } }]) + tags = { "warmbly-image-variant" = var.services["backend"].image_variant } depends_on = [aws_iam_role_policy.migration_execution] } @@ -134,3 +135,31 @@ resource "aws_ecs_task_definition" "realtime_database_check" { output "realtime_database_check_task_definition" { value = aws_ecs_task_definition.realtime_database_check.arn } + +# warmblyctl with the backend's configuration, for the commands a deploy or rotation runs. +resource "aws_ecs_task_definition" "ops" { + skip_destroy = true + family = "${var.name}-ops" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = "256" + memory = "512" + execution_role_arn = aws_iam_role.execution["backend"].arn + task_role_arn = aws_iam_role.application["backend"].arn + runtime_platform { + cpu_architecture = "ARM64" + operating_system_family = "LINUX" + } + container_definitions = jsonencode([merge({ + for key, value in local.service_containers["backend"] : key => value if !contains(["portMappings", "healthCheck"], key) + }, { + name = "ops" + entryPoint = ["warmblyctl"] + command = ["status"] + })]) + tags = { "warmbly-image-variant" = var.services["backend"].image_variant } +} + +output "ops_task_definition" { + value = aws_ecs_task_definition.ops.arn +} diff --git a/deploy/terraform/aws/runtime/rotate/rotate.cjs b/deploy/terraform/aws/runtime/rotate/rotate.cjs new file mode 100644 index 000000000..68cb0da22 --- /dev/null +++ b/deploy/terraform/aws/runtime/rotate/rotate.cjs @@ -0,0 +1,195 @@ +// Runs inside AWS as a one-off task. Generates replacement keys, writes them +// into the service secrets as new versions, and keeps the previous values +// readable until they are retired. It logs key names, phases and version ids, +// never a value; the last line is ROTATION_RESULT with the same. +const { createHash, randomBytes } = require("node:crypto"); + +const DAY = 24 * 60 * 60 * 1000; +// kind: seal = stored ciphertext, sign = issued links, token = shared bearer token. +const KEYS = { + CREDENTIALS_ENCRYPTION_KEY: { kind: "seal", read: "CREDENTIALS_ENCRYPTION_READ_KEYS", introduce: true }, + TWOFA_SECRET: { kind: "seal", read: "TWOFA_READ_SECRETS" }, + UNSUBSCRIBE_SIGNING_SECRET: { kind: "sign", read: "UNSUBSCRIBE_SIGNING_READ_SECRETS", until: "UNSUBSCRIBE_SIGNING_READ_UNTIL", window: 366 * DAY }, + INTERNAL_API_TOKEN: { kind: "token", read: "INTERNAL_API_PREVIOUS_TOKENS", until: "INTERNAL_API_PREVIOUS_UNTIL", window: 7 * DAY, receivers: ["backend"] }, + NODE_BROKER_TOKEN: { kind: "token", read: "NODE_BROKER_PREVIOUS_TOKENS", until: "NODE_BROKER_PREVIOUS_UNTIL", window: 7 * DAY, receivers: ["backend"] }, +}; +const MAX_READ = 4; + +class RotationError extends Error {} + +function list(raw) { + if (raw === undefined || raw === null || String(raw).trim() === "") return []; + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed) || parsed.some((v) => typeof v !== "string" || !v)) throw new RotationError("a read list is not a JSON array of strings"); + return parsed; +} + +function withRead(values, add, drop = []) { + const out = [...add, ...values.filter((v) => !add.includes(v) && !drop.includes(v))]; + if (out.length > MAX_READ) throw new RotationError(`the read list would hold more than ${MAX_READ} values; retire the previous rotation first`); + return JSON.stringify(out); +} + +const fresh = () => randomBytes(32).toString("hex"); + +function makeStore(client, commands) { + const { GetSecretValueCommand, PutSecretValueCommand } = commands; + return { + async read(arn, versionId) { + try { + const out = await client.send(new GetSecretValueCommand({ SecretId: arn, ...(versionId ? { VersionId: versionId } : {}) })); + return { versionId: out.VersionId, value: out.SecretString ? JSON.parse(out.SecretString) : {} }; + } catch (error) { + if (error?.name === "ResourceNotFoundException") return { versionId: null, value: {} }; + throw error; + } + }, + async write(arn, value, token) { + const out = await client.send(new PutSecretValueCommand({ SecretId: arn, SecretString: JSON.stringify(value), ClientRequestToken: token })); + return out.VersionId; + }, + }; +} + +async function rotate({ store, action, key, secrets, stateArn, run = "local", now = Date.now(), log = console.log }) { + const spec = KEYS[key]; + if (action !== "status" && !spec) throw new RotationError(`unknown key ${key}; expected one of ${Object.keys(KEYS).join(", ")}`); + const state = (await store.read(stateArn)).value; + const versions = {}; + const token = (service) => createHash("sha256").update(`${run}/${action}/${key}/${state.phase || "none"}/${service}`).digest("hex").replace(/^(.{8})(.{4})(.{4})(.{4})(.{12}).*/, "$1-$2-$3-$4-$5"); + + // holders are the services whose secret carries the key at all. + const loaded = {}; + for (const [service, arn] of Object.entries(secrets)) loaded[service] = await store.read(arn); + const holders = spec ? Object.keys(loaded).filter((s) => typeof loaded[s].value[key] === "string" && loaded[s].value[key]) : []; + if (spec && !holders.length && action !== "status") throw new RotationError(`no service secret holds ${key}`); + const readers = (s) => !spec.receivers || spec.receivers.includes(s); + + const save = async (service, value) => { + versions[service] = await store.write(secrets[service], value, token(service)); + log(`${service}: wrote secret version ${versions[service]}`); + }; + const saveState = async (next) => { + await store.write(stateArn, next, token("state")); + log(`rotation state: ${next.key ? `${next.key} ${next.phase}` : "clear"}`); + }; + const busy = state.key && state.key !== key; + + switch (action) { + case "status": + log(state.key ? `${state.key} is ${state.phase} since ${state.updatedAt}` : "no rotation in progress"); + return { phase: state.phase || "none", key: state.key || null, versions, pinned: state.versions || {}, until: state.until || null }; + + case "prepare": { + // Overlap keys must exist before Terraform may reference them. + for (const service of holders) { + const value = { ...loaded[service].value }; + let changed = false; + if (readers(service) && value[spec.read] === undefined) { value[spec.read] = "[]"; changed = true; } + if (readers(service) && spec.until && value[spec.until] === undefined) { value[spec.until] = ""; changed = true; } + if (changed) await save(service, value); + else log(`${service}: already carries the overlap keys`); + } + return { phase: state.key === key ? state.phase : "none", versions, holders }; + } + + case "stage": { + if (busy) throw new RotationError(`${state.key} is mid-rotation; finish or abort it first`); + for (const service of holders) { + if (readers(service) && loaded[service].value[spec.read] === undefined) { + throw new RotationError(`${service} has no ${spec.read}; run prepare and add it to that service's Terraform secret_keys first`); + } + } + const before = state.before || Object.fromEntries(holders.map((s) => [s, loaded[s].versionId])); + if (!state.key && spec.introduce) { + // Every reader learns the new key before anything writes with it. + const pending = fresh(); + for (const service of holders) { + const value = { ...loaded[service].value }; + value[spec.read] = withRead(list(value[spec.read]), [pending]); + await save(service, value); + } + await saveState({ key, phase: "introduced", pending, previous: loaded[holders[0]].value[key], before, versions, updatedAt: new Date(now).toISOString() }); + return { phase: "introduced", versions, holders }; + } + if (state.key && state.phase !== "introduced") throw new RotationError(`${key} is already ${state.phase}; retire or abort it`); + const next = state.pending || fresh(); + const previous = state.previous || loaded[holders[0]].value[key]; + if (holders.some((s) => loaded[s].value[key] !== previous)) throw new RotationError(`services disagree on the current ${key}; align them before rotating`); + const until = spec.until ? new Date(now + spec.window).toISOString() : null; + for (const service of holders) { + const value = { ...loaded[service].value, [key]: next }; + if (readers(service)) { + value[spec.read] = withRead(list(value[spec.read]), [previous], [next]); + if (until) value[spec.until] = until; + } + await save(service, value); + } + await saveState({ key, phase: "staged", previous, before, until, versions, updatedAt: new Date(now).toISOString() }); + return { phase: "staged", versions, holders, until }; + } + + case "retire": { + if (state.key !== key || state.phase !== "staged") throw new RotationError(`${key} has no staged rotation to retire`); + if (state.until && spec.kind === "sign" && now < Date.parse(state.until)) { + throw new RotationError(`links signed with the previous ${key} stay valid until ${state.until}; retire after that`); + } + for (const service of holders) { + const value = { ...loaded[service].value }; + if (!readers(service)) continue; + const remaining = list(value[spec.read]).filter((v) => v !== state.previous); + value[spec.read] = JSON.stringify(remaining); + if (spec.until && !remaining.length) value[spec.until] = ""; + await save(service, value); + } + await saveState({}); + return { phase: "none", versions, holders }; + } + + case "abort": { + if (state.key !== key) throw new RotationError(`no rotation of ${key} is in progress`); + for (const service of holders) { + const value = { ...loaded[service].value }; + if (state.phase === "introduced") { + if (!readers(service)) continue; + value[spec.read] = withRead(list(value[spec.read]), [], [state.pending]); + } else { + const replacement = value[key]; + value[key] = state.previous; + if (readers(service)) { + // A token is forgotten; a key stays readable because something may already be sealed or signed with it. + const keep = spec.kind === "token" ? [] : [replacement]; + value[spec.read] = withRead(list(value[spec.read]), keep, [state.previous, replacement]); + if (spec.until && !keep.length && !list(value[spec.read]).length) value[spec.until] = ""; + } + } + await save(service, value); + } + await saveState({}); + return { phase: "none", versions, holders }; + } + + default: + throw new RotationError(`unknown action ${action}`); + } +} + +if (require.main === module) { + const sdk = require("@aws-sdk/client-secrets-manager"); + const client = new sdk.SecretsManagerClient({ region: process.env.AWS_REGION || "us-east-1" }); + rotate({ + store: makeStore(client, sdk), + action: process.env.ROTATE_ACTION, + key: process.env.ROTATE_KEY, + secrets: JSON.parse(process.env.SERVICE_SECRET_ARNS || "{}"), + stateArn: process.env.ROTATION_SECRET_ARN, + run: process.env.ROTATE_RUN || "local", + }).then((result) => { + console.log(`ROTATION_RESULT ${JSON.stringify(result)}`); + }).catch((error) => { + console.error(error instanceof RotationError ? error.message : `rotation failed: ${error?.name || "error"}`); + process.exitCode = 1; + }); +} + +module.exports = { rotate, makeStore, KEYS, RotationError }; diff --git a/deploy/terraform/aws/runtime/rotate/rotate.test.cjs b/deploy/terraform/aws/runtime/rotate/rotate.test.cjs new file mode 100644 index 000000000..22fff3503 --- /dev/null +++ b/deploy/terraform/aws/runtime/rotate/rotate.test.cjs @@ -0,0 +1,139 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const { rotate, makeStore } = require("./rotate.cjs"); + +const OLD = "o".repeat(64); +const STATE = "arn:state"; + +function memory(initial) { + const secrets = {}; + let n = 0; + for (const [arn, value] of Object.entries(initial)) secrets[arn] = [{ id: `v${n++}`, value: structuredClone(value) }]; + const tokens = new Set(); + const client = { async send(command) { + const name = command.constructor.name; + const versions = secrets[command.input.SecretId]; + if (name === "GetSecretValueCommand") { + if (!versions) throw Object.assign(new Error("missing"), { name: "ResourceNotFoundException" }); + const v = command.input.VersionId ? versions.find((x) => x.id === command.input.VersionId) : versions.at(-1); + return { VersionId: v.id, SecretString: JSON.stringify(v.value) }; + } + if (name === "PutSecretValueCommand") { + assert.ok(!tokens.has(command.input.ClientRequestToken), "request tokens are unique per write"); + assert.match(command.input.ClientRequestToken, /^[0-9a-f-]{36}$/); + tokens.add(command.input.ClientRequestToken); + const id = `v${n++}`; + (secrets[command.input.SecretId] ||= []).push({ id, value: JSON.parse(command.input.SecretString) }); + return { VersionId: id }; + } + throw new Error(name); + } }; + class GetSecretValueCommand { constructor(input) { this.input = input; } } + class PutSecretValueCommand { constructor(input) { this.input = input; } } + return { store: makeStore(client, { GetSecretValueCommand, PutSecretValueCommand }), current: (arn) => secrets[arn]?.at(-1).value }; +} + +function setup(extra = {}) { + const backend = { CREDENTIALS_ENCRYPTION_KEY: OLD, CREDENTIALS_ENCRYPTION_READ_KEYS: "[]", INTERNAL_API_TOKEN: OLD, INTERNAL_API_PREVIOUS_TOKENS: "[]", INTERNAL_API_PREVIOUS_UNTIL: "", UNSUBSCRIBE_SIGNING_SECRET: OLD, UNSUBSCRIBE_SIGNING_READ_SECRETS: "", UNSUBSCRIBE_SIGNING_READ_UNTIL: "", PRIMARY_DB: "postgres://fixture", ...extra.backend }; + const consumer = { CREDENTIALS_ENCRYPTION_KEY: OLD, CREDENTIALS_ENCRYPTION_READ_KEYS: "[]", ...extra.consumer }; + const tracking = { INTERNAL_API_TOKEN: OLD, ...extra.tracking }; + const m = memory({ "arn:backend": backend, "arn:consumer": consumer, "arn:tracking": tracking }); + const logs = []; + const call = (action, key, run = "r1", now = Date.parse("2026-10-24T00:00:00Z")) => rotate({ + store: m.store, action, key, run: `${run}/${action}/${Math.random()}`, now, stateArn: STATE, + secrets: { backend: "arn:backend", consumer: "arn:consumer", tracking: "arn:tracking" }, log: (l) => logs.push(l), + }); + return { ...m, logs, call }; +} + +test("the credentials key is introduced to every reader before it becomes primary, then retired", async () => { + const t = setup(); + const one = await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY"); + assert.equal(one.phase, "introduced"); + assert.deepEqual(Object.keys(one.versions).sort(), ["backend", "consumer"]); + const [pending] = JSON.parse(t.current("arn:backend").CREDENTIALS_ENCRYPTION_READ_KEYS); + assert.match(pending, /^[0-9a-f]{64}$/); + assert.equal(t.current("arn:backend").CREDENTIALS_ENCRYPTION_KEY, OLD); + assert.deepEqual(JSON.parse(t.current("arn:consumer").CREDENTIALS_ENCRYPTION_READ_KEYS), [pending]); + + const two = await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY"); + assert.equal(two.phase, "staged"); + for (const arn of ["arn:backend", "arn:consumer"]) { + assert.equal(t.current(arn).CREDENTIALS_ENCRYPTION_KEY, pending); + assert.deepEqual(JSON.parse(t.current(arn).CREDENTIALS_ENCRYPTION_READ_KEYS), [OLD]); + } + await assert.rejects(t.call("stage", "INTERNAL_API_TOKEN"), /mid-rotation/); + await t.call("retire", "CREDENTIALS_ENCRYPTION_KEY"); + assert.deepEqual(JSON.parse(t.current("arn:backend").CREDENTIALS_ENCRYPTION_READ_KEYS), []); + assert.equal(t.current("arn:backend").PRIMARY_DB, "postgres://fixture"); + assert.deepEqual(t.current(STATE), {}); +}); + +test("a token keeps the previous value only on its receiver, with a window", async () => { + const t = setup(); + const out = await t.call("stage", "INTERNAL_API_TOKEN"); + const fresh = t.current("arn:backend").INTERNAL_API_TOKEN; + assert.notEqual(fresh, OLD); + assert.equal(t.current("arn:tracking").INTERNAL_API_TOKEN, fresh); + assert.deepEqual(JSON.parse(t.current("arn:backend").INTERNAL_API_PREVIOUS_TOKENS), [OLD]); + assert.equal(t.current("arn:backend").INTERNAL_API_PREVIOUS_UNTIL, "2026-10-31T00:00:00.000Z"); + assert.equal(t.current("arn:tracking").INTERNAL_API_PREVIOUS_TOKENS, undefined); + assert.deepEqual(Object.keys(out.versions).sort(), ["backend", "tracking"]); + await t.call("retire", "INTERNAL_API_TOKEN"); + assert.equal(t.current("arn:backend").INTERNAL_API_PREVIOUS_TOKENS, "[]"); + assert.equal(t.current("arn:backend").INTERNAL_API_PREVIOUS_UNTIL, ""); +}); + +test("aborting a token restores the old value everywhere and forgets the new one", async () => { + const t = setup(); + await t.call("stage", "INTERNAL_API_TOKEN"); + await t.call("abort", "INTERNAL_API_TOKEN"); + assert.equal(t.current("arn:backend").INTERNAL_API_TOKEN, OLD); + assert.equal(t.current("arn:tracking").INTERNAL_API_TOKEN, OLD); + assert.deepEqual(JSON.parse(t.current("arn:backend").INTERNAL_API_PREVIOUS_TOKENS), []); +}); + +test("aborting a staged key keeps the replacement readable", async () => { + const t = setup(); + await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY"); + await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY"); + const replacement = t.current("arn:backend").CREDENTIALS_ENCRYPTION_KEY; + await t.call("abort", "CREDENTIALS_ENCRYPTION_KEY"); + assert.equal(t.current("arn:backend").CREDENTIALS_ENCRYPTION_KEY, OLD); + assert.deepEqual(JSON.parse(t.current("arn:consumer").CREDENTIALS_ENCRYPTION_READ_KEYS), [replacement]); +}); + +test("a signing secret cannot be retired while links signed with it are valid", async () => { + const t = setup(); + await t.call("stage", "UNSUBSCRIBE_SIGNING_SECRET"); + await assert.rejects(t.call("retire", "UNSUBSCRIBE_SIGNING_SECRET"), /stay valid until 2027-10-25/); + await t.call("retire", "UNSUBSCRIBE_SIGNING_SECRET", "r2", Date.parse("2027-10-26T00:00:00Z")); + assert.equal(t.current("arn:backend").UNSUBSCRIBE_SIGNING_READ_UNTIL, ""); +}); + +test("prepare adds empty overlap keys once, and stage refuses without them", async () => { + const t = setup({ backend: { TWOFA_SECRET: OLD } }); + await assert.rejects(t.call("stage", "TWOFA_SECRET"), /run prepare/); + const first = await t.call("prepare", "TWOFA_SECRET"); + assert.deepEqual(Object.keys(first.versions), ["backend"]); + assert.equal(t.current("arn:backend").TWOFA_READ_SECRETS, "[]"); + const again = await t.call("prepare", "TWOFA_SECRET"); + assert.deepEqual(again.versions, {}); +}); + +test("a full read list refuses instead of dropping a key", async () => { + const full = JSON.stringify(["a", "b", "c", "d"].map((c) => c.repeat(64))); + const t = setup({ backend: { CREDENTIALS_ENCRYPTION_READ_KEYS: full } }); + await assert.rejects(t.call("stage", "CREDENTIALS_ENCRYPTION_KEY"), /more than 4/); +}); + +test("no generated or stored value reaches the log or the result", async () => { + const t = setup(); + const results = []; + results.push(await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY")); + results.push(await t.call("stage", "CREDENTIALS_ENCRYPTION_KEY")); + results.push(await t.call("stage", "INTERNAL_API_TOKEN").catch((e) => e.message)); + const secrets = [OLD, t.current("arn:backend").CREDENTIALS_ENCRYPTION_KEY, "postgres://fixture"]; + const text = t.logs.join("\n") + JSON.stringify(results); + for (const s of secrets) assert.ok(!text.includes(s)); +}); diff --git a/deploy/terraform/aws/runtime/rotation.tf b/deploy/terraform/aws/runtime/rotation.tf new file mode 100644 index 000000000..bb87b77d1 --- /dev/null +++ b/deploy/terraform/aws/runtime/rotation.tf @@ -0,0 +1,74 @@ +# The rotator generates replacement keys inside AWS and writes them as new +# secret versions. Nothing outside this task ever holds a value; callers see +# version ids only. +resource "aws_cloudwatch_log_group" "rotator" { + name = "/ecs/${var.name}/rotator" + retention_in_days = 30 +} +resource "aws_iam_role" "rotator_execution" { + name = "${var.name}-rotator-execution" + assume_role_policy = data.aws_iam_policy_document.trust.json +} +resource "aws_iam_role" "rotator" { + name = "${var.name}-rotator-task" + assume_role_policy = data.aws_iam_policy_document.trust.json +} +data "aws_iam_policy_document" "rotator_execution" { + statement { + actions = ["logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.rotator.arn}:*"] + } +} +resource "aws_iam_role_policy" "rotator_execution" { + name = "rotator-logs" + role = aws_iam_role.rotator_execution.id + policy = data.aws_iam_policy_document.rotator_execution.json +} +data "aws_iam_policy_document" "rotator" { + statement { + actions = ["secretsmanager:GetSecretValue", "secretsmanager:PutSecretValue", "secretsmanager:DescribeSecret"] + resources = concat([for name in keys(var.services) : local.foundation.secret_arns[name]], [local.foundation.rotation_secret_arn]) + } +} +resource "aws_iam_role_policy" "rotator" { + name = "rotate-service-secrets" + role = aws_iam_role.rotator.id + policy = data.aws_iam_policy_document.rotator.json +} +resource "aws_ecs_task_definition" "rotator" { + skip_destroy = true + family = "${var.name}-rotator" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = "256" + memory = "512" + execution_role_arn = aws_iam_role.rotator_execution.arn + task_role_arn = aws_iam_role.rotator.arn + runtime_platform { + cpu_architecture = "ARM64" + operating_system_family = "LINUX" + } + container_definitions = jsonencode([{ + name = "rotator" + image = "public.ecr.aws/docker/library/node:22-alpine" + essential = true + command = ["sh", "-c", "npm install --prefix /tmp/rotate --omit=dev --ignore-scripts --no-audit --no-fund @aws-sdk/client-secrets-manager@3.1155.0 >/dev/null 2>&1 && printf '%s' \"$ROTATE_SOURCE\" > /tmp/rotate/rotate.cjs && NODE_PATH=/tmp/rotate/node_modules exec node /tmp/rotate/rotate.cjs"] + environment = [ + { name = "AWS_REGION", value = "us-east-1" }, + { name = "ROTATE_ACTION", value = "status" }, + { name = "SERVICE_SECRET_ARNS", value = jsonencode({ for name in keys(var.services) : name => local.foundation.secret_arns[name] }) }, + { name = "ROTATION_SECRET_ARN", value = local.foundation.rotation_secret_arn }, + { name = "ROTATE_SOURCE", value = file("${path.module}/rotate/rotate.cjs") } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.rotator.name + awslogs-region = "us-east-1" + awslogs-stream-prefix = "rotate" + } + } + }]) + depends_on = [aws_iam_role_policy.rotator_execution, aws_iam_role_policy.rotator] +} +output "rotator_task_definition" { value = aws_ecs_task_definition.rotator.arn } diff --git a/deploy/terraform/aws/runtime/terraform-ci.tf b/deploy/terraform/aws/runtime/terraform-ci.tf new file mode 100644 index 000000000..c5fe89585 --- /dev/null +++ b/deploy/terraform/aws/runtime/terraform-ci.tf @@ -0,0 +1,122 @@ +variable "terraform_ci" { + description = "Create the OIDC roles terraform-aws.yml plans and applies the AWS stacks with" + type = bool + default = false +} + +locals { + terraform_ci = var.terraform_ci && var.github_repository != null +} + +data "aws_iam_policy_document" "terraform_trust" { + for_each = local.terraform_ci ? { + plan = ["aws-infrastructure-plan"] + apply = ["aws-infrastructure", "aws-infrastructure-approval"] + } : {} + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.github[0].arn] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = [for environment in each.value : "repo:${var.github_repository}:environment:${environment}"] + } + } +} + +resource "aws_iam_role" "terraform" { + for_each = data.aws_iam_policy_document.terraform_trust + name = "${var.name}-terraform-${each.key}" + assume_role_policy = each.value.json + max_session_duration = 3600 +} + +# Plans run on pull requests, so the plan role reads configuration and never data. +resource "aws_iam_role_policy_attachment" "terraform_plan_read" { + count = local.terraform_ci ? 1 : 0 + role = aws_iam_role.terraform["plan"].name + policy_arn = "arn:aws:iam::aws:policy/ReadOnlyAccess" +} +data "aws_iam_policy_document" "terraform_plan" { + count = local.terraform_ci ? 1 : 0 + statement { + actions = ["s3:PutObject", "s3:DeleteObject"] + resources = ["arn:aws:s3:::${var.state_bucket}/warmbly/production/*.tflock"] + } + statement { + effect = "Deny" + actions = [ + "secretsmanager:GetSecretValue", "kms:Decrypt", + "logs:GetLogEvents", "logs:FilterLogEvents", "logs:StartQuery", "logs:GetQueryResults", + "dynamodb:GetItem", "dynamodb:BatchGetItem", "dynamodb:Query", "dynamodb:Scan", + "sqs:ReceiveMessage", "kinesis:GetRecords", "athena:GetQueryResults", "ecr:GetDownloadUrlForLayer", + ] + resources = ["*"] + } + statement { + effect = "Deny" + actions = ["ssm:GetParameter*"] + resources = ["arn:aws:ssm:us-east-1:${data.aws_caller_identity.current.account_id}:parameter/${var.name}/*"] + } + statement { + effect = "Deny" + actions = ["s3:GetObject", "s3:GetObjectVersion"] + not_resources = ["arn:aws:s3:::${var.state_bucket}/warmbly/production/*"] + } +} +resource "aws_iam_role_policy" "terraform_plan" { + count = local.terraform_ci ? 1 : 0 + name = "state-lock-and-no-data" + role = aws_iam_role.terraform["plan"].id + policy = data.aws_iam_policy_document.terraform_plan[0].json +} + +# The apply role administers the stacks; it is reachable only from main through the protected environments. +resource "aws_iam_role_policy_attachment" "terraform_apply_power" { + count = local.terraform_ci ? 1 : 0 + role = aws_iam_role.terraform["apply"].name + policy_arn = "arn:aws:iam::aws:policy/PowerUserAccess" +} +data "aws_iam_policy_document" "terraform_apply" { + count = local.terraform_ci ? 1 : 0 + statement { + actions = ["iam:Get*", "iam:List*", "iam:CreateServiceLinkedRole"] + resources = ["*"] + } + statement { + actions = ["iam:*"] + resources = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.name}-*", + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:instance-profile/${var.name}-*", + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/token.actions.githubusercontent.com", + ] + } + statement { + effect = "Deny" + actions = ["iam:AttachRolePolicy", "iam:PutRolePermissionsBoundary"] + resources = ["*"] + condition { + test = "ArnLike" + variable = "iam:PolicyARN" + values = ["arn:aws:iam::aws:policy/AdministratorAccess", "arn:aws:iam::aws:policy/IAMFullAccess"] + } + } +} +resource "aws_iam_role_policy" "terraform_apply" { + count = local.terraform_ci ? 1 : 0 + name = "manage-warmbly-roles" + role = aws_iam_role.terraform["apply"].id + policy = data.aws_iam_policy_document.terraform_apply[0].json +} + +output "terraform_role_arns" { + value = { for name, role in aws_iam_role.terraform : name => role.arn } +} diff --git a/deploy/terraform/aws/workers/README.md b/deploy/terraform/aws/workers/README.md index ecb0c5988..5ddf2270e 100644 --- a/deploy/terraform/aws/workers/README.md +++ b/deploy/terraform/aws/workers/README.md @@ -34,6 +34,24 @@ prepared service allows 120 seconds for Docker shutdown. Preserve the previous host for rollback. Static AWS IPs remain stable, but differ from the previous provider's addresses, so verify mailbox authentication during the handover. +Once a host's worker is running and heartbeating, hand its updates to the +fleet so nobody needs to sign in to it again. First make sure the fleet target +matches what the host runs (`warmblyctl fleet version`), then on the host: + +```sh +curl -fsSL https:///join.sh | sudo sh -s -- --adopt --env-refresh +``` + +`--adopt` keeps the existing `node.env` and worker ID and installs the standard +unit and the `warmbly-node-update` timer. The host then pulls the version its +heartbeat names, in the canary waves a release starts, and puts back the +previous image when an update does not stay up. `--env-refresh` lets it replace +`node.env` when the control plane's rendering changes, which is how a key +rotation reaches it; it refuses, naming the keys, when a refresh would drop a +value only this host holds, so move those into `node.local.env` with +`--keep-local`. Run `warmblyctl fleet env-status` afterwards: every adopted host +should report `current`. + Private Valkey access is allowed from the worker security group. Coordinate its activation with every control-plane client, and check configured mail ports and AWS outbound restrictions before moving work. Do not switch one worker to an diff --git a/docs/content/docs/development/admin-panel.mdx b/docs/content/docs/development/admin-panel.mdx index 4b92ecd84..8f39170bc 100644 --- a/docs/content/docs/development/admin-panel.mdx +++ b/docs/content/docs/development/admin-panel.mdx @@ -134,7 +134,7 @@ Reading the tabs needs `view_organizations`; every action needs `manage_organiza | Page | What it answers | |---|---| | **Setup and health** | Three tabs. **Findings** is the [instance health](/development/instance-health/) check list, and the sidebar badge counts its errors and warnings, not operational signals. **Services** holds scoped service reachability probes and the platform mail card. **Operations** holds read-only measurements, coverage, affected counts, evidence age, waits and permission-gated investigation links | -| **Configuration** | Four tabs. **Settings** is the database-backed settings document; **Notifications** is the [operator notification](/development/operator-notifications/) channels; **Environment** is the read-only [configuration reference](/development/configuration/) view of what the backend resolved; **Effective limits** lists the compiled pacing constants | +| **Configuration** | Five tabs. **Settings** is the database-backed settings document; **Notifications** is the [operator notification](/development/operator-notifications/) channels; **Environment** is the read-only [configuration reference](/development/configuration/) view of what the backend resolved; **Effective limits** lists the compiled pacing constants; **Keys** shows whether a [key rotation](/development/configuration/#rotate-instance-keys) can be finished: sealed values still opening only under a read key or under no key, open overlap windows for previous tokens and unsubscribe secrets, and live nodes that have not applied the current env. It is read only; rotation runs where the keys are stored | | **Transfers** | Workspace export and import for any organization, from the operator side, with the same archives as `warmblyctl org export` and `org import`. A whole-instance backup stays `warmblyctl backup` and `restore` | ## Permission per page diff --git a/docs/content/docs/development/architecture.mdx b/docs/content/docs/development/architecture.mdx index 7110fd706..418882409 100644 --- a/docs/content/docs/development/architecture.mdx +++ b/docs/content/docs/development/architecture.mdx @@ -87,9 +87,9 @@ The fleet is pull-based. A node joins by running one command with the instance j 3. The backend returns a node id, the config rendered from its own environment, and the version to run. The script writes `/etc/warmbly/node.env`, installs a systemd service and an update timer, and starts the node container. 4. The node subscribes to the event bus topic named for its id and heartbeats, pacing its own beat at a third of `models.NodeLivenessWindow` from the value the server returns. -The heartbeat reply carries exactly one instruction: `desired_version`. The node writes it to a file and a systemd timer pulls and restarts, so the process being replaced is never the process doing the replacing. An empty `desired_version` means "no opinion", never "downgrade to nothing". +The heartbeat reply carries one instruction: what to run. That is `desired_version`, optionally pinned to an image by `desired_digest`, and `env_revision`, the revision of the env the control plane would render for the node now. The node writes them to files and a systemd timer pulls and restarts, so the process being replaced is never the process doing the replacing. An empty `desired_version` means "no opinion", never "downgrade to nothing". -The machine needs no inbound port, no SSH key and no cloud account. A worker does need outbound access to mail servers on `465`, `587` and `993` (and `143` where a mailbox uses STARTTLS IMAP). Many cloud providers block outbound `25` and `465` on new servers until asked; a worker that learns its `465` is blocked reports it on every heartbeat, and it shows on the worker's page in the admin panel (the **Has node error** filter finds it). There is no restart, logs, reboot or uninstall action in the dashboard, because nothing reaches into a machine. The operator surface is `warmblyctl fleet` (join-token, list, show, remove, pin, version, channel) and the admin panel's Fleet section. +The machine needs no inbound port, no SSH key and no cloud account. A worker does need outbound access to mail servers on `465`, `587` and `993` (and `143` where a mailbox uses STARTTLS IMAP). Many cloud providers block outbound `25` and `465` on new servers until asked; a worker that learns its `465` is blocked reports it on every heartbeat, and it shows on the worker's page in the admin panel (the **Has node error** filter finds it). There is no restart, logs, reboot or uninstall action in the dashboard, because nothing reaches into a machine. The operator surface is `warmblyctl fleet` (join-token, list, show, remove, pin, version, channel, rollout, env-status) and the admin panel's Fleet section. ### Why per-VPS instead of a Kubernetes DaemonSet @@ -175,6 +175,10 @@ An empty `desired_version` means "no opinion" and must never be read as "downgra The backend is deliberately excluded from this. It is what tells everyone else their version, and a self-update that goes wrong leaves nothing to recover with. +A staged rollout (`warmblyctl fleet rollout`) changes who gets the new version, not how it arrives. It is held in `fleet.release` next to the tag and lists the nodes admitted so far: an admitted node is told the target, everyone else the tag, which stays at the old version until the rollout finishes. A backend that predates rollouts reads only the tag and holds every node where it is. Every backend replica advances the rollout on a 30-second tick, and each change is a compare-and-swap on the settings row, so replicas never admit the same wave twice. A canary goes first, then cumulative waves, each only after every admitted node has reported the target and stayed up on it for the bake. A node that goes silent, misses the deadline, keeps restarting or rolls back halts the rollout, and a halted rollout sends the admitted nodes back to where they came from. See `internal/app/fleetnode/rollout.go`. + +The updater checks its own work. After restarting onto a new version or env it watches the container for two minutes; one that stops or restarts in that time is put back on the previous image, or the previous `node.env`. A failed version is reported as `update_failed` so it is not tried again and a rollout halts on it. With `WARMBLY_ENV_REFRESH=true` it also replaces `node.env` when `env_revision` changes, fetching it from `GET /api/v1/internal/fleet/node-env` with the node's broker token. That route renders exactly what join hands out, only for an enrolled node asking as its own role, so a configuration change or a key rotation reaches the fleet without anyone signing in to a machine. `node.local.env` is never touched. + On an instance that encodes the bus with Avro, the fleet also waits for the backend. A booting backend registers every schema its release publishes, and the fleet only moves to a release once the backend runs it and the Schema Registry has accepted those schemas. A refusal holds the fleet on the release it has and shows in the Fleet section, instead of reaching a worker that would then fail every event it reports. The release check runs once when the backend boots, and only with `RELEASES_ENABLED=true`: it is off by default, so a self-hosted fleet is never rolled onto a vendor image. An operator moves the fleet by hand from the admin panel's Fleet section or with `warmblyctl fleet version` and `warmblyctl fleet channel`, which set the channel or pin a tag directly. Configuration is env-driven (`RELEASES_ENABLED`, `RELEASES_GITHUB_REPO`, `RELEASES_WORKER_IMAGE_REPO`, `RELEASES_GITHUB_TOKEN`) so self-hosters can point at their own fork and registry. diff --git a/docs/content/docs/development/aws-fargate.mdx b/docs/content/docs/development/aws-fargate.mdx index 800af472e..7bfe1b1cb 100644 --- a/docs/content/docs/development/aws-fargate.mdx +++ b/docs/content/docs/development/aws-fargate.mdx @@ -1,6 +1,6 @@ --- title: Prepare AWS Fargate -description: Provision an AWS foundation with Terraform and plan a coordinated control plane cutover. +description: Provision AWS with Terraform, plan the control plane cutover, then deploy releases, rotate keys and apply infrastructure changes from CI. --- The configurations in `deploy/terraform/` manage infrastructure through separate @@ -127,9 +127,10 @@ switches. Put passwords, tokens, encryption material and credential-bearing connection strings in Secrets Manager. Reference both stores from ECS task definitions. Running tasks must be replaced to pick up updated injected values. -The foundation also creates a separate rotation secret for pending key -replacements. Keep that secret restricted to operators. Generating pending keys -does not activate them or retire the keys used by the running deployment. +The foundation also creates a separate rotation secret. It holds the state of a +[key rotation](#rotate-keys) in progress, including the pending and previous +values, so the rotator task can resume or undo it. Keep that secret restricted +to operators and the rotator; the deployment role reads no secret value. Preserve current configuration before preparing replacements. Keep a dependency map of each value's readers, encrypted data and rollback requirements. The @@ -191,3 +192,168 @@ before changing it. A successful sign-in check alone does not verify these paths The infrastructure foundation alone does not provide a zero-outage migration. Credential rotation, scheduler ownership, shared state and connection draining must all be coordinated before the traffic switch. + +## Deploy releases + +Once the cutover is done, releases reach production on their own. A stable tag +(`vX.Y.Z`, never a prerelease) builds and publishes the images, and the release +workflow then calls `.github/workflows/deploy-aws.yml` for that tag when the +repository variable `AWS_DEPLOY_ENABLED` is `true`. Set it to anything else to +stop automatic deploys; nothing else changes. Deploys, key rotations and +Terraform applies share one queue, so only one change reaches production at a +time and none is cancelled by the next. + +Terraform owns the shape of every task definition (environment, secret +references, sizes, health checks) and the workflow owns only the image. A deploy: + +1. Checks that the tag's commit is on `main`, that every image digest carries + build provenance from the release workflow, and that the tag is not older + than the one running. +2. Registers a revision of each service from its latest Terraform revision with + the release's digests, choosing the `-kafka` image where the task definition + says so, and refuses one that would stop referencing a secret the running + service reads. +3. Runs the dependency validation task, then the migration task on the backend's + image. Either failing stops the deploy before any service moves. +4. Rolls backend, then consumer, then forms, tracking and realtime together. + Each service is watched for a bake (five minutes by default) after it is + stable. +5. Starts a [canary rollout](/development/warmblyctl/#fleet) of the worker and + consumer hosts from the `ops` task. The first host moves alone and the rest + follow after it has stayed healthy for 30 minutes. + +During the bake, a service fails when one of its alarms fires or ECS rolls it +back. Each public service has two alarms, which ECS also uses to roll back on +its own: more than 5% of requests answered with a 5xx in two of three minutes +(counted only once a minute sees at least 20 requests), and any unhealthy target +for three minutes. A failure at any step puts every service already moved back +on its previous revision and task count, newest first, and the run fails with a +summary. The schema stays where the migration left it, which the previous +release reads. + +### Redeploy, roll back or change a secret + +Dispatch **Deploy AWS control plane** from the Actions tab: + +| Mode | Use it to | +|---|---| +| `redeploy` | Roll the running release again, for example to pick up task definitions Terraform changed | +| `rollback` | Go back to an earlier release. The migration is skipped and the fleet follows in a canary rollout | +| `deploy` | Deploy a release by hand, for example while `AWS_DEPLOY_ENABLED` is off | +| `refresh-secrets` | Roll out a value changed in Secrets Manager, such as a database password or a third-party API key. Leave the tag empty to keep the running images | + +`refresh-secrets` repins every secret reference to the version currently marked +`AWSCURRENT` and rolls with the same bake and rollback, so a wrong value is +caught and put back like a bad release. Change the value in Secrets Manager +first; the workflow only ever sees version ids. Nodes on worker hosts pick up a +change to a value they hold once their env refreshes +([keeping a node's env current](/development/configuration/#keeping-a-nodes-env-current)). + +`services` limits a run to some of the five services, and `bake_seconds` changes +the bake. Releases published before build provenance existed fail the provenance +check; dispatch those once with `require_attestation` off. The automatic +redeploy after a Terraform apply cannot turn it off, so until a release with +provenance is running it stops at that check and changes nothing. + +## Rotate keys + +Dispatch **Rotate instance keys** from the Actions tab and choose a key and an +action. A rotator task inside AWS generates the replacement, writes it as a new +secret version and reports only version ids, so no key passes through GitHub or +a log. The workflow then rolls the services onto those versions, waits for the +worker and consumer hosts to apply their new env, and runs `warmblyctl` in the +`ops` task to prove each step before the previous value is retired. + +| Key | What a rotation does | +|---|---| +| `INTERNAL_API_TOKEN` | The backend accepts the previous token for up to seven days while tracking, forms and the hosts move to the new one, then it is retired | +| `NODE_BROKER_TOKEN` | The same, for the hosts' broker routes | +| `CREDENTIALS_ENCRYPTION_KEY` | The new key is first made readable everywhere, then made primary, stored values are resealed under it, and the old key is retired once `warmblyctl keys status` finds nothing left under it | +| `TWOFA_SECRET` | The new secret seals new enrollments, existing ones are resealed, then the old secret is retired | +| `UNSUBSCRIBE_SIGNING_SECRET` | New links are signed with the new secret. Links already in inboxes stay valid for a year, so the old secret is kept for reads until then | + +`rotate` runs every step it can prove safe in one run. It stops, and says what +it is waiting for, when hosts have not applied the new env within +`node_wait_minutes` (30 by default) or values still open only under the old key. +Run `rotate` again to continue, or `retire` once the reason has cleared. +`retire` refuses while it still has a reason to. `abort` puts the previous value +back as primary; a replacement key that may already have sealed or signed +something stays readable. `status` reports where a rotation stands. Only one key +rotates at a time. + +The [Keys tab](/development/admin-panel/#instance) under Configuration in the +admin panel shows the same checks without starting anything. + +### Before the first rotation + +Each service reads the previous value from an overlap key that has to exist and +be referenced before a rotation can use it: + +1. Run the workflow with `prepare` for the key. It adds the empty overlap keys + (such as `INTERNAL_API_PREVIOUS_TOKENS` and `INTERNAL_API_PREVIOUS_UNTIL` on + the backend) to every service that reads the key, and lists those services. +2. Add those names to each listed service's `secret_keys` in the runtime var + file and apply the runtime stack. +3. Confirm the backend runs a release deployed by the workflow. A rotation rolls + the running release, and a backend started any other way carries no release + tag to roll. + +A rotation leaves the services pinned to newer secret versions than the runtime +var file names. Deploys always use the newest pin, so nothing reverts, but +update the version ids in the var file afterwards to keep Terraform's revisions +current. Authentication keys (`AUTH_SECRET` and realtime's `JWT_SECRET`) are not +offered: they have no overlap, and rotating them signs everyone out. + +## Run Terraform from CI + +Changes to `deploy/terraform/aws/` are planned and applied by +`.github/workflows/terraform-aws.yml`. Var files never enter the repository; +they live in the private state bucket, every value in them is masked before +Terraform runs, and only resource addresses and actions are printed. No full +plan reaches a log, a comment or an artifact. + +- A pull request from this repository gets one comment with the runtime and + workers plans. The foundation plans on merge, because it reads the cache + password, which the plan role may not. +- On merge, each stack is planned in order: foundation, runtime, workers. A + stack that only creates and updates applies on its own. One that replaces or + destroys anything stops and waits for approval in the + `aws-infrastructure-approval` environment, and the stacks after it wait too. + On approval it plans again and applies only if the plan is still the one that + was approved. A new task definition revision counts as an update, because the + previous revision is kept. +- When an apply changes a service's task definition, the running release is + redeployed so the change reaches the services. + +The bootstrap stack stays manual. + +## One-time GitHub setup + +1. On the `aws-production` environment, allow the `v*.*.*` tag pattern next to + `main` in deployment branches and tags, and add a tag ruleset so only release + managers can create `v*` tags. +2. Set `terraform_ci = true` (with `github_repository`) in the runtime var file + and apply it once by hand. Its `terraform_role_arns` output names the plan + and apply roles. +3. Upload `foundation.tfvars.json`, `runtime.tfvars.json` and + `workers.tfvars.json` to `s3:///warmbly/production/vars/`. +4. Create three environments: `aws-infrastructure-plan` with no branch limit, + because pull requests plan in it and its role only reads, `aws-infrastructure` + limited to `main`, and `aws-infrastructure-approval` limited to `main` with + required reviewers. +5. Set these variables. `AWS_DEPLOY_ENABLED` and `AWS_TERRAFORM_ENABLED` have + to be repository variables, because job conditions read them before any + environment applies: + +| Variable | Value | +|---|---| +| `AWS_DEPLOY_ENABLED` | `true` to deploy stable releases automatically | +| `AWS_DEPLOY_ROLE_ARN`, `AWS_ECS_CLUSTER`, `AWS_ECS_SERVICE_PREFIX` | Runtime outputs, on the `aws-production` environment as the runtime README describes | +| `AWS_TERRAFORM_ENABLED` | `true` to plan and apply from CI | +| `AWS_TERRAFORM_PLAN_ROLE_ARN` | The `plan` entry of `terraform_role_arns` | +| `AWS_TERRAFORM_APPLY_ROLE_ARN` | The `apply` entry of `terraform_role_arns` | +| `AWS_TERRAFORM_STATE_BUCKET` | The state bucket name | + +Hand each worker host's updates to the fleet once its worker runs, with the +`--adopt --env-refresh` step in the workers README. Until then it neither follows +releases nor receives rotated values. diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 910f32072..c56446b5f 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -126,6 +126,8 @@ Retain old keys until stored ciphertext, retained events, pending jobs and the s To rotate sessions while preserving existing MFA enrollments, unsubscribe links and published DNS proofs, first pin `TWOFA_SECRET`, `UNSUBSCRIBE_SIGNING_SECRET` and `DOMAIN_PROOF_SECRET` to their existing resolved values. Then change `AUTH_SECRET` and realtime's matching `JWT_SECRET` together. Users sign in again. Rotate the independent purpose keys separately; issued unsubscribe links remain valid for one year, so their read cutoff must cover the last previously issued link's expiry. +Nodes hold `CREDENTIALS_ENCRYPTION_KEY`, `CREDENTIALS_ENCRYPTION_READ_KEYS`, `INTERNAL_API_TOKEN` and `NODE_BROKER_TOKEN` in `node.env`. A node that [keeps its env current](#keeping-a-nodes-env-current) receives each step of a rotation on its own; wait for `warmblyctl fleet env-status` to report every node current before activating a new credential key, and again before retiring an old key or token. A node still on the old values keeps sealing and authenticating with them. The [Keys tab](/development/admin-panel/#instance) under Configuration in the admin panel shows the same three answers in one place: values still under a read key, open overlap windows and nodes that have not applied the current env. On AWS, the rotation workflow runs the whole procedure; see [Rotate keys](/development/aws-fargate/#rotate-keys). + ### Seal mailbox credentials under workspace keys `MAILBOX_CREDENTIALS_SEALING` chooses the key that seals newly stored mailbox credentials: SMTP and IMAP hosts, usernames and passwords, and OAuth access and refresh tokens. @@ -511,6 +513,8 @@ Keep the copies private. GeoLite2 may not be redistributed, which is also why no | `WARMBLY_BACKEND_URL` | Backend base a joined node calls for its heartbeat, and the brokered key and blob providers, when `ENCRYPTED_KEYS_BACKEND_URL` is unset. Written by the join script | unset | yes | | `WARMBLY_NODE_NAME` | Display name the node reports to the fleet list and admin panel. Written by the join script from `--name`, else the hostname | unset | yes | | `WARMBLY_TARGET_VERSION_PATH` | File the node writes the control plane's `desired_version` to, for the `warmbly-node-update` timer to apply. Unset, the node never writes one and does not update itself. Written by the join script | unset | yes | +| `WARMBLY_ENV_REFRESH` | Read by the host updater, not the node. `true` lets it replace `node.env` with what the control plane renders now whenever the heartbeat names a different env revision, then restart the node. `node.local.env` is never touched. Written by the join script on a new join and by `--adopt --env-refresh`; a node without it keeps its env until it rejoins | unset | no (read by the updater) | +| `WARMBLY_UPDATE_HEALTH_SECONDS` | Read by the host updater. How long a node restarted onto a new version or env has to stay up on one start before the change is kept. A container that stops or restarts in that time is put back on what it ran before | `120` | no (read by the updater) | | `APP_INTERNAL_URL` | Read by the backend: the address a joining node is told to call back on when `ENCRYPTED_KEYS_BACKEND_URL` is unset on the backend. Instance health checks whichever of the two is set | unset | no (read per join) | @@ -538,6 +542,18 @@ printf 'PRIMARY_DB=%s\n' "postgres://..." >> /etc/warmbly/node.local.env systemctl restart warmbly-consumer ``` +### Keeping a node's env current + +A node joined with `WARMBLY_ENV_REFRESH=true` picks up configuration changes without anyone signing in to it. Every heartbeat reply carries the revision of the env the control plane would render for that node now. When it differs from what the node applied, the updater fetches the new env from the backend with the node's own broker token, writes `node.env` in one step, restarts the node and reports the new revision. If the restarted node does not stay up, the previous `node.env` is put back and that revision is not tried again. + +`warmblyctl fleet env-status` lists every live node as `current`, `stale` or `manual` (a node that does not refresh) and exits non-zero until all of them are current. A key rotation that reaches nodes waits on it, so a `manual` node holds a rotation open until it is rejoined, or adopted with `--env-refresh`: + +```bash +curl -fsSL https:///join.sh | sudo sh -s -- --adopt --env-refresh +``` + +`--adopt` installs the standard unit and updater for the node the existing `node.env` describes, without enrolling again. With `--env-refresh` it first checks that a refresh would keep every key in `node.env`, and refuses, listing the names, when one exists only on this machine. Name those with `--keep-local` to move them into `node.local.env`. It also refuses when the control plane asks for a different version than the node runs, because the updater would move it within two minutes; set the fleet version first, or pass `--accept-target`. + ## Mailbox connections Needed on the backend **and** every worker: the backend starts the OAuth flow, and each worker refreshes the token when it expires. diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index 080437736..91afabfe2 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -536,6 +536,7 @@ It also lists any `INTERNAL_API_PREVIOUS_UNTIL`, `NODE_BROKER_PREVIOUS_UNTIL` an | `--apply` | Write the resealed values. Without it, `reseal` reports what it would do | | `--batch` | Rows read per query, 1 to 10000. Defaults to 500 | | `--json` | Print the report as JSON (also on `keys status`) | +| `--require-primary` | On `keys status`, also exit non-zero while any value opens only under a read key, which is the check that a read key can be retired | Both exit non-zero when any value is unreadable. Restore the key that sealed it as a read key before retiring anything; an unreadable mailbox credential is a mailbox that can no longer sign in. @@ -747,6 +748,35 @@ warmblyctl fleet pin # clear that pin Setting a tag also pins the channel, so a release landing later does not undo a deliberate rollback. +`fleet version` moves every node at once. A rollout moves them in stages instead, and stops on the first sign of trouble: + +```bash +warmblyctl fleet rollout start v1.4.3 --canary 1 --waves 50,100 --bake 30m +warmblyctl fleet rollout status +warmblyctl fleet rollout advance # admit the next stage now, skipping the bake +warmblyctl fleet rollout halt --reason "watching sync errors" +warmblyctl fleet rollout abort # forget the rollout, fleet stays on the old tag +``` + +A rollout admits the canary nodes alone, workers before consumers, and admits each wave only after every admitted node has reported the target and stayed up on it for the bake. Waves are cumulative percentages of the live, unpinned nodes. The backend checks the rollout every 30 seconds and halts it when an admitted node stops heartbeating, rolls itself back on the host, misses the target within `--deadline` (15 minutes by default) or keeps restarting. A halted rollout admits nobody else and sends the admitted nodes back to the version they came from. Until it finishes, the fleet target stays at that old version, so a backend that predates rollouts holds every node where it is. Pinned nodes are left out. + +| Flag | Does | +|---|---| +| `--canary` | Nodes admitted first, alone. Defaults to `1` | +| `--waves` | Comma-separated cumulative percentages admitted after the canary. Defaults to `100` | +| `--bake` | Healthy time on the target before the next stage. Defaults to `30m` | +| `--deadline` | Time an admitted node has to report the target. Defaults to `15m` | +| `--digests` | `image=sha256:...` pairs (`worker`, `worker-kafka`, `consumer`, `consumer-kafka`). A node then pulls that exact image rather than the tag | +| `--json` | Machine-readable output, on every rollout command | + +The host updater checks each update itself too. A node restarted onto a new version that does not stay up for two minutes is put back on the image it ran, and reports the failure so the same version is not tried again. + +```bash +warmblyctl fleet env-status +``` + +Lists every live node as `current` (it applied the env the control plane renders now), `stale` or `manual` (it does not [refresh its env](/development/configuration/#keeping-a-nodes-env-current)), and exits non-zero until every node is current. Run it where the backend runs, so it renders from the same configuration. A key rotation that reaches the fleet waits on it. + ```bash warmblyctl fleet remove ``` diff --git a/internal/api/handler/admin_keys.go b/internal/api/handler/admin_keys.go new file mode 100644 index 000000000..39b65b5c8 --- /dev/null +++ b/internal/api/handler/admin_keys.go @@ -0,0 +1,87 @@ +package handler + +import ( + "context" + "log" + "net/http" + "sync" + "time" + + "github.com/gin-gonic/gin" + + "github.com/warmbly/warmbly/internal/app/fleetnode" + "github.com/warmbly/warmbly/internal/app/keystatus" + "github.com/warmbly/warmbly/internal/errx" +) + +// keyStatusTTL bounds how often the panel can make the backend scan every sealed column. +const keyStatusTTL = time.Minute + +type keyStatusCache struct { + mu sync.Mutex + at time.Time + report *keystatus.Report +} + +var adminKeyStatus keyStatusCache + +type adminKeyStatusResponse struct { + Keys *keystatus.Report `json:"keys"` + Fleet *fleetnode.EnvStatus `json:"fleet,omitempty"` + Ready adminKeyStatusReadies `json:"ready"` +} + +// adminKeyStatusReadies says which retirements the evidence already allows. +type adminKeyStatusReadies struct { + RetireReadKeys bool `json:"retire_read_keys"` + RetireTokens bool `json:"retire_tokens"` +} + +// AdminKeyStatus reports where every sealed value stands against the configured +// keys and whether every node runs the current env. Read only: it never reseals. +func (h *Handler) AdminKeyStatus(c *gin.Context) { + if h.KeyStatusDB == nil { + errx.Handle(c, errx.New(errx.NotImplemented, "Key status is not available on this instance.")) + return + } + report, err := h.keyStatus(c.Request.Context(), c.Query("refresh") == "true") + if err != nil { + errx.Handle(c, err) + return + } + resp := adminKeyStatusResponse{Keys: report} + resp.Ready.RetireReadKeys = report.InstanceReadKeys+report.MFAReadKeys > 0 && report.OnReadKey() == 0 && report.Unreadable() == 0 + if h.FleetNodes != nil { + nodes, err := h.FleetNodes.List(c.Request.Context(), "") + if err != nil { + errx.Handle(c, err) + return + } + fleet := fleetnode.EnvStatusOf(nodes) + resp.Fleet = &fleet + resp.Ready.RetireTokens = fleet.AllCurrent + } + c.JSON(http.StatusOK, resp) +} + +func (h *Handler) keyStatus(ctx context.Context, refresh bool) (*keystatus.Report, error) { + adminKeyStatus.mu.Lock() + defer adminKeyStatus.mu.Unlock() + if !refresh && adminKeyStatus.report != nil && time.Since(adminKeyStatus.at) < keyStatusTTL { + return adminKeyStatus.report, nil + } + keys, err := keystatus.LoadKeyring(ctx) + if err != nil { + log.Printf("admin: key status keyring: %v", err) + return nil, errx.NewPublic(errx.Unprocessable, "The instance keys could not be loaded. Run `warmblyctl keys status` inside the backend container for the reason.") + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + report, err := keystatus.Scan(ctx, h.KeyStatusDB, keys, keystatus.Options{}) + if err != nil { + log.Printf("admin: key status scan: %v", err) + return nil, errx.InternalError() + } + adminKeyStatus.report, adminKeyStatus.at = &report, time.Now() + return &report, nil +} diff --git a/internal/api/handler/fleet_nodes.go b/internal/api/handler/fleet_nodes.go index 2c1a02809..96e8384aa 100644 --- a/internal/api/handler/fleet_nodes.go +++ b/internal/api/handler/fleet_nodes.go @@ -4,10 +4,8 @@ import ( "context" "encoding/base64" "errors" - "fmt" "log" "net/http" - "os" "sort" "strings" "sync" @@ -17,7 +15,6 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/api/middleware" "github.com/warmbly/warmbly/internal/app/fleetnode" - "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/observability/errs" @@ -30,6 +27,7 @@ import ( // // POST /api/v1/fleet/join join token -> node id + env file // POST /api/v1/internal/fleet/heartbeat internal -> desired version +// GET /api/v1/internal/fleet/node-env node broker -> current env file // // Nothing here reaches into a machine, which is why onboarding needs no // keypair, no inbound port and no cloud account. @@ -209,9 +207,51 @@ func (h *Handler) FleetHeartbeat(c *gin.Context) { }(beat.NodeID) } + if beat.EnvRefresh { + if _, rev, ok := h.nodeEnvFor(c.Request.Context(), beat.NodeID, beat.Role); ok { + reply.EnvRevision = rev + } + } c.JSON(http.StatusOK, reply) } +// nodeEnvFor renders the env an enrolled node of role should have, with its revision. +func (h *Handler) nodeEnvFor(ctx context.Context, id uuid.UUID, role models.NodeRole) (string, string, bool) { + node, err := h.FleetNodes.Get(ctx, id) + if err != nil || node == nil || node.Role != role { + return "", "", false + } + env := renderNodeEnv(node.ID, node.Role, node.Region) + return env, fleetnode.EnvRevision(env), true +} + +// InternalFleetNodeEnv hands an enrolled node the env file join would write +// today, so a rotated value reaches it by pulling instead of rejoining. +func (h *Handler) InternalFleetNodeEnv(c *gin.Context) { + if h.FleetNodes == nil { + c.Status(http.StatusNotFound) + return + } + id, err := uuid.Parse(c.Query("node_id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "node_id must be a uuid"}) + return + } + role := models.NodeRole(c.Query("role")) + if !role.Valid() { + c.JSON(http.StatusBadRequest, gin.H{"error": "role must be worker or consumer"}) + return + } + env, rev, ok := h.nodeEnvFor(c.Request.Context(), id, role) + if !ok { + c.JSON(http.StatusNotFound, gin.H{"error": "no enrolled node with that id and role"}) + return + } + c.Header("Cache-Control", "no-store") + c.Header("X-Warmbly-Env-Revision", rev) + c.Data(http.StatusOK, "text/plain; charset=utf-8", []byte(env)) +} + // legacyTokenNodes remembers which nodes were already warned about, so the log says it once per process. var legacyTokenNodes sync.Map @@ -250,165 +290,9 @@ func nodeHeartbeatSeconds(livenessSeconds int) int { return n } -// nodeEnvKeys are the settings a node needs to do its job, in the order they -// are written. Every one is read from the backend's own environment, so a node -// is configured with exactly the infrastructure the control plane uses and -// there is nothing to keep in sync by hand. -// -// Every name here must be one the node's own code reads. Two of them were not -// (S3_BUCKET and KMS_KEY_ID, against a storage layer reading BLOB_BUCKET and a -// KMS factory reading KMS_AWS_KEY_ID), which sent an AWS-backed node to the -// default bucket and the default key alias with nothing logged. -// -// KMS_PROVIDER and BLOB_PROVIDER are absent because they are not copied but -// translated; see nodeProviders. The AWS-shaped settings above stay, so an -// operator who deliberately overrides a node back to a direct provider in -// node.local.env only has to add the credential. -// -// Deliberately excluded: PRIMARY_DB for a worker, and anything else that would -// give one direct database access. A worker reaches relational data through -// the internal API and nothing else. renderNodeEnv sends the DSN to a consumer, -// which is control plane and updates relational state itself. -var nodeEnvKeys = []string{ - "APP_ENV", - "EVENTBUS_PROVIDER", - "NATS_URL", - // Base64, not a path: a node has no file to point at, and the env file - // docker reads cannot hold a multi-line value. - "NATS_CREDS_B64", - "NATS_MAX_BYTES", - "KAFKA_BOOTSTRAP_SERVERS", - "KAFKA_SASL_USERNAME", - "KAFKA_SASL_PASSWORD", - "SCHEMA_REGISTRY_URL", - "SCHEMA_REGISTRY_KEY", - "SCHEMA_REGISTRY_SECRET", - "CODEC_PROVIDER", - "REDIS", - "KMS_LOCAL_MASTER_KEY", - "KMS_AWS_KEY_ID", - "CREDENTIALS_ENCRYPTION_KEY", - "BLOB_FS_ROOT", - "BLOB_BUCKET", - "AWS_REGION", - "AWS_ENDPOINT_URL_S3", - "BOX_GOOGLE_CLIENT_ID", - "BOX_GOOGLE_CLIENT_SECRET", - "BOX_OUTLOOK_CLIENT_ID", - "BOX_OUTLOOK_CLIENT_SECRET", - "MAIL_TLS_INSECURE", - "POSTHOG_KEY", - "POSTHOG_HOST", - "POSTHOG_ERROR_TRACKING", - "SENTRY_DSN", -} - -// consumerEnvKeys reach a consumer and never a worker. A consumer runs the -// integration actions, refreshes their OAuth tokens and drains the CRM outbox, -// so it reads the integration client credentials (integration.NewOAuthManager), -// APP_URL (the "Open in Warmbly" link written to the CRM) and the backend's -// public URL (where it registers Pipedrive's webhooks). -var consumerEnvKeys = []string{ - "APP_URL", - "BACKEND_PUBLIC_URL", - "API_PUBLIC_URL", - "HUBSPOT_OAUTH_CLIENT_ID", - "HUBSPOT_OAUTH_CLIENT_SECRET", - "SLACK_OAUTH_CLIENT_ID", - "SLACK_OAUTH_CLIENT_SECRET", - "GOOGLE_SHEETS_OAUTH_CLIENT_ID", - "GOOGLE_SHEETS_OAUTH_CLIENT_SECRET", - "PIPEDRIVE_OAUTH_CLIENT_ID", - "PIPEDRIVE_OAUTH_CLIENT_SECRET", - "SALESFORCE_OAUTH_CLIENT_ID", - "SALESFORCE_OAUTH_CLIENT_SECRET", -} - -// nodeProviders translates the control plane's own crypto and blob providers -// into the ones a node should run. -// -// A node has no cloud credentials and no way to be handed any: node.env is -// regenerated from this instance's environment on every join, and shipping an -// IAM key to every machine in the fleet is exactly the thing worth avoiding. So -// a provider that needs a credential becomes its brokered form, which -// authenticates with the internal API token the node already holds and asks -// this instance to do the one privileged operation. A provider that needs no -// credential (local KMS, filesystem blobs) passes through unchanged. -// -// Brokering costs one HTTPS call to the control plane per DEK open (Redis -// caches the result) and per blob operation. The bytes still go straight -// between the node and the object store. -func nodeProviders() (kmsProvider, blobProvider string) { - kmsProvider = config.KMSProvider() - if kmsProvider == "aws" || kmsProvider == "aws-kms" { - kmsProvider = "brokered" - } - blobProvider = config.BlobProvider() - if blobProvider == "s3" { - blobProvider = "brokered" - } - return kmsProvider, blobProvider -} - // renderNodeEnv builds the env file a node writes to disk on join. func renderNodeEnv(nodeID uuid.UUID, role models.NodeRole, region string) string { - var b strings.Builder - b.WriteString("# Written by `warmbly join`. Regenerate by joining again.\n") - fmt.Fprintf(&b, "WARMBLY_NODE_ID=%s\n", nodeID) - fmt.Fprintf(&b, "WARMBLY_NODE_ROLE=%s\n", role) - if region != "" { - fmt.Fprintf(&b, "WARMBLY_NODE_REGION=%s\n", region) - } - // A worker resolves its own identity from WORKER_ID; keeping the two equal - // means the placement row and the node row are the same machine. - if role == models.NodeRoleWorker { - fmt.Fprintf(&b, "WORKER_ID=%s\n", nodeID) - } - - backend := strings.TrimRight(os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"), "/") - if backend == "" { - backend = strings.TrimRight(os.Getenv("APP_INTERNAL_URL"), "/") - } - // A consumer is control plane: it opens Postgres itself, so it gets the DSN - // a worker is deliberately never given, and reads keys straight from the - // table rather than back through the API it sits behind. An instance whose - // DSN lives in SSM rather than the environment has nothing to send, so the - // node falls back to the worker's HTTP key path and join.sh tells the - // operator to supply PRIMARY_DB in node.local.env. - dsn := os.Getenv("PRIMARY_DB") - keysProvider := "http" - if role == models.NodeRoleConsumer && dsn != "" { - keysProvider = "postgres" - fmt.Fprintf(&b, "PRIMARY_DB=%s\n", dsn) - } - - fmt.Fprintf(&b, "WARMBLY_BACKEND_URL=%s\n", backend) - fmt.Fprintf(&b, "ENCRYPTED_KEYS_PROVIDER=%s\n", keysProvider) - fmt.Fprintf(&b, "ENCRYPTED_KEYS_BACKEND_URL=%s\n", backend) - // A node calls only the node-only internal routes. With a separate - // NODE_BROKER_TOKEN it is sent that alone and never the edge services' token. - if v := os.Getenv("NODE_BROKER_TOKEN"); v != "" { - fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", v) - fmt.Fprintf(&b, "NODE_BROKER_TOKEN=%s\n", v) - } else { - fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN")) - fmt.Fprintf(&b, "INTERNAL_API_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN")) - } - - kmsProvider, blobProvider := nodeProviders() - fmt.Fprintf(&b, "KMS_PROVIDER=%s\n", kmsProvider) - fmt.Fprintf(&b, "BLOB_PROVIDER=%s\n", blobProvider) - - keys := nodeEnvKeys - if role == models.NodeRoleConsumer { - keys = append(append([]string{}, nodeEnvKeys...), consumerEnvKeys...) - } - for _, k := range keys { - if v := os.Getenv(k); v != "" { - fmt.Fprintf(&b, "%s=%s\n", k, v) - } - } - return b.String() + return fleetnode.RenderEnv(nodeID, role, region) } // ---- admin ---- @@ -735,6 +619,8 @@ func (h *Handler) AdminFleetSetRelease(c *gin.Context) { } } state.Tag = strings.TrimSpace(body.Tag) + // An explicit tag moves everyone at once, replacing any staged rollout. + state.Rollout = nil state.Channel = models.FleetChannelPinned state.ResolvedAt = time.Now() state.Source = "admin" diff --git a/internal/api/handler/fleet_nodes_test.go b/internal/api/handler/fleet_nodes_test.go index 6b88f32cd..f792d839f 100644 --- a/internal/api/handler/fleet_nodes_test.go +++ b/internal/api/handler/fleet_nodes_test.go @@ -427,3 +427,54 @@ func TestRenderNodeEnvSendsOnlyTheNodeToken(t *testing.T) { t.Error("a single-token instance must keep sending the shared token") } } + +// A rotation reaches nodes by pulling: the env endpoint and the revision in +// the heartbeat reply must describe the same file, or the updater loops. +func TestFleetNodeEnvMatchesHeartbeatRevision(t *testing.T) { + t.Setenv("WARMBLY_VERSION", "v0.6.44") + t.Setenv("FLEET_IMAGE_VARIANT", "") + t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", `["`+strings.Repeat("a", 64)+`"]`) + id := uuid.New() + nodes := &joinNodes{nodeID: id, role: models.NodeRoleWorker} + h := &Handler{FleetNodes: fleetnode.New(nodes, nil, joinSettings{})} + + get := func(query string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/internal/fleet/node-env?"+query, nil) + h.InternalFleetNodeEnv(c) + return w + } + w := get("node_id=" + id.String() + "&role=worker") + if w.Code != http.StatusOK { + t.Fatalf("enrolled node was refused its env: %d", w.Code) + } + env := envLines(t, w.Body.String()) + if env["WARMBLY_NODE_ID"] != id.String() || env["CREDENTIALS_ENCRYPTION_READ_KEYS"] == "" { + t.Fatal("node env must carry the node identity and the credential read keys") + } + if _, ok := env["PRIMARY_DB"]; ok { + t.Fatal("a worker env must never carry a database DSN") + } + rev := w.Header().Get("X-Warmbly-Env-Revision") + if rev == "" || rev != fleetnode.EnvRevision(w.Body.String()) { + t.Fatal("the revision header must name the body it came with") + } + + for _, q := range []string{"node_id=" + id.String() + "&role=consumer", "node_id=" + uuid.NewString() + "&role=worker", "node_id=x&role=worker"} { + if code := get(q).Code; code == http.StatusOK { + t.Fatalf("%s must be refused", q) + } + } + + hw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(hw) + c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/internal/fleet/heartbeat", + strings.NewReader(`{"node_id":"`+id.String()+`","role":"worker","env_refresh":true}`)) + c.Request.Header.Set("Content-Type", "application/json") + h.FleetHeartbeat(c) + var reply models.NodeHeartbeatReply + if err := json.Unmarshal(hw.Body.Bytes(), &reply); err != nil || reply.EnvRevision != rev { + t.Fatalf("heartbeat revision %q must match the env endpoint's %q", reply.EnvRevision, rev) + } +} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index a64ddff5f..e38bd9dc1 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -42,6 +42,7 @@ import ( "github.com/warmbly/warmbly/internal/app/instanceconfig" "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/app/integration" + "github.com/warmbly/warmbly/internal/app/keystatus" "github.com/warmbly/warmbly/internal/app/leadsync" "github.com/warmbly/warmbly/internal/app/mailboximport" "github.com/warmbly/warmbly/internal/app/mcp" @@ -195,7 +196,9 @@ type Handler struct { // Fleet. Nodes enrol with the join token and pull everything else; the // control plane never reaches into a machine. - FleetNodes *fleetnode.Service + FleetNodes *fleetnode.Service + // KeyStatusDB is the pool the admin key status scans read through. Nil turns the page off. + KeyStatusDB keystatus.DB NodeLogs nodelogs.Store BrokerDiagnostics eventbus.EventBus WorkerRepo repository.WorkerRepository diff --git a/internal/api/handler/nodescript/join.sh b/internal/api/handler/nodescript/join.sh index 69cceb7ea..718abc647 100755 --- a/internal/api/handler/nodescript/join.sh +++ b/internal/api/handler/nodescript/join.sh @@ -30,6 +30,11 @@ STATE_DIR="/var/lib/warmbly" AGENT_DIR="/var/lib/warmbly/node" DRY_RUN="false" PRINT_UNIT="false" +PRINT_UPDATER="false" +ADOPT="false" +ADOPT_ACCEPT_TARGET="false" +ADOPT_ENV_REFRESH="false" +ADOPT_KEEP_LOCAL="" log() { printf '%s\n' "$*"; } warn() { printf '%s\n' "$*" >&2; } @@ -51,9 +56,21 @@ Join a machine to a Warmbly fleet. ghcr.io/warmbly/warmbly. --config-dir Where to write the env file. Default /etc/warmbly. --dry-run Enrol and print what would be written, change nothing. + --print-updater Print the updater that would be installed and exit. --print-unit Print the systemd unit that would be installed and exit. Contacts nothing and writes nothing; used by `make join-check`. + +Adopting a machine that already runs a node (no enrolment, node.env kept): + --adopt Install the standard unit and updater for the node + described by the existing node.env. + --keep-local Copy these node.env keys into node.local.env, for values + only this machine knows. + --env-refresh Let the updater refresh node.env from the control plane, + enabled only when a refresh would keep every key. + --accept-target Adopt even though the control plane asks for a different + version than this node runs. + -h, --help This text. A second run re-enrols the same machine: it keeps the existing node id, so the @@ -77,6 +94,11 @@ parse_args() { --config-dir) CONFIG_DIR="${2:-}"; shift 2 ;; --dry-run) DRY_RUN="true"; shift ;; --print-unit) PRINT_UNIT="true"; shift ;; + --print-updater) PRINT_UPDATER="true"; shift ;; + --adopt) ADOPT="true"; shift ;; + --accept-target) ADOPT_ACCEPT_TARGET="true"; shift ;; + --env-refresh) ADOPT_ENV_REFRESH="true"; shift ;; + --keep-local) ADOPT_KEEP_LOCAL="${2:-}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown option: $1 (try --help)" ;; esac @@ -263,6 +285,8 @@ write_config() { printf 'WARMBLY_VERSION=%s\n' "$DESIRED_VERSION" printf 'WARMBLY_TARGET_VERSION_PATH=%s/target-version\n' "$AGENT_DIR" printf 'WARMBLY_NODE_NAME=%s\n' "$WARMBLY_NAME" + # node.env is exactly the render plus the lines above, so a refresh loses nothing. + printf 'WARMBLY_ENV_REFRESH=true\n' } > "$CONFIG_DIR/node.env" chmod 600 "$CONFIG_DIR/node.env" @@ -503,64 +527,249 @@ WantedBy=multi-user.target UNIT } -install_units() { - [ "$DRY_RUN" = "false" ] || return 0 - - service="warmbly-$WARMBLY_ROLE" - if blobs_are_local; then - root=$(blob_root) - if [ -n "$root" ]; then - # Keep this a standalone statement: make join-check asserts on it by - # first field, because a looser match was satisfied by the name - # appearing inside a warn string. - ensure_blob_root "$root" - fi - fi - render_unit > "/etc/systemd/system/$service.service" - - # The updater is what makes auto-update work without anything reaching into - # this machine. The node writes the version the control plane wants into - # $STATE_DIR/target-version on each heartbeat; this notices the file changed, - # pulls, and restarts. Keeping it outside the service means the process being - # replaced is never the process doing the replacing. - cat > "/usr/local/bin/warmbly-node-update" <<'UPDATER' +# render_updater prints the root updater exactly as install_updater writes it. +# The node only writes requests into $AGENT_DIR; this decides whether to act +# on them, so the process being replaced is never the one doing the replacing. +render_updater() { + cat <<'UPDATER' #!/bin/sh +# Applies what the control plane asked this node to run: a version, and with +# WARMBLY_ENV_REFRESH=true in node.env, a refreshed env. Never reaches out +# except to pull the image and, for a refresh, to fetch this node's own env. set -eu -STATE_DIR="/var/lib/warmbly" -AGENT_DIR="/var/lib/warmbly/node" -CONFIG_DIR="/etc/warmbly" -[ -f "$AGENT_DIR/target-version" ] || exit 0 -[ -f "$STATE_DIR/image" ] || exit 0 +# ROOT and the timings exist for make join-check; the timer sets none of them. +ROOT="${WARMBLY_UPDATER_ROOT:-}" +STATE_DIR="$ROOT/var/lib/warmbly" +AGENT_DIR="$ROOT/var/lib/warmbly/node" +CONFIG_DIR="$ROOT/etc/warmbly" +HEALTH_SECONDS="${WARMBLY_UPDATE_HEALTH_SECONDS:-120}" +SETTLE_SECONDS="${WARMBLY_UPDATE_SETTLE_SECONDS:-15}" +POLL_SECONDS="${WARMBLY_UPDATE_POLL_SECONDS:-10}" +# Kept across an env refresh: written by join or by this updater, never rendered. +PRESERVED_KEYS="WARMBLY_VERSION WARMBLY_TARGET_VERSION_PATH WARMBLY_NODE_NAME WARMBLY_ENV_REFRESH NODE_LOG_TOKEN" -target="$(cat "$AGENT_DIR/target-version")" -image="$(cat "$STATE_DIR/image")" -current="$(sed -n 's/^WARMBLY_VERSION=//p' "$CONFIG_DIR/node.env" | head -n 1)" +log() { printf 'warmbly-node-update: %s\n' "$*"; } -[ -n "$target" ] || exit 0 -[ "$target" != "$current" ] || exit 0 +env_value() { + sed -n "s/^$1=//p" "$CONFIG_DIR/node.env" | head -n 1 +} -# The node writes this file, and root runs whatever image it names, so the -# value is validated rather than trusted: tag characters only, no registry or -# path separators that could redirect the pull somewhere else. -case "$target" in - *[!A-Za-z0-9._-]*) echo "warmbly-node-update: refusing malformed target '$target'"; exit 0 ;; -esac +# read_token reads a file the node wrote. The node is untrusted here, so a +# symlink or anything but a plain token reads as empty. +read_token() { + [ -f "$1" ] || return 0 + [ ! -L "$1" ] || return 0 + v=$(head -c 256 "$1" | head -n 1) + case "$v" in + ""|*[!A-Za-z0-9._:+-]*) return 0 ;; + esac + printf '%s\n' "$v" +} -role="$(sed -n 's/^WARMBLY_NODE_ROLE=//p' "$CONFIG_DIR/node.env" | head -n 1)" -[ -n "$role" ] || exit 0 +# write_agent replaces a file in the node's directory by rename, so a symlink +# the node planted is replaced, never followed. +write_agent() { + tmp=$(mktemp "$AGENT_DIR/.update.XXXXXX") + printf '%s\n' "$2" > "$tmp" + chmod 0644 "$tmp" + chown 1000:1000 "$tmp" 2>/dev/null || true + mv -f "$tmp" "$AGENT_DIR/$1" +} -# Pull first. If the image is not there yet, leave the node on the version it -# is running rather than restarting it into a pull failure. -if ! docker pull "$image:$target" >/dev/null 2>&1; then - echo "warmbly-node-update: $image:$target is not pullable yet; staying on $current" - exit 0 -fi +set_version() { + if grep -q '^WARMBLY_VERSION=' "$CONFIG_DIR/node.env"; then + sed -i "s|^WARMBLY_VERSION=.*|WARMBLY_VERSION=$1|" "$CONFIG_DIR/node.env" + else + printf 'WARMBLY_VERSION=%s\n' "$1" >> "$CONFIG_DIR/node.env" + fi +} -sed -i "s|^WARMBLY_VERSION=.*|WARMBLY_VERSION=$target|" "$CONFIG_DIR/node.env" -printf 'WARMBLY_IMAGE_REF=%s:%s\n' "$image" "$target" > "$STATE_DIR/image-ref" -echo "warmbly-node-update: $current -> $target" -systemctl restart "warmbly-$role" +# digest_for prints the digest the control plane pinned for tag, if any. +digest_for() { + [ -f "$AGENT_DIR/target-digest" ] || return 0 + [ ! -L "$AGENT_DIR/target-digest" ] || return 0 + line=$(head -c 256 "$AGENT_DIR/target-digest" | head -n 1) + printf '%s\n' "$line" | grep -Eq '^[A-Za-z0-9._-]+ sha256:[0-9a-f]{64}$' || return 0 + [ "${line%% *}" = "$1" ] || return 0 + printf '%s\n' "${line#* }" +} + +started() { + docker inspect -f '{{.State.Running}} {{.State.StartedAt}}' "$service" 2>/dev/null || echo "false -" +} + +# healthy passes when the restarted container comes up and stays on one start +# for HEALTH_SECONDS. A crash loop restarts it, which changes StartedAt. +healthy() { + sleep "$SETTLE_SECONDS" + first=$(started) + case "$first" in + "true "*) ;; + *) return 1 ;; + esac + waited=0 + while [ "$waited" -lt "$HEALTH_SECONDS" ]; do + sleep "$POLL_SECONDS" + waited=$((waited + POLL_SECONDS)) + [ "$(started)" = "$first" ] || return 1 + done + return 0 +} + +update_version() { + target=$(read_token "$AGENT_DIR/target-version") + [ -n "$target" ] || return 0 + case "$target" in + *[!A-Za-z0-9._-]*) log "refusing malformed target '$target'"; return 0 ;; + esac + current=$(env_value WARMBLY_VERSION) + [ "$target" != "$current" ] || return 0 + if [ "$(read_token "$AGENT_DIR/update-failed")" = "$target" ]; then + return 0 + fi + image=$(head -n 1 "$STATE_DIR/image") + case "$image" in + ""|*[!A-Za-z0-9./_-]*) log "refusing malformed image '$image'"; return 0 ;; + esac + ref="$image:$target" + digest=$(digest_for "$target") + if [ -n "$digest" ]; then + ref="$image@$digest" + fi + # Pull first, so a missing image leaves the node on what it runs. + if ! docker pull "$ref" >/dev/null 2>&1; then + log "$ref is not pullable yet; staying on $current" + return 0 + fi + previous=$(cat "$STATE_DIR/image-ref") + set_version "$target" + printf 'WARMBLY_IMAGE_REF=%s\n' "$ref" > "$STATE_DIR/image-ref" + log "$current -> $target ($ref)" + systemctl restart "$service" + if healthy; then + rm -f "$AGENT_DIR/update-failed" + return 0 + fi + log "$target did not stay up; returning to $current" + set_version "$current" + printf '%s\n' "$previous" > "$STATE_DIR/image-ref" + systemctl restart "$service" + write_agent update-failed "$target" +} + +# fetch_env writes this node's env as the control plane renders it now into $1. +fetch_env() { + url=$(env_value WARMBLY_BACKEND_URL) + id=$(env_value WARMBLY_NODE_ID) + token=$(env_value NODE_BROKER_TOKEN) + [ -n "$token" ] || token=$(env_value INTERNAL_API_TOKEN) + if [ -z "$url" ] || [ -z "$id" ] || [ -z "$token" ]; then + log "node.env lacks WARMBLY_BACKEND_URL, WARMBLY_NODE_ID or a node token; cannot fetch env" + return 1 + fi + # The token goes through curl's stdin config, never argv, so ps never shows it. + code=$(printf 'header = "Authorization: Bearer %s"\n' "$token" | + curl -sS -K - --max-time 30 -o "$1" -w '%{http_code}' \ + "${url%/}/api/v1/internal/fleet/node-env?node_id=$id&role=$role") || code=000 + if [ "$code" != "200" ]; then + log "fetching env failed (HTTP $code)" + return 1 + fi + if ! grep -qx "WARMBLY_NODE_ID=$id" "$1" || ! grep -qx "WARMBLY_NODE_ROLE=$role" "$1"; then + log "fetched env names another node; ignored" + return 1 + fi + return 0 +} + +refresh_env() { + [ "$(env_value WARMBLY_ENV_REFRESH)" = "true" ] || return 0 + want=$(read_token "$AGENT_DIR/target-env-revision") + [ -n "$want" ] || return 0 + have=$(read_token "$AGENT_DIR/env-revision") + # A revision that failed its health check waits for the next one. + if [ "$want" = "$have" ] || [ "$want-failed" = "$have" ]; then + return 0 + fi + umask 077 + fresh=$(mktemp "$CONFIG_DIR/.node.env.XXXXXX") + if ! fetch_env "$fresh"; then + rm -f "$fresh" + return 0 + fi + revision=$(sha256sum "$fresh" | cut -c1-32) + for key in $PRESERVED_KEYS; do + grep "^$key=" "$CONFIG_DIR/node.env" | head -n 1 >> "$fresh" || true + done + chmod 600 "$fresh" + cp -p "$CONFIG_DIR/node.env" "$CONFIG_DIR/.node.env.previous" + mv -f "$fresh" "$CONFIG_DIR/node.env" + log "applied env revision $revision" + systemctl restart "$service" + if healthy; then + rm -f "$CONFIG_DIR/.node.env.previous" + write_agent env-revision "$revision" + return 0 + fi + log "env revision $revision did not stay up; restoring the previous env" + mv -f "$CONFIG_DIR/.node.env.previous" "$CONFIG_DIR/node.env" + systemctl restart "$service" + write_agent env-revision "$want-failed" +} + +# check_env lists, by name only, keys in node.env a refresh would drop. +check_env() { + umask 077 + fresh=$(mktemp "$CONFIG_DIR/.node.env.XXXXXX") + if ! fetch_env "$fresh"; then + rm -f "$fresh" + return 1 + fi + dropped="" + keys=$(sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG_DIR/node.env" | tr '\n' ' ') + for key in $keys; do + case " $PRESERVED_KEYS " in *" $key "*) continue ;; esac + grep -q "^$key=" "$fresh" && continue + if [ -f "$CONFIG_DIR/node.local.env" ] && grep -q "^$key=" "$CONFIG_DIR/node.local.env"; then + continue + fi + dropped="$dropped $key" + done + rm -f "$fresh" + if [ -n "$dropped" ]; then + log "a refresh would drop:$dropped (move them to node.local.env first)" + return 1 + fi + log "a refresh keeps every key node.env has" + return 0 +} + +main() { + [ -f "$CONFIG_DIR/node.env" ] || exit 0 + [ -f "$STATE_DIR/image" ] || exit 0 + role=$(env_value WARMBLY_NODE_ROLE) + case "$role" in + worker|consumer) ;; + *) exit 0 ;; + esac + service="warmbly-$role" + exec 8>"$ROOT/run/warmbly-node-update.lock" + flock -n 8 || exit 0 + if [ "${1:-}" = "--check-env" ]; then + check_env + return + fi + refresh_env + update_version + return 0 +} + +main "$@" UPDATER +} + +install_updater() { + render_updater > /usr/local/bin/warmbly-node-update chmod 755 /usr/local/bin/warmbly-node-update cat > /etc/systemd/system/warmbly-node-update.service <<'UNIT' @@ -583,7 +792,96 @@ OnUnitActiveSec=2min [Install] WantedBy=timers.target UNIT +} +# adopt puts an already-running node under the standard unit and updater +# without enrolling it again: node.env stays as it is, apart from opting in to +# env refresh when asked, and node.local.env gains only the keys named. +adopt() { + [ "$(id -u)" = "0" ] || die "run as root: --adopt installs a systemd unit" + need_cmd flock + need_cmd sha256sum + command -v docker >/dev/null 2>&1 || die "docker is required but not installed" + command -v systemctl >/dev/null 2>&1 || die "systemd is required" + [ -f "$CONFIG_DIR/node.env" ] || die "--adopt needs an existing $CONFIG_DIR/node.env" + NODE_ENV=$(cat "$CONFIG_DIR/node.env") + NODE_ID=$(printf '%s\n' "$NODE_ENV" | sed -n 's/^WARMBLY_NODE_ID=//p' | head -n 1) + valid_node_id "$NODE_ID" || die "node.env has no valid WARMBLY_NODE_ID; join this machine instead" + role=$(printf '%s\n' "$NODE_ENV" | sed -n 's/^WARMBLY_NODE_ROLE=//p' | head -n 1) + [ -n "$WARMBLY_ROLE" ] || WARMBLY_ROLE="$role" + [ "$WARMBLY_ROLE" = "$role" ] || die "node.env says role '$role', not '$WARMBLY_ROLE'" + case "$WARMBLY_ROLE" in + worker|consumer) ;; + *) die "node.env has no usable WARMBLY_NODE_ROLE" ;; + esac + current=$(printf '%s\n' "$NODE_ENV" | sed -n 's/^WARMBLY_VERSION=//p' | head -n 1) + target="" + if [ -f "$AGENT_DIR/target-version" ] && [ ! -L "$AGENT_DIR/target-version" ]; then + target=$(head -n 1 "$AGENT_DIR/target-version") + fi + if [ -n "$target" ] && [ "$target" != "$current" ] && [ "$ADOPT_ACCEPT_TARGET" != "true" ]; then + die "the control plane asks for $target and this node runs ${current:-an unknown version}; the updater would move it within two minutes. Set the fleet version first, or pass --accept-target." + fi + + mkdir -p "$STATE_DIR" "$AGENT_DIR" + chown 1000:1000 "$AGENT_DIR" 2>/dev/null || true + chmod 0700 "$AGENT_DIR" + printf '%s\n' "$WARMBLY_IMAGE_REPO/$WARMBLY_ROLE" > "$STATE_DIR/image" + ensure_local_env + keep_local + render_unit > "/etc/systemd/system/warmbly-$WARMBLY_ROLE.service" + # Keep this a standalone statement: make join-check asserts on it by first field. + install_updater + systemctl daemon-reload + if [ "$ADOPT_ENV_REFRESH" = "true" ] && ! grep -qx 'WARMBLY_ENV_REFRESH=true' "$CONFIG_DIR/node.env"; then + # Opt in only when a refresh would keep every key this node runs with. + /usr/local/bin/warmbly-node-update --check-env || die "env refresh left off: name the keys listed above with --keep-local and run --adopt again" + printf 'WARMBLY_ENV_REFRESH=true\n' >> "$CONFIG_DIR/node.env" + log "Enabled env refresh" + fi + systemctl enable --now warmbly-node-update.timer >/dev/null 2>&1 || true + log "Adopted $WARMBLY_ROLE node $NODE_ID. The updater runs every two minutes;" + log "run \`systemctl restart warmbly-$WARMBLY_ROLE\` to start it under the new unit." + return 0 +} + +# keep_local copies the named node.env keys into node.local.env, so a refresh +# that renders node.env afresh cannot drop a value only this machine knows. +keep_local() { + [ -n "$ADOPT_KEEP_LOCAL" ] || return 0 + for key in $(printf '%s' "$ADOPT_KEEP_LOCAL" | tr ',' ' '); do + case "$key" in + *[!A-Za-z0-9_]*|"") die "--keep-local takes variable names, got '$key'" ;; + esac + grep -q "^$key=" "$CONFIG_DIR/node.local.env" && continue + line=$(grep "^$key=" "$CONFIG_DIR/node.env" | head -n 1) || true + if [ -z "$line" ]; then + warn "$key is not in node.env; nothing to keep" + continue + fi + printf '%s\n' "$line" >> "$CONFIG_DIR/node.local.env" + log "Kept $key in node.local.env" + done + return 0 +} + +install_units() { + [ "$DRY_RUN" = "false" ] || return 0 + + service="warmbly-$WARMBLY_ROLE" + if blobs_are_local; then + root=$(blob_root) + if [ -n "$root" ]; then + # Keep this a standalone statement: make join-check asserts on it by + # first field, because a looser match was satisfied by the name + # appearing inside a warn string. + ensure_blob_root "$root" + fi + fi + render_unit > "/etc/systemd/system/$service.service" + + # Keep this a standalone statement: make join-check asserts on it by first field. + install_updater systemctl daemon-reload log "Installed $service.service and warmbly-node-update.timer" } @@ -618,6 +916,14 @@ main() { # first field, so each stays a standalone statement. Reformatting one into # `if ! x; then` fails the build with a message about ordering. parse_args "$@" + if [ "$PRINT_UPDATER" = "true" ]; then + render_updater + return 0 + fi + if [ "$ADOPT" = "true" ]; then + adopt + return 0 + fi if [ "$PRINT_UNIT" = "true" ]; then # No enrolment, no network, no files. NODE_ENV is whatever the caller # supplied, so the blob branches can be exercised from a test. diff --git a/internal/api/routes.go b/internal/api/routes.go index b5c8c5646..097ff3768 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -185,6 +185,9 @@ func Run( // internal API moves. broker.GET("/cloud-link/token/:id", h.InternalCloudLinkToken) + // The env an enrolled node should have, pulled by its host updater. + broker.GET("/fleet/node-env", h.InternalFleetNodeEnv) + node.GET("/dek/:orgID", h.InternalGetDEK) node.PUT("/dek/:orgID", h.InternalPutDEK) // No DELETE: a lost DEK is unrecoverable, so nothing holding this token may remove one. @@ -1820,6 +1823,8 @@ func Run( // Platform mail diagnostics. A broken relay locks everyone out, so the // operator needs to see the SMTP dialogue from inside the panel rather // than inferring it from a 500 on the login screen. + // Instance key rotation status. Read only; rotation itself runs where the keys are stored. + adminRoutes.GET("/keys/status", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminKeyStatus) adminRoutes.GET("/mail/status", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminMailStatus) adminRoutes.POST("/mail/test", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminSendTestEmail) diff --git a/internal/app/fleetnode/nodeenv.go b/internal/app/fleetnode/nodeenv.go new file mode 100644 index 000000000..59e5caf20 --- /dev/null +++ b/internal/app/fleetnode/nodeenv.go @@ -0,0 +1,213 @@ +package fleetnode + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/models" +) + +// NodeEnvKeys are the settings a node needs to do its job, in the order they +// are written. Every one is read from the backend's own environment, so a node +// is configured with exactly the infrastructure the control plane uses and +// there is nothing to keep in sync by hand. +// +// Every name here must be one the node's own code reads. Two of them were not +// (S3_BUCKET and KMS_KEY_ID, against a storage layer reading BLOB_BUCKET and a +// KMS factory reading KMS_AWS_KEY_ID), which sent an AWS-backed node to the +// default bucket and the default key alias with nothing logged. +// +// KMS_PROVIDER and BLOB_PROVIDER are absent because they are not copied but +// translated; see nodeProviders. The AWS-shaped settings above stay, so an +// operator who deliberately overrides a node back to a direct provider in +// node.local.env only has to add the credential. +// +// Deliberately excluded: PRIMARY_DB for a worker, and anything else that would +// give one direct database access. A worker reaches relational data through +// the internal API and nothing else. RenderEnv sends the DSN to a consumer, +// which is control plane and updates relational state itself. +var NodeEnvKeys = []string{ + "APP_ENV", + "EVENTBUS_PROVIDER", + "NATS_URL", + // Base64, not a path: a node has no file to point at, and the env file + // docker reads cannot hold a multi-line value. + "NATS_CREDS_B64", + "NATS_MAX_BYTES", + "KAFKA_BOOTSTRAP_SERVERS", + "KAFKA_SASL_USERNAME", + "KAFKA_SASL_PASSWORD", + "SCHEMA_REGISTRY_URL", + "SCHEMA_REGISTRY_KEY", + "SCHEMA_REGISTRY_SECRET", + "CODEC_PROVIDER", + "REDIS", + "KMS_LOCAL_MASTER_KEY", + "KMS_AWS_KEY_ID", + "CREDENTIALS_ENCRYPTION_KEY", + // Earlier keys stay readable on nodes while a rotation reseals. + "CREDENTIALS_ENCRYPTION_READ_KEYS", + "BLOB_FS_ROOT", + "BLOB_BUCKET", + "AWS_REGION", + "AWS_ENDPOINT_URL_S3", + "BOX_GOOGLE_CLIENT_ID", + "BOX_GOOGLE_CLIENT_SECRET", + "BOX_OUTLOOK_CLIENT_ID", + "BOX_OUTLOOK_CLIENT_SECRET", + "MAIL_TLS_INSECURE", + "POSTHOG_KEY", + "POSTHOG_HOST", + "POSTHOG_ERROR_TRACKING", + "SENTRY_DSN", +} + +// consumerEnvKeys reach a consumer and never a worker. A consumer runs the +// integration actions, refreshes their OAuth tokens and drains the CRM outbox, +// so it reads the integration client credentials (integration.NewOAuthManager), +// APP_URL (the "Open in Warmbly" link written to the CRM) and the backend's +// public URL (where it registers Pipedrive's webhooks). +var consumerEnvKeys = []string{ + "APP_URL", + "BACKEND_PUBLIC_URL", + "API_PUBLIC_URL", + "HUBSPOT_OAUTH_CLIENT_ID", + "HUBSPOT_OAUTH_CLIENT_SECRET", + "SLACK_OAUTH_CLIENT_ID", + "SLACK_OAUTH_CLIENT_SECRET", + "GOOGLE_SHEETS_OAUTH_CLIENT_ID", + "GOOGLE_SHEETS_OAUTH_CLIENT_SECRET", + "PIPEDRIVE_OAUTH_CLIENT_ID", + "PIPEDRIVE_OAUTH_CLIENT_SECRET", + "SALESFORCE_OAUTH_CLIENT_ID", + "SALESFORCE_OAUTH_CLIENT_SECRET", +} + +// nodeProviders translates the control plane's own crypto and blob providers +// into the ones a node should run. +// +// A node has no cloud credentials and no way to be handed any: node.env is +// regenerated from this instance's environment on every join, and shipping an +// IAM key to every machine in the fleet is exactly the thing worth avoiding. So +// a provider that needs a credential becomes its brokered form, which +// authenticates with the internal API token the node already holds and asks +// this instance to do the one privileged operation. A provider that needs no +// credential (local KMS, filesystem blobs) passes through unchanged. +// +// Brokering costs one HTTPS call to the control plane per DEK open (Redis +// caches the result) and per blob operation. The bytes still go straight +// between the node and the object store. +func nodeProviders() (kmsProvider, blobProvider string) { + kmsProvider = config.KMSProvider() + if kmsProvider == "aws" || kmsProvider == "aws-kms" { + kmsProvider = "brokered" + } + blobProvider = config.BlobProvider() + if blobProvider == "s3" { + blobProvider = "brokered" + } + return kmsProvider, blobProvider +} + +// RenderEnv builds the env file a node writes to disk on join. +func RenderEnv(nodeID uuid.UUID, role models.NodeRole, region string) string { + var b strings.Builder + b.WriteString("# Written by `warmbly join`. Regenerate by joining again.\n") + fmt.Fprintf(&b, "WARMBLY_NODE_ID=%s\n", nodeID) + fmt.Fprintf(&b, "WARMBLY_NODE_ROLE=%s\n", role) + if region != "" { + fmt.Fprintf(&b, "WARMBLY_NODE_REGION=%s\n", region) + } + // A worker resolves its own identity from WORKER_ID; keeping the two equal + // means the placement row and the node row are the same machine. + if role == models.NodeRoleWorker { + fmt.Fprintf(&b, "WORKER_ID=%s\n", nodeID) + } + + backend := strings.TrimRight(os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"), "/") + if backend == "" { + backend = strings.TrimRight(os.Getenv("APP_INTERNAL_URL"), "/") + } + // A consumer is control plane: it opens Postgres itself, so it gets the DSN + // a worker is deliberately never given, and reads keys straight from the + // table rather than back through the API it sits behind. An instance whose + // DSN lives in SSM rather than the environment has nothing to send, so the + // node falls back to the worker's HTTP key path and join.sh tells the + // operator to supply PRIMARY_DB in node.local.env. + dsn := os.Getenv("PRIMARY_DB") + keysProvider := "http" + if role == models.NodeRoleConsumer && dsn != "" { + keysProvider = "postgres" + fmt.Fprintf(&b, "PRIMARY_DB=%s\n", dsn) + } + + fmt.Fprintf(&b, "WARMBLY_BACKEND_URL=%s\n", backend) + fmt.Fprintf(&b, "ENCRYPTED_KEYS_PROVIDER=%s\n", keysProvider) + fmt.Fprintf(&b, "ENCRYPTED_KEYS_BACKEND_URL=%s\n", backend) + // A node calls only the node-only internal routes. With a separate + // NODE_BROKER_TOKEN it is sent that alone and never the edge services' token. + if v := os.Getenv("NODE_BROKER_TOKEN"); v != "" { + fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", v) + fmt.Fprintf(&b, "NODE_BROKER_TOKEN=%s\n", v) + } else { + fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN")) + fmt.Fprintf(&b, "INTERNAL_API_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN")) + } + + kmsProvider, blobProvider := nodeProviders() + fmt.Fprintf(&b, "KMS_PROVIDER=%s\n", kmsProvider) + fmt.Fprintf(&b, "BLOB_PROVIDER=%s\n", blobProvider) + + keys := NodeEnvKeys + if role == models.NodeRoleConsumer { + keys = append(append([]string{}, NodeEnvKeys...), consumerEnvKeys...) + } + for _, k := range keys { + if v := os.Getenv(k); v != "" { + fmt.Fprintf(&b, "%s=%s\n", k, v) + } + } + return b.String() +} + +// EnvRevision names one rendered env, so a node can tell its file is stale. +func EnvRevision(env string) string { + sum := sha256.Sum256([]byte(env)) + return hex.EncodeToString(sum[:16]) +} + +// NodeEnvState is whether one live node runs the env the backend renders now. +type NodeEnvState struct { + ID string `json:"id"` + Name string `json:"name"` + Role string `json:"role"` + Applied string `json:"applied,omitempty"` + Tracked bool `json:"tracked"` + Current bool `json:"current"` +} + +// EnvStatus is the fleet's answer to "has every node picked up the current env". +type EnvStatus struct { + AllCurrent bool `json:"all_current"` + Nodes []NodeEnvState `json:"nodes"` +} + +// EnvStatusOf compares each live node's applied revision with the env this process renders for it. +func EnvStatusOf(nodes []models.FleetNode) EnvStatus { + out := EnvStatus{AllCurrent: true, Nodes: []NodeEnvState{}} + for _, n := range nodes { + if !n.Live() { + continue + } + want := EnvRevision(RenderEnv(n.ID, n.Role, n.Region)) + row := NodeEnvState{ID: n.ID.String(), Name: n.Name, Role: string(n.Role), Applied: n.EnvRevision, Tracked: n.EnvRevision != "", Current: n.EnvRevision == want} + out.AllCurrent = out.AllCurrent && row.Current + out.Nodes = append(out.Nodes, row) + } + return out +} diff --git a/internal/app/fleetnode/rollout.go b/internal/app/fleetnode/rollout.go new file mode 100644 index 000000000..d61bfb2fe --- /dev/null +++ b/internal/app/fleetnode/rollout.go @@ -0,0 +1,294 @@ +package fleetnode + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "log" + "sort" + "strings" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +// ErrRolloutConflict means another backend changed the release state first. +var ErrRolloutConflict = errors.New("fleet release state changed concurrently; retry") + +// ErrNoRollout means there is no rollout to act on. +var ErrNoRollout = errors.New("no fleet rollout is in progress") + +func (s *Service) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +// StartRollout moves the fleet to target in stages: a canary first, then the +// policy's waves, each admitted only after the previous one has baked. +// Starting the rollout already running for target is a no-op. +func (s *Service) StartRollout(ctx context.Context, target string, digests map[string]string, policy models.FleetRolloutPolicy) (*models.FleetReleaseState, error) { + target = strings.TrimSpace(target) + if target == "" { + return nil, errors.New("tag required") + } + if s.variant != "" { + target = strings.TrimSuffix(target, s.variant) + } + if err := s.CheckTarget(target); err != nil { + return nil, err + } + prev, err := s.settings.GetRelease(ctx) + if err != nil { + return nil, err + } + if prev != nil && prev.Rollout.Running() && prev.Rollout.Target == target { + return prev, nil + } + now := s.clock() + next := &models.FleetReleaseState{Channel: models.FleetChannelPinned, ResolvedAt: now, Source: "rollout"} + if prev != nil { + next.Tag = prev.Tag + next.ResolvedAt = prev.ResolvedAt + next.Source = prev.Source + } + if next.Tag == target { + next.Rollout = nil + return s.commit(ctx, prev, next) + } + clean := map[string]string{} + for k, v := range digests { + if d := validDigest(strings.TrimSpace(v)); d != "" { + clean[strings.TrimSpace(k)] = d + } + } + r := &models.FleetRollout{ + Target: target, + From: next.Tag, + Digests: clean, + Policy: policy.Normalize(), + Stage: models.FleetRolloutCanary, + StartedAt: now, + StageStartedAt: now, + } + candidates, err := s.rolloutCandidates(ctx, target) + if err != nil { + return nil, err + } + if len(candidates) == 0 { + return s.commit(ctx, prev, finish(next, r, now)) + } + n := r.Policy.Canary + if n > len(candidates) { + n = len(candidates) + } + for _, c := range candidates[:n] { + r.Admitted = append(r.Admitted, c.ID) + } + next.Rollout = r + return s.commit(ctx, prev, next) +} + +// AdvanceRollout evaluates the running rollout once: it halts on a failed or +// silent node, admits the next stage once the current one has baked, and +// finishes when every node is admitted. force skips the bake, never the checks. +func (s *Service) AdvanceRollout(ctx context.Context, force bool) (*models.FleetReleaseState, error) { + prev, err := s.settings.GetRelease(ctx) + if err != nil { + return nil, err + } + if prev == nil || !prev.Rollout.Running() { + return prev, nil + } + nodes, err := s.nodes.List(ctx, "") + if err != nil { + return nil, err + } + now := s.clock() + r := *prev.Rollout + r.Admitted = append([]uuid.UUID(nil), prev.Rollout.Admitted...) + next := *prev + next.Rollout = &r + + if reason, done := s.stageVerdict(&r, nodes, now); reason != "" { + r.Stage = models.FleetRolloutHalted + r.HaltReason = reason + log.Printf("fleet: rollout to %s halted: %s", r.Target, reason) + return s.commit(ctx, prev, &next) + } else if !done { + return prev, nil + } else if !force && now.Sub(r.StageStartedAt) < time.Duration(r.Policy.BakeSeconds)*time.Second { + return prev, nil + } + + candidates, err := s.rolloutCandidates(ctx, r.Target) + if err != nil { + return nil, err + } + remaining := make([]models.FleetNode, 0, len(candidates)) + for _, c := range candidates { + if !r.IsAdmitted(c.ID) { + remaining = append(remaining, c) + } + } + wave := 0 + if r.Stage == models.FleetRolloutWave { + wave = r.Wave + 1 + } + total := len(r.Admitted) + len(remaining) + for ; wave < len(r.Policy.Waves) && len(remaining) > 0; wave++ { + want := (total*r.Policy.Waves[wave] + 99) / 100 + added := 0 + for len(r.Admitted) < want && len(remaining) > 0 { + r.Admitted = append(r.Admitted, remaining[0].ID) + remaining = remaining[1:] + added++ + } + if added > 0 { + r.Stage = models.FleetRolloutWave + r.Wave = wave + r.StageStartedAt = now + log.Printf("fleet: rollout to %s admitted wave %d (%d of %d nodes)", r.Target, wave+1, len(r.Admitted), total) + return s.commit(ctx, prev, &next) + } + } + log.Printf("fleet: rollout to %s finished on %d nodes", r.Target, len(r.Admitted)) + return s.commit(ctx, prev, finish(&next, &r, now)) +} + +// HaltRollout stops admitting nodes and sends admitted ones back to From. +func (s *Service) HaltRollout(ctx context.Context, reason string) (*models.FleetReleaseState, error) { + prev, err := s.settings.GetRelease(ctx) + if err != nil { + return nil, err + } + if prev == nil || !prev.Rollout.Running() { + return nil, ErrNoRollout + } + r := *prev.Rollout + r.Stage = models.FleetRolloutHalted + if reason = strings.TrimSpace(reason); reason == "" { + reason = "halted by an operator" + } + r.HaltReason = reason + next := *prev + next.Rollout = &r + return s.commit(ctx, prev, &next) +} + +// AbortRollout discards the rollout, running or halted, leaving Tag at From. +func (s *Service) AbortRollout(ctx context.Context) (*models.FleetReleaseState, error) { + prev, err := s.settings.GetRelease(ctx) + if err != nil { + return nil, err + } + if prev == nil || prev.Rollout == nil { + return nil, ErrNoRollout + } + next := *prev + next.Rollout = nil + return s.commit(ctx, prev, &next) +} + +// RunRolloutLoop advances any running rollout every interval until ctx ends. +func (s *Service) RunRolloutLoop(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if _, err := s.AdvanceRollout(ctx, false); err != nil && !errors.Is(err, ErrRolloutConflict) { + log.Printf("fleet: rollout advance: %v", err) + } + } + } +} + +// stageVerdict returns a halt reason, or whether every admitted node has +// reached the target and stayed up for the bake. +func (s *Service) stageVerdict(r *models.FleetRollout, nodes []models.FleetNode, now time.Time) (string, bool) { + byID := make(map[uuid.UUID]*models.FleetNode, len(nodes)) + for i := range nodes { + byID[nodes[i].ID] = &nodes[i] + } + target := s.withVariant(r.Target) + bake := time.Duration(r.Policy.BakeSeconds) * time.Second + deadline := time.Duration(r.Policy.DeadlineSeconds) * time.Second + elapsed := now.Sub(r.StageStartedAt) + done := true + for _, id := range r.Admitted { + n := byID[id] + switch { + case n == nil: + return fmt.Sprintf("node %s was removed during the rollout", id), false + case n.PinnedVersion != "": + continue + case !n.Live(): + return fmt.Sprintf("node %s (%s) stopped heartbeating", n.Name, id), false + case n.UpdateFailed != "" && n.UpdateFailed == target: + return fmt.Sprintf("node %s (%s) rolled back from %s on the host", n.Name, id, target), false + case n.Version != target: + if elapsed > deadline { + return fmt.Sprintf("node %s (%s) did not reach %s within %s", n.Name, id, target, deadline), false + } + done = false + case n.Usage.UptimeSeconds == nil || time.Duration(*n.Usage.UptimeSeconds)*time.Second < bake: + if elapsed > deadline+2*bake { + return fmt.Sprintf("node %s (%s) kept restarting on %s", n.Name, id, target), false + } + done = false + } + } + return "", done +} + +// rolloutCandidates are live, unpinned nodes in admission order: workers +// first, then a stable shuffle per target so no machine always goes first. +func (s *Service) rolloutCandidates(ctx context.Context, target string) ([]models.FleetNode, error) { + nodes, err := s.nodes.List(ctx, "") + if err != nil { + return nil, err + } + out := make([]models.FleetNode, 0, len(nodes)) + for _, n := range nodes { + if n.Live() && n.PinnedVersion == "" { + out = append(out, n) + } + } + rank := func(n models.FleetNode) string { + h := sha256.Sum256([]byte(target + "/" + n.ID.String())) + return fmt.Sprintf("%x", h[:8]) + } + sort.SliceStable(out, func(i, j int) bool { + if (out[i].Role == models.NodeRoleWorker) != (out[j].Role == models.NodeRoleWorker) { + return out[i].Role == models.NodeRoleWorker + } + return rank(out[i]) < rank(out[j]) + }) + return out, nil +} + +func finish(next *models.FleetReleaseState, r *models.FleetRollout, now time.Time) *models.FleetReleaseState { + next.Tag = r.Target + next.Channel = models.FleetChannelPinned + next.ResolvedAt = now + next.Source = "rollout" + next.Rollout = nil + return next +} + +func (s *Service) commit(ctx context.Context, prev, next *models.FleetReleaseState) (*models.FleetReleaseState, error) { + ok, err := s.settings.CompareAndSetRelease(ctx, prev, next) + if err != nil { + return nil, err + } + if !ok { + return nil, ErrRolloutConflict + } + return next, nil +} diff --git a/internal/app/fleetnode/rollout_test.go b/internal/app/fleetnode/rollout_test.go new file mode 100644 index 000000000..beb31bd73 --- /dev/null +++ b/internal/app/fleetnode/rollout_test.go @@ -0,0 +1,283 @@ +package fleetnode + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type fleetNodes struct { + repository.FleetNodeRepository + nodes []models.FleetNode +} + +func (f *fleetNodes) Get(_ context.Context, id uuid.UUID) (*models.FleetNode, error) { + for i := range f.nodes { + if f.nodes[i].ID == id { + n := f.nodes[i] + return &n, nil + } + } + return nil, nil +} + +func (f *fleetNodes) List(context.Context, models.NodeRole) ([]models.FleetNode, error) { + return append([]models.FleetNode(nil), f.nodes...), nil +} + +// casSettings stores the state as JSON, the way admin_settings does, and +// refuses a write whose prev no longer matches. +type casSettings struct { + repository.FleetSettingsRepository + raw []byte + conflict bool +} + +func (c *casSettings) GetRelease(context.Context) (*models.FleetReleaseState, error) { + if c.raw == nil { + return nil, nil + } + var st models.FleetReleaseState + return &st, json.Unmarshal(c.raw, &st) +} + +func (c *casSettings) CompareAndSetRelease(_ context.Context, prev, next *models.FleetReleaseState) (bool, error) { + if c.conflict { + return false, nil + } + if prev == nil { + if c.raw != nil { + return false, nil + } + } else { + old, _ := json.Marshal(prev) + if string(old) != string(c.raw) { + return false, nil + } + } + c.raw, _ = json.Marshal(next) + return true, nil +} + +func (c *casSettings) state(t *testing.T) *models.FleetReleaseState { + t.Helper() + st, err := c.GetRelease(context.Background()) + if err != nil { + t.Fatal(err) + } + return st +} + +type rolloutFixture struct { + svc *Service + nodes *fleetNodes + settings *casSettings + now time.Time +} + +func newRolloutFixture(t *testing.T, workers, consumers int) *rolloutFixture { + t.Helper() + t.Setenv("WARMBLY_VERSION", "v1.5.0") + f := &rolloutFixture{nodes: &fleetNodes{}, settings: &casSettings{}, now: time.Date(2026, 10, 24, 12, 0, 0, 0, time.UTC)} + seen := f.now + add := func(role models.NodeRole) { + f.nodes.nodes = append(f.nodes.nodes, models.FleetNode{ + ID: uuid.New(), Role: role, Name: string(role), Version: "v1.4.0-kafka", Active: true, LastSeenAt: &seen, + }) + } + for i := 0; i < workers; i++ { + add(models.NodeRoleWorker) + } + for i := 0; i < consumers; i++ { + add(models.NodeRoleConsumer) + } + raw, _ := json.Marshal(models.FleetReleaseState{Channel: models.FleetChannelPinned, Tag: "v1.4.0"}) + f.settings.raw = raw + f.svc = &Service{nodes: f.nodes, settings: f.settings, variant: "-kafka", now: func() time.Time { return f.now }} + return f +} + +// advanceTime moves the clock and keeps every node beating, so liveness is +// judged against the fixture clock rather than the wall clock. +func (f *rolloutFixture) advanceTime(d time.Duration) { + f.now = f.now.Add(d) + seen := time.Now() + for i := range f.nodes.nodes { + if f.nodes.nodes[i].Active { + f.nodes.nodes[i].LastSeenAt = &seen + } + } +} + +// converge makes every admitted node report the target and an uptime. +func (f *rolloutFixture) converge(admitted []uuid.UUID, version string, uptime time.Duration) { + up := int64(uptime.Seconds()) + for i := range f.nodes.nodes { + for _, id := range admitted { + if f.nodes.nodes[i].ID == id { + f.nodes.nodes[i].Version = version + f.nodes.nodes[i].Usage.UptimeSeconds = &up + } + } + } +} + +const testDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestRolloutCanaryThenWavesThenDone(t *testing.T) { + f := newRolloutFixture(t, 4, 1) + ctx := context.Background() + policy := models.FleetRolloutPolicy{Canary: 1, Waves: []int{50, 100}, BakeSeconds: 600, DeadlineSeconds: 300} + st, err := f.svc.StartRollout(ctx, "v1.5.0-kafka", map[string]string{"worker-kafka": testDigest, "consumer-kafka": "not-a-digest"}, policy) + if err != nil { + t.Fatal(err) + } + r := st.Rollout + if st.Tag != "v1.4.0" || r.Target != "v1.5.0" || r.From != "v1.4.0" || len(r.Admitted) != 1 { + t.Fatalf("unexpected start state: tag=%s rollout=%+v", st.Tag, r) + } + if _, ok := r.Digests["consumer-kafka"]; ok { + t.Fatal("an invalid digest must be dropped, not handed to a node") + } + canary, _ := f.nodes.Get(ctx, r.Admitted[0]) + if canary.Role != models.NodeRoleWorker { + t.Fatal("the canary must be a worker") + } + + // The canary is told the target and digest; everyone else stays on From. + for _, n := range f.nodes.nodes { + v, d := f.svc.desired(ctx, n.ID) + if n.ID == canary.ID { + if v != "v1.5.0-kafka" || d != testDigest { + t.Fatalf("canary got %q %q", v, d) + } + } else if v != "v1.4.0-kafka" || d != "" { + t.Fatalf("non-admitted node got %q %q", v, d) + } + } + + // Not converged: holds. + f.advanceTime(time.Minute) + if st, _ = f.svc.AdvanceRollout(ctx, false); len(st.Rollout.Admitted) != 1 { + t.Fatal("must not advance before the canary reports the target") + } + // Converged but not baked: holds. + f.converge(st.Rollout.Admitted, "v1.5.0-kafka", time.Minute) + if st, _ = f.svc.AdvanceRollout(ctx, false); st.Rollout.Stage != models.FleetRolloutCanary { + t.Fatal("must not advance before the canary has stayed up for the bake") + } + // Baked: wave one takes the fleet to half. + f.advanceTime(10 * time.Minute) + f.converge(st.Rollout.Admitted, "v1.5.0-kafka", 11*time.Minute) + st, err = f.svc.AdvanceRollout(ctx, false) + if err != nil { + t.Fatal(err) + } + if st.Rollout.Stage != models.FleetRolloutWave || len(st.Rollout.Admitted) != 3 { + t.Fatalf("wave one should admit up to 50%% of 5 nodes, got %d in %s", len(st.Rollout.Admitted), st.Rollout.Stage) + } + f.advanceTime(11 * time.Minute) + f.converge(st.Rollout.Admitted, "v1.5.0-kafka", 11*time.Minute) + st, _ = f.svc.AdvanceRollout(ctx, false) + if len(st.Rollout.Admitted) != 5 { + t.Fatalf("last wave should admit everyone, got %d", len(st.Rollout.Admitted)) + } + f.advanceTime(11 * time.Minute) + f.converge(st.Rollout.Admitted, "v1.5.0-kafka", 11*time.Minute) + st, _ = f.svc.AdvanceRollout(ctx, false) + if st.Rollout != nil || st.Tag != "v1.5.0" { + t.Fatalf("rollout should finish with Tag at the target, got tag=%s rollout=%+v", st.Tag, st.Rollout) + } +} + +func TestRolloutHaltsAndRevertsOnHostRollback(t *testing.T) { + f := newRolloutFixture(t, 3, 0) + ctx := context.Background() + st, err := f.svc.StartRollout(ctx, "v1.5.0", nil, models.FleetRolloutPolicy{}) + if err != nil { + t.Fatal(err) + } + canary := st.Rollout.Admitted[0] + for i := range f.nodes.nodes { + if f.nodes.nodes[i].ID == canary { + f.nodes.nodes[i].UpdateFailed = "v1.5.0-kafka" + } + } + st, err = f.svc.AdvanceRollout(ctx, false) + if err != nil { + t.Fatal(err) + } + if st.Rollout.Stage != models.FleetRolloutHalted || st.Rollout.HaltReason == "" { + t.Fatalf("a host rollback must halt the rollout, got %+v", st.Rollout) + } + if v, _ := f.svc.desired(ctx, canary); v != "v1.4.0-kafka" { + t.Fatalf("a halted rollout must send the canary back to From, got %q", v) + } + if st, _ = f.svc.AdvanceRollout(ctx, true); st.Rollout.Stage != models.FleetRolloutHalted { + t.Fatal("forcing must never resume a halted rollout") + } +} + +func TestRolloutHaltsOnDeadlineAndSilentNode(t *testing.T) { + f := newRolloutFixture(t, 2, 0) + ctx := context.Background() + st, _ := f.svc.StartRollout(ctx, "v1.5.0", nil, models.FleetRolloutPolicy{DeadlineSeconds: 300}) + f.advanceTime(6 * time.Minute) + if st, _ = f.svc.AdvanceRollout(ctx, false); st.Rollout.Stage != models.FleetRolloutHalted { + t.Fatal("an admitted node missing the deadline must halt the rollout") + } + + g := newRolloutFixture(t, 2, 0) + st, _ = g.svc.StartRollout(ctx, "v1.5.0", nil, models.FleetRolloutPolicy{}) + for i := range g.nodes.nodes { + if g.nodes.nodes[i].ID == st.Rollout.Admitted[0] { + old := time.Now().Add(-time.Hour) + g.nodes.nodes[i].LastSeenAt = &old + } + } + if st, _ = g.svc.AdvanceRollout(ctx, false); st.Rollout.Stage != models.FleetRolloutHalted { + t.Fatal("an admitted node that stops beating must halt the rollout") + } +} + +func TestRolloutStartIsIdempotentAndPinWins(t *testing.T) { + f := newRolloutFixture(t, 2, 0) + ctx := context.Background() + first, _ := f.svc.StartRollout(ctx, "v1.5.0", nil, models.FleetRolloutPolicy{}) + again, err := f.svc.StartRollout(ctx, "v1.5.0-kafka", nil, models.FleetRolloutPolicy{}) + if err != nil || again.Rollout.Admitted[0] != first.Rollout.Admitted[0] { + t.Fatal("starting the running rollout again must change nothing") + } + f.nodes.nodes[0].PinnedVersion = "v1.3.0" + if v, d := f.svc.desired(ctx, f.nodes.nodes[0].ID); v != "v1.3.0-kafka" || d != "" { + t.Fatalf("a pin must win over a rollout, got %q %q", v, d) + } +} + +func TestRolloutConflictIsReported(t *testing.T) { + f := newRolloutFixture(t, 1, 0) + f.settings.conflict = true + if _, err := f.svc.StartRollout(context.Background(), "v1.5.0", nil, models.FleetRolloutPolicy{}); !errors.Is(err, ErrRolloutConflict) { + t.Fatalf("a lost compare-and-set must be reported, got %v", err) + } +} + +// A backend that predates rollouts reads only Tag, which stays at From. +func TestRolloutStateIsSafeForOlderReaders(t *testing.T) { + f := newRolloutFixture(t, 2, 0) + if _, err := f.svc.StartRollout(context.Background(), "v1.5.0", nil, models.FleetRolloutPolicy{}); err != nil { + t.Fatal(err) + } + var old struct { + Tag string `json:"tag"` + } + if err := json.Unmarshal(f.settings.raw, &old); err != nil || old.Tag != "v1.4.0" { + t.Fatalf("older readers must still see From, got %q", old.Tag) + } +} diff --git a/internal/app/fleetnode/service.go b/internal/app/fleetnode/service.go index bd86fb712..7e41878b2 100644 --- a/internal/app/fleetnode/service.go +++ b/internal/app/fleetnode/service.go @@ -49,6 +49,9 @@ type Service struct { // a version names an image and some builds of an image are not // interchangeable. See imageVariant. variant string + + // now is the clock rollouts are judged by; nil means time.Now. + now func() time.Time } func New( @@ -193,7 +196,7 @@ func (s *Service) Heartbeat(ctx context.Context, beat models.NodeHeartbeat) (*mo reply := &models.NodeHeartbeatReply{ LivenessSeconds: int(models.NodeLivenessWindow.Seconds()), } - reply.DesiredVersion = s.desiredVersion(ctx, beat.NodeID) + reply.DesiredVersion, reply.DesiredDigest = s.desired(ctx, beat.NodeID) return reply, nil } @@ -219,20 +222,59 @@ func (s *Service) UpdateOnly(ctx context.Context, beat models.NodeHeartbeat) (*m } // desiredVersion resolves what this node should run: its own pin if it has -// one, otherwise the fleet-wide resolved release. +// one, otherwise the fleet-wide resolved release, or a staged rollout's target +// once the node has been admitted to it. // // Any failure answers "" — no opinion. A node that cannot be told what to run // must keep running what it has, because the alternative is a control-plane // hiccup rolling the whole fleet. func (s *Service) desiredVersion(ctx context.Context, nodeID uuid.UUID) string { - if node, err := s.nodes.Get(ctx, nodeID); err == nil && node != nil && node.PinnedVersion != "" { - return s.withVariant(node.PinnedVersion) + v, _ := s.desired(ctx, nodeID) + return v +} + +func (s *Service) desired(ctx context.Context, nodeID uuid.UUID) (version, digest string) { + node, err := s.nodes.Get(ctx, nodeID) + if err == nil && node != nil && node.PinnedVersion != "" { + return s.withVariant(node.PinnedVersion), "" } state, err := s.settings.GetRelease(ctx) if err != nil { + return "", "" + } + role := models.NodeRole("") + if node != nil { + role = node.Role + } + return s.resolve(state, nodeID, role) +} + +// resolve answers what an unpinned node runs under state. A halted rollout +// answers Tag, which is From, so admitted nodes roll themselves back. +func (s *Service) resolve(state *models.FleetReleaseState, nodeID uuid.UUID, role models.NodeRole) (version, digest string) { + if state == nil { + return "", "" + } + if r := state.Rollout; r.Running() && r.IsAdmitted(nodeID) { + version = s.withVariant(r.Target) + if version != "" && role != "" { + digest = validDigest(r.Digests[string(role)+s.variant]) + } + return version, digest + } + return s.withVariant(state.DesiredVersion()), "" +} + +func validDigest(d string) string { + if len(d) != len("sha256:")+64 || !strings.HasPrefix(d, "sha256:") { return "" } - return s.withVariant(state.DesiredVersion()) + for _, c := range d[len("sha256:"):] { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return "" + } + } + return d } // JoinVersion falls back to the backend's exact release, never a floating tag. @@ -254,13 +296,12 @@ func (s *Service) List(ctx context.Context, role models.NodeRole) ([]models.Flee if err != nil { return nil, err } - fleetTarget := s.withVariant(state.DesiredVersion()) for i := range nodes { if nodes[i].PinnedVersion != "" { nodes[i].DesiredVersion = s.withVariant(nodes[i].PinnedVersion) continue } - nodes[i].DesiredVersion = fleetTarget + nodes[i].DesiredVersion, _ = s.resolve(state, nodes[i].ID, nodes[i].Role) } return nodes, nil } @@ -279,6 +320,6 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*models.FleetNode, err if err != nil { return nil, err } - node.DesiredVersion = s.withVariant(state.DesiredVersion()) + node.DesiredVersion, _ = s.resolve(state, node.ID, node.Role) return node, nil } diff --git a/internal/app/keystatus/keystatus.go b/internal/app/keystatus/keystatus.go new file mode 100644 index 000000000..920d1113b --- /dev/null +++ b/internal/app/keystatus/keystatus.go @@ -0,0 +1,324 @@ +// Package keystatus reports, and optionally reseals, the values the instance-wide keys seal. +package keystatus + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/warmbly/warmbly/internal/app/mailcred" + "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/app/twofa" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/pkg/encrypt" + "github.com/warmbly/warmbly/internal/pkg/secretset" +) + +// A rotation ships the new primary with the old key as a read key, reseals, and only then retires the old key. + +// Domain names which configured key seals a column. +type Domain string + +const ( + DomainInstance Domain = "instance" + DomainMFA Domain = "mfa" +) + +// SealedColumn is one column holding values sealed under an instance-wide key. +type SealedColumn struct { + Table string `json:"table"` + Column string `json:"column"` + Domain Domain `json:"domain"` + guard string +} + +// instanceOnlyColumns are sealed under an instance-wide key on tables that never travel in a workspace archive. +var instanceOnlyColumns = []SealedColumn{ + {Table: "cloud_link", Column: "token", Domain: DomainInstance}, + {Table: "mailbox_erasures", Column: "refresh_token", Domain: DomainInstance}, + {Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: DomainMFA}, +} + +// Columns is every column the instance keys seal: the archive registry's instance-key and mailbox secrets plus the instance-only ones. +func Columns() []SealedColumn { + var out []SealedColumn + for _, t := range orgtransfer.Tables { + for _, s := range t.Secrets { + if s.Domain == orgtransfer.KeyDomainInstance || s.Domain == orgtransfer.KeyDomainMailbox { + out = append(out, SealedColumn{Table: t.Name, Column: s.Column, Domain: DomainInstance, guard: s.Guard}) + } + } + } + return append(out, instanceOnlyColumns...) +} + +// Count is where one column's values stand against the configured keys. +type Count struct { + SealedColumn + Primary int `json:"primary"` + ReadKey int `json:"read_key"` + Workspace int `json:"workspace"` + Unsealed int `json:"unsealed"` + Unreadable int `json:"unreadable"` + Resealed int `json:"resealed,omitempty"` + Raced int `json:"changed_meanwhile,omitempty"` + Missing bool `json:"missing,omitempty"` +} + +// Report is the whole answer, shaped for JSON as much as for a terminal. +type Report struct { + Applied bool `json:"applied"` + InstanceReadKeys int `json:"instance_read_keys"` + MFAReadKeys int `json:"mfa_read_keys"` + MFAConfigured bool `json:"mfa_configured"` + Columns []Count `json:"columns"` + Windows []Window `json:"grace_windows"` + CheckedAt time.Time `json:"checked_at"` +} + +// OnReadKey is how many values still open only under a read key. +func (r Report) OnReadKey() int { + n := 0 + for _, c := range r.Columns { + n += c.ReadKey + } + return n +} + +// Unreadable is how many values open under no configured key. +func (r Report) Unreadable() int { + n := 0 + for _, c := range r.Columns { + n += c.Unreadable + } + return n +} + +// Window is one previous-credential acceptance window and whether it is still open. +type Window struct { + Name string `json:"name"` + Until time.Time `json:"until"` + Open bool `json:"open"` + Entries int `json:"entries"` +} + +// Keyring is the configured keys the backend opens and seals with. +type Keyring struct { + instance *encrypt.Encrypter + instanceRead int + mfa *[32]byte + mfaRead [][32]byte +} + +// ErrUnreadable means some stored values open under no configured key. +var ErrUnreadable = errors.New("some stored values do not open under any configured key") + +// ErrOnReadKey means a read key is still needed; run keys reseal --apply before retiring it. +var ErrOnReadKey = errors.New("some stored values still open only under a read key") + +// DB is the subset of a pgx pool a scan needs. +type DB interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + BeginTx(ctx context.Context, opts pgx.TxOptions) (pgx.Tx, error) +} + +// Options controls one scan. +type Options struct { + Batch int + Write bool +} + +// LoadKeyring resolves the keys exactly as the backend does at boot. +func LoadKeyring(ctx context.Context) (*Keyring, error) { + instance, err := encrypt.FromEnv() + if err != nil { + return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY or CREDENTIALS_ENCRYPTION_READ_KEYS is invalid: each must be 64 hex characters") + } + if instance == nil { + return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY is not set. Run this inside the backend container, where the keys are configured.") + } + reads, _ := secretset.Parse(os.Getenv("CREDENTIALS_ENCRYPTION_READ_KEYS")) + k := &Keyring{instance: instance} + for _, r := range reads { + if r != os.Getenv("CREDENTIALS_ENCRYPTION_KEY") { + k.instanceRead++ + } + } + + var primary string + var mfaReads []string + if cfg, err := config.NewConfig(ctx); err == nil { + if auth, aerr := cfg.LoadAuthConfig(ctx); aerr == nil { + primary, mfaReads = auth.TwoFASecret, auth.TwoFAReadSecrets + } + } + if primary == "" { + primary = strings.TrimSpace(os.Getenv("TWOFA_SECRET")) + if primary == "" { + primary = strings.TrimSpace(os.Getenv("AUTH_SECRET")) + } + if mfaReads, err = secretset.Parse(os.Getenv("TWOFA_READ_SECRETS")); err != nil { + return nil, fmt.Errorf("TWOFA_READ_SECRETS: %w", err) + } + } + if primary != "" { + key := twofa.DeriveKey(primary) + k.mfa = &key + for _, s := range mfaReads { + if s != primary { + k.mfaRead = append(k.mfaRead, twofa.DeriveKey(s)) + } + } + } + return k, nil +} + +// Scan classifies every sealed value and, with Write, reseals the read-key ones. +func Scan(ctx context.Context, db DB, keys *Keyring, opts Options) (Report, error) { + size := opts.Batch + if size < 1 { + size = 500 + } + now := time.Now() + report := Report{Applied: opts.Write, InstanceReadKeys: keys.instanceRead, MFAReadKeys: len(keys.mfaRead), MFAConfigured: keys.mfa != nil, Windows: Windows(now), CheckedAt: now.UTC()} + for _, col := range Columns() { + if col.Domain == DomainMFA && keys.mfa == nil { + continue + } + count, err := scanColumn(ctx, db, keys, col, size, opts.Write) + if err != nil { + return report, err + } + report.Columns = append(report.Columns, count) + } + return report, nil +} + +// scanColumn classifies every value in one column and, with write, reseals the read-key ones by compare-and-swap. +func scanColumn(ctx context.Context, db DB, keys *Keyring, col SealedColumn, size int, write bool) (Count, error) { + out := Count{SealedColumn: col} + var exists bool + if err := db.QueryRow(ctx, `SELECT to_regclass($1) IS NOT NULL`, col.Table).Scan(&exists); err != nil { + return out, fmt.Errorf("reading %s: %w", col.Table, err) + } + if !exists { + out.Missing = true + return out, nil + } + table, column := pgx.Identifier{col.Table}.Sanitize(), pgx.Identifier{col.Column}.Sanitize() + filter := "" + if col.guard != "" { + filter = " AND " + pgx.Identifier{col.guard}.Sanitize() + } + query := fmt.Sprintf(`SELECT ctid::text AS row_tid, %[2]s::text FROM %[1]s WHERE %[2]s IS NOT NULL AND %[2]s::text <> ''%[3]s AND ctid > $1::tid ORDER BY ctid LIMIT $2`, table, column, filter) + // The ctid and the old value together are the compare: a row rewritten meanwhile matches neither. + update := fmt.Sprintf(`UPDATE %[1]s SET %[2]s = $1 WHERE ctid = $2::tid AND %[2]s::text = $3`, table, column) + + // One snapshot for the reads, so a tuple this run rewrites is never read back. + snap, err := db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) + } + defer func() { _ = snap.Rollback(ctx) }() + + after := "(0,0)" + for { + rows, err := snap.Query(ctx, query, after, size) + if err != nil { + return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) + } + type row struct{ ctid, value string } + var page []row + for rows.Next() { + var r row + if err := rows.Scan(&r.ctid, &r.value); err != nil { + rows.Close() + return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) + } + page = append(page, r) + } + rows.Close() + if err := rows.Err(); err != nil { + return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err) + } + for _, r := range page { + resealed, changed, err := resealValue(keys, col, r.value) + switch { + case errors.Is(err, errUnsealed): + out.Unsealed++ + continue + case errors.Is(err, errWorkspaceSealed): + out.Workspace++ + continue + case err != nil: + out.Unreadable++ + continue + case !changed: + out.Primary++ + continue + } + out.ReadKey++ + if !write { + continue + } + tag, err := db.Exec(ctx, update, resealed, r.ctid, r.value) + if err != nil { + return out, fmt.Errorf("resealing %s.%s: %w", col.Table, col.Column, err) + } + if tag.RowsAffected() == 1 { + out.Resealed++ + } else { + out.Raced++ + } + } + if len(page) < size { + return out, nil + } + after = page[len(page)-1].ctid + } +} + +var ( + errUnsealed = errors.New("value is stored outside the sealed format") + errWorkspaceSealed = errors.New("value is sealed under its workspace key") +) + +func resealValue(keys *Keyring, col SealedColumn, value string) (string, bool, error) { + if col.Domain == DomainMFA { + return twofa.Reseal(*keys.mfa, keys.mfaRead, value) + } + // A workspace-sealed value needs no instance key, so a rotation leaves it alone. + if mailcred.IsOrgSealed(value) { + return "", false, errWorkspaceSealed + } + if !encrypt.LooksSealed(value) { + return "", false, errUnsealed + } + return keys.instance.Reseal(value) +} + +// Windows reports every previous-credential window the instance is configured with. +func Windows(now time.Time) []Window { + var out []Window + for _, w := range []struct{ name, list string }{ + {"INTERNAL_API_PREVIOUS_UNTIL", "INTERNAL_API_PREVIOUS_TOKENS"}, + {"NODE_BROKER_PREVIOUS_UNTIL", "NODE_BROKER_PREVIOUS_TOKENS"}, + {"UNSUBSCRIBE_SIGNING_READ_UNTIL", "UNSUBSCRIBE_SIGNING_READ_SECRETS"}, + } { + raw := strings.TrimSpace(os.Getenv(w.name)) + entries, _ := secretset.Parse(os.Getenv(w.list)) + if raw == "" && len(entries) == 0 { + continue + } + until, _ := time.Parse(time.RFC3339, raw) + out = append(out, Window{Name: w.name, Until: until, Open: now.Before(until), Entries: len(entries)}) + } + return out +} diff --git a/internal/app/keystatus/keystatus_test.go b/internal/app/keystatus/keystatus_test.go new file mode 100644 index 000000000..1174cb8f5 --- /dev/null +++ b/internal/app/keystatus/keystatus_test.go @@ -0,0 +1,101 @@ +package keystatus + +import ( + "encoding/hex" + "errors" + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/app/mailcred" + "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/app/twofa" + "github.com/warmbly/warmbly/internal/pkg/encrypt" +) + +func TestSealedColumnsCoverEveryInstanceKeySecret(t *testing.T) { + got := map[string]Domain{} + for _, c := range Columns() { + got[c.Table+"."+c.Column] = c.Domain + } + for _, want := range []string{ + "email_accounts_smtp_imap.smtp_password", + "email_accounts_smtp_imap.imap_password", + "email_accounts_oauth.access_token", + "email_accounts_oauth.refresh_token", + "webhook_endpoints.secret", + "oauth_applications.webhook_secret", + "cloud_link.token", + "mailbox_erasures.refresh_token", + } { + if got[want] != DomainInstance { + t.Fatalf("%s must be scanned under the instance credential key", want) + } + } + if got["user_totp_settings.totp_secret_sealed"] != DomainMFA { + t.Fatal("TOTP enrollments must be scanned under the MFA key") + } + + for _, c := range instanceOnlyColumns { + if _, ok := orgtransfer.ExcludedTables[c.Table]; c.Domain == DomainInstance && !ok { + t.Fatalf("%s is listed as instance-only but is not in orgtransfer.ExcludedTables", c.Table) + } + } +} + +func TestResealValueClassifiesEveryShape(t *testing.T) { + oldKey, newKey := strings.Repeat("11", 32), strings.Repeat("22", 32) + t.Setenv("CREDENTIALS_ENCRYPTION_KEY", oldKey) + t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", "") + old, err := encrypt.FromEnv() + if err != nil { + t.Fatal(err) + } + legacy, err := old.Encrypt("smtp-password") + if err != nil { + t.Fatal(err) + } + t.Setenv("CREDENTIALS_ENCRYPTION_KEY", newKey) + t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", `["`+oldKey+`"]`) + current, err := encrypt.FromEnv() + if err != nil { + t.Fatal(err) + } + keys := &Keyring{instance: current, instanceRead: 1} + col := SealedColumn{Table: "email_accounts_smtp_imap", Column: "smtp_password", Domain: DomainInstance} + + resealed, changed, err := resealValue(keys, col, legacy) + if err != nil || !changed { + t.Fatalf("a read-key value must be resealed, got changed=%v err=%v", changed, err) + } + if _, changed, err := resealValue(keys, col, resealed); err != nil || changed { + t.Fatal("a resealed value must open under the primary") + } + if _, _, err := resealValue(keys, col, mailcred.OrgPrefix+"c2VhbGVk"); !errors.Is(err, errWorkspaceSealed) { + t.Fatal("a workspace-sealed value must be counted apart and never touched by an instance rotation") + } + if _, _, err := resealValue(keys, col, "ya29.a0Af-legacy-token"); !errors.Is(err, errUnsealed) { + t.Fatal("a value outside the sealed format must be reported unsealed, never rewritten") + } + foreign := hex.EncodeToString(make([]byte, 64)) + if _, _, err := resealValue(keys, col, foreign); err == nil || errors.Is(err, errUnsealed) { + t.Fatal("a sealed value no key opens must be reported unreadable") + } + + mfaOld, mfaNew := twofa.DeriveKey("old-mfa-secret"), twofa.DeriveKey("new-mfa-secret") + sealed, err := twofa.Seal(mfaOld, "JBSWY3DPEHPK3PXP") + if err != nil { + t.Fatal(err) + } + keys.mfa, keys.mfaRead = &mfaNew, [][32]byte{mfaOld} + mfa := SealedColumn{Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: DomainMFA} + if _, changed, err := resealValue(keys, mfa, sealed); err != nil || !changed { + t.Fatal("an enrollment sealed under the MFA read key must be resealed") + } +} + +func TestReportOnReadKeyCountsValuesStillNeedingAReadKey(t *testing.T) { + r := Report{Columns: []Count{{ReadKey: 2, Primary: 5}, {ReadKey: 1, Unreadable: 3}}} + if r.OnReadKey() != 3 || r.Unreadable() != 3 { + t.Fatalf("OnReadKey=%d Unreadable=%d", r.OnReadKey(), r.Unreadable()) + } +} diff --git a/internal/app/releases/service_test.go b/internal/app/releases/service_test.go index bf9d158c0..3f1b51109 100644 --- a/internal/app/releases/service_test.go +++ b/internal/app/releases/service_test.go @@ -23,6 +23,11 @@ func (m *memSettings) SetRelease(_ context.Context, s *models.FleetReleaseState) return nil } +func (m *memSettings) CompareAndSetRelease(_ context.Context, _, next *models.FleetReleaseState) (bool, error) { + m.release = next + return true, nil +} + func (m *memSettings) GetJoinToken(context.Context) (string, *time.Time, error) { return "", nil, nil } func (m *memSettings) SetJoinToken(context.Context, string, time.Time) error { return nil } diff --git a/internal/infrastructure/db/migrations/000323_fleet_node_update_state.down.sql b/internal/infrastructure/db/migrations/000323_fleet_node_update_state.down.sql new file mode 100644 index 000000000..5f884a891 --- /dev/null +++ b/internal/infrastructure/db/migrations/000323_fleet_node_update_state.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE fleet_nodes + DROP COLUMN IF EXISTS env_revision, + DROP COLUMN IF EXISTS update_failed; diff --git a/internal/infrastructure/db/migrations/000323_fleet_node_update_state.up.sql b/internal/infrastructure/db/migrations/000323_fleet_node_update_state.up.sql new file mode 100644 index 000000000..6a46bdd19 --- /dev/null +++ b/internal/infrastructure/db/migrations/000323_fleet_node_update_state.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE fleet_nodes + ADD COLUMN IF NOT EXISTS update_failed text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS env_revision text NOT NULL DEFAULT ''; diff --git a/internal/models/fleet_node.go b/internal/models/fleet_node.go index 89b127639..ff72f120b 100644 --- a/internal/models/fleet_node.go +++ b/internal/models/fleet_node.go @@ -76,6 +76,10 @@ type FleetNode struct { MailboxCount *int `json:"mailbox_count,omitempty"` LastError string `json:"last_error,omitempty"` + // UpdateFailed is the tag the host updater last rolled back from. + UpdateFailed string `json:"update_failed,omitempty"` + // EnvRevision is the rendered env revision the node last applied. + EnvRevision string `json:"env_revision,omitempty"` Tags []string `json:"tags,omitempty"` @@ -142,6 +146,12 @@ type NodeHeartbeat struct { // Stopping is set on the farewell beat, so the row goes inactive at once // rather than staying selectable until the beat ages out. Stopping bool `json:"stopping,omitempty"` + // UpdateFailed is the tag the host updater rolled back from, "" when none. + UpdateFailed string `json:"update_failed,omitempty"` + // EnvRefresh says the host updater applies env revisions from the reply. + EnvRefresh bool `json:"env_refresh,omitempty"` + // EnvRevision is the env revision the host updater last applied. + EnvRevision string `json:"env_revision,omitempty"` } // NodeHeartbeatReply is the control plane's answer. It is the only channel by @@ -154,4 +164,10 @@ type NodeHeartbeatReply struct { // LivenessSeconds tells the node how long its beat is trusted for, so the // beat interval and the server's window can never drift apart. LivenessSeconds int `json:"liveness_seconds"` + // DesiredDigest pins DesiredVersion to an image digest ("sha256:..."), + // set only while a staged rollout moves this node. + DesiredDigest string `json:"desired_digest,omitempty"` + // EnvRevision is the revision of the env file this node should have, set + // only for a node that reported EnvRefresh. + EnvRevision string `json:"env_revision,omitempty"` } diff --git a/internal/models/fleet_release.go b/internal/models/fleet_release.go index 2d56b6eeb..c81f201cd 100644 --- a/internal/models/fleet_release.go +++ b/internal/models/fleet_release.go @@ -1,6 +1,10 @@ package models -import "time" +import ( + "time" + + "github.com/google/uuid" +) // Admin settings keys for the fleet. Both live in `admin_settings` so every // backend replica agrees and the values survive a restart. @@ -40,6 +44,8 @@ type FleetReleaseState struct { // Source records how Tag was set, so an operator can tell an automatic // resolution from a manual pin. Source string `json:"source,omitempty"` + // Rollout is a staged move to a newer tag, nil when none is in progress. + Rollout *FleetRollout `json:"rollout,omitempty"` } // DesiredVersion is the tag nodes should converge on, or "" when the control @@ -50,3 +56,96 @@ func (s *FleetReleaseState) DesiredVersion() string { } return s.Tag } + +// Rollout stages. A rollout admits nodes stage by stage and only moves on +// once every admitted node has run the target healthily for the bake time. +const ( + FleetRolloutCanary = "canary" + FleetRolloutWave = "wave" + FleetRolloutHalted = "halted" +) + +// FleetRolloutPolicy shapes a staged rollout. +type FleetRolloutPolicy struct { + // Canary is how many nodes go first, alone. + Canary int `json:"canary"` + // Waves are cumulative percentages of the fleet admitted after the canary. + Waves []int `json:"waves"` + // BakeSeconds is how long every admitted node must run the target before + // the next stage is admitted. + BakeSeconds int `json:"bake_seconds"` + // DeadlineSeconds is how long an admitted node has to report the target + // before the rollout halts. + DeadlineSeconds int `json:"deadline_seconds"` +} + +// DefaultFleetRolloutPolicy is one canary, then everyone, with a 30 minute bake. +func DefaultFleetRolloutPolicy() FleetRolloutPolicy { + return FleetRolloutPolicy{Canary: 1, Waves: []int{100}, BakeSeconds: 1800, DeadlineSeconds: 900} +} + +// Normalize fills unset fields with the defaults and bounds the rest. +func (p FleetRolloutPolicy) Normalize() FleetRolloutPolicy { + def := DefaultFleetRolloutPolicy() + if p.Canary <= 0 { + p.Canary = def.Canary + } + waves := make([]int, 0, len(p.Waves)+1) + last := 0 + for _, w := range p.Waves { + if w > last && w <= 100 { + waves = append(waves, w) + last = w + } + } + if last < 100 { + waves = append(waves, 100) + } + p.Waves = waves + if p.BakeSeconds <= 0 { + p.BakeSeconds = def.BakeSeconds + } + if p.DeadlineSeconds <= 0 { + p.DeadlineSeconds = def.DeadlineSeconds + } + return p +} + +// FleetRollout is a staged move of the fleet from From to Target. +// +// While it runs, FleetReleaseState.Tag stays at From, so a backend that does +// not know about rollouts keeps every node where it is. Only admitted nodes +// are told Target. +type FleetRollout struct { + Target string `json:"target"` + From string `json:"from"` + // Digests maps an image key ("worker", "worker-kafka", "consumer", ...) to + // the sha256 digest of Target, so an admitted node pulls exactly what was + // released rather than whatever the tag points at. + Digests map[string]string `json:"digests,omitempty"` + Policy FleetRolloutPolicy `json:"policy"` + Stage string `json:"stage"` + Wave int `json:"wave"` + Admitted []uuid.UUID `json:"admitted,omitempty"` + StartedAt time.Time `json:"started_at"` + StageStartedAt time.Time `json:"stage_started_at"` + HaltReason string `json:"halt_reason,omitempty"` +} + +// Running reports whether admitted nodes are being moved to Target. +func (r *FleetRollout) Running() bool { + return r != nil && (r.Stage == FleetRolloutCanary || r.Stage == FleetRolloutWave) +} + +// IsAdmitted reports whether id has been admitted to the rollout. +func (r *FleetRollout) IsAdmitted(id uuid.UUID) bool { + if r == nil { + return false + } + for _, a := range r.Admitted { + if a == id { + return true + } + } + return false +} diff --git a/internal/pkg/nodeagent/agent.go b/internal/pkg/nodeagent/agent.go index 78772316e..214482ef4 100644 --- a/internal/pkg/nodeagent/agent.go +++ b/internal/pkg/nodeagent/agent.go @@ -170,6 +170,9 @@ func (a *Agent) beat(ctx context.Context, booted, stopping bool) *models.NodeHea Booted: booted, Stopping: stopping, } + if !stopping { + a.readUpdaterState(&beat) + } if !stopping { beat.Address = a.publicAddress(ctx) } @@ -208,9 +211,83 @@ func (a *Agent) beat(ctx context.Context, booted, stopping bool) *models.NodeHea return nil } a.applyTarget(reply.DesiredVersion) + a.applyDigest(reply.DesiredVersion, reply.DesiredDigest) + a.applyEnvRevision(reply.EnvRevision) return reply } +// The host updater shares the target directory: it writes update-failed and +// env-revision there, and reads target-digest and target-env-revision. +func (a *Agent) stateFile(name string) string { + if a.cfg.TargetVersionPath == "" { + return "" + } + return filepath.Join(filepath.Dir(a.cfg.TargetVersionPath), name) +} + +func (a *Agent) readUpdaterState(beat *models.NodeHeartbeat) { + if a.cfg.TargetVersionPath == "" { + return + } + beat.UpdateFailed = readToken(a.stateFile("update-failed")) + beat.EnvRefresh = os.Getenv("WARMBLY_ENV_REFRESH") == "true" + if beat.EnvRefresh { + beat.EnvRevision = readToken(a.stateFile("env-revision")) + } +} + +// readToken returns the first line of a small state file, or "" when it is +// absent or holds anything but a plain token. +func readToken(path string) string { + raw, err := os.ReadFile(path) + if err != nil || len(raw) > 256 { + return "" + } + v := strings.TrimSpace(strings.SplitN(string(raw), "\n", 2)[0]) + for _, c := range v { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune(".-_+:", c)) { + return "" + } + } + return v +} + +// applyDigest records the digest a staged rollout pins the target to. It is +// written as " " so the updater uses it only for that tag. +func (a *Agent) applyDigest(version, digest string) { + path := a.stateFile("target-digest") + if path == "" { + return + } + if version == "" || digest == "" { + _ = os.Remove(path) + return + } + writeAtomic(path, version+" "+digest+"\n") +} + +func (a *Agent) applyEnvRevision(rev string) { + path := a.stateFile("target-env-revision") + if path == "" || rev == "" || os.Getenv("WARMBLY_ENV_REFRESH") != "true" { + return + } + if readToken(path) == rev { + return + } + writeAtomic(path, rev+"\n") +} + +func writeAtomic(path, body string) { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(body), 0o644); err != nil { + log.Println("nodeagent: cannot write updater state:", err) + return + } + if err := os.Rename(tmp, path); err != nil { + log.Println("nodeagent: cannot publish updater state:", err) + } +} + // applyTarget records the version the control plane wants. Writing a file the // host-side updater polls keeps the decision (control plane) separate from the // mechanism (systemd timer), which is what lets a node replace itself without diff --git a/internal/repository/pg_fleet_node.go b/internal/repository/pg_fleet_node.go index 75b5c3654..2c111d1f4 100644 --- a/internal/repository/pg_fleet_node.go +++ b/internal/repository/pg_fleet_node.go @@ -56,7 +56,7 @@ const fleetNodeSelect = ` n.active, n.last_seen_at, n.enrolled_at, n.cpu_percent, n.memory_mb, n.goroutines, n.uptime_seconds, n.cpu_scope, n.memory_scope, n.memory_used_mb, n.memory_limit_mb, n.resident_mb, - n.last_error, n.created_at, n.updated_at, + n.last_error, n.update_failed, n.env_revision, n.created_at, n.updated_at, w.account_count FROM fleet_nodes n LEFT JOIN workers w ON w.id = n.id` @@ -69,7 +69,7 @@ func scanFleetNode(row pgx.Row) (*models.FleetNode, error) { &n.Active, &n.LastSeenAt, &n.EnrolledAt, &n.Usage.CPUPercent, &n.Usage.MemoryMB, &n.Usage.Goroutines, &n.Usage.UptimeSeconds, &n.Usage.CPUScope, &n.Usage.MemoryScope, &n.Usage.MemoryUsedMB, &n.Usage.MemoryLimitMB, &n.Usage.ResidentMB, - &n.LastError, &n.CreatedAt, &n.UpdatedAt, + &n.LastError, &n.UpdateFailed, &n.EnvRevision, &n.CreatedAt, &n.UpdatedAt, &n.MailboxCount, ) if err != nil { @@ -88,8 +88,9 @@ func (r *fleetNodeRepository) UpsertOnHeartbeat(ctx context.Context, beat models INSERT INTO fleet_nodes ( id, role, name, region, address, capacity_target, version, active, last_seen_at, cpu_percent, memory_mb, goroutines, uptime_seconds, last_error, - cpu_scope, memory_scope, memory_used_mb, memory_limit_mb, resident_mb, warmup_send_protocol - ) VALUES ($1, $2, $3, $4, $5, COALESCE(NULLIF($6, 0), 100), $7, TRUE, now(), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + cpu_scope, memory_scope, memory_used_mb, memory_limit_mb, resident_mb, warmup_send_protocol, + update_failed, env_revision + ) VALUES ($1, $2, $3, $4, $5, COALESCE(NULLIF($6, 0), 100), $7, TRUE, now(), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (id) DO UPDATE SET -- Role is NOT updated. A node that re-registers under a different -- role would keep its workers row and the mailboxes assigned to @@ -114,6 +115,9 @@ func (r *fleetNodeRepository) UpsertOnHeartbeat(ctx context.Context, beat models resident_mb = EXCLUDED.resident_mb, warmup_send_protocol = EXCLUDED.warmup_send_protocol, last_error = EXCLUDED.last_error, + update_failed = EXCLUDED.update_failed, + -- An updater that does not report one keeps the last revision it did. + env_revision = CASE WHEN EXCLUDED.env_revision <> '' THEN EXCLUDED.env_revision ELSE fleet_nodes.env_revision END, updated_at = now() ` _, err := r.db.Exec(ctx, q, @@ -122,6 +126,7 @@ func (r *fleetNodeRepository) UpsertOnHeartbeat(ctx context.Context, beat models beat.LastError, beat.Usage.CPUScope, beat.Usage.MemoryScope, beat.Usage.MemoryUsedMB, beat.Usage.MemoryLimitMB, beat.Usage.ResidentMB, beat.WarmupSendProtocol, + beat.UpdateFailed, beat.EnvRevision, ) return err } diff --git a/internal/repository/pg_fleet_settings.go b/internal/repository/pg_fleet_settings.go index 18d995482..a82f57b87 100644 --- a/internal/repository/pg_fleet_settings.go +++ b/internal/repository/pg_fleet_settings.go @@ -18,6 +18,10 @@ import ( type FleetSettingsRepository interface { GetRelease(ctx context.Context) (*models.FleetReleaseState, error) SetRelease(ctx context.Context, state *models.FleetReleaseState) error + // CompareAndSetRelease writes next only while the stored state still + // equals prev (nil meaning none stored), so concurrent backends never + // overwrite each other's rollout decisions. It reports whether it wrote. + CompareAndSetRelease(ctx context.Context, prev, next *models.FleetReleaseState) (bool, error) // GetJoinToken returns "" when no token has been issued yet, which callers // must treat as "nothing may join" rather than "anything may". A nil @@ -72,6 +76,28 @@ func (r *fleetSettingsRepository) SetRelease(ctx context.Context, state *models. return r.setRaw(ctx, models.FleetSettingsKeyRelease, state) } +func (r *fleetSettingsRepository) CompareAndSetRelease(ctx context.Context, prev, next *models.FleetReleaseState) (bool, error) { + raw, err := json.Marshal(next) + if err != nil { + return false, err + } + if prev == nil { + tag, err := r.db.Exec(ctx, ` + INSERT INTO admin_settings (key, value, updated_at) VALUES ($1, $2, now()) + ON CONFLICT (key) DO NOTHING`, models.FleetSettingsKeyRelease, raw) + return err == nil && tag.RowsAffected() == 1, err + } + old, err := json.Marshal(prev) + if err != nil { + return false, err + } + // jsonb equality is semantic, so key order in the stored value is irrelevant. + tag, err := r.db.Exec(ctx, ` + UPDATE admin_settings SET value = $2, updated_at = now() + WHERE key = $1 AND value = $3::jsonb`, models.FleetSettingsKeyRelease, raw, old) + return err == nil && tag.RowsAffected() == 1, err +} + type joinTokenValue struct { Hash string `json:"hash"` ExpiresAt *time.Time `json:"expires_at,omitempty"` diff --git a/scripts/check-join-script.sh b/scripts/check-join-script.sh index a84d4bd43..f1c0b9bd5 100755 --- a/scripts/check-join-script.sh +++ b/scripts/check-join-script.sh @@ -20,6 +20,11 @@ # created without ever truncating one that is already there # - --dry-run masks every credential, including the ones that hide inside a # URL, and does it with an expression this sed actually supports +# - the rendered updater, run against fake docker, systemctl and curl: digest +# pulls, local rollback on a crash loop, no retry of a failed tag or env +# revision, node files never followed through a symlink, the node token +# kept off argv, an env refresh keeping local keys, and --adopt refusing a +# node the control plane would move set -eu SCRIPT="internal/api/handler/nodescript/join.sh" @@ -169,8 +174,13 @@ ok "--dry-run masks every credential and keeps the addresses" # body_of prints one function's body. Tolerant about the definition's spacing, # and loud when the function is not found: an empty body would otherwise fail # every assertion below with a message about the wrong thing. +# Heredoc bodies are skipped: the rendered updater defines its own main(). body_of() { out=$(awk -v fn="$1" ' + !inside && term != "" { if ($0 == term) term = ""; next } + !inside && match($0, /<<-?[ \t]*'"'"'?[A-Za-z_]+'"'"'?/) { + term = substr($0, RSTART, RLENGTH); gsub(/^<<-?[ \t]*|'"'"'/, "", term) + } $0 ~ "^" fn "[ \t]*\\([ \t]*\\)[ \t]*{" { inside = 1 } inside { print } inside && $0 == "}" { exit }' "$SCRIPT") @@ -307,4 +317,212 @@ run_enrol success true || fail "dry-run fixture failed" [ ! -e "$join_fixture/config" ] || fail "dry-run persisted an identity or lock" ok "dry-run leaves no pending identity or lock" +# The updater runs as root every two minutes and reads files the node wrote, so +# its static shape is asserted first and its behaviour below, against fakes. +for fn in install_units adopt; do + fn_body=$(body_of "$fn") + printf '%s\n' "$fn_body" | awk '$1 == "install_updater" { found = 1 } END { exit !found }' \ + || fail "$fn must call install_updater as a standalone statement, or the node never updates" +done +ok "join and adopt both install the updater" + +write_body=$(body_of write_config) +printf '%s\n' "$write_body" | grep -q "WARMBLY_ENV_REFRESH=true" \ + || fail "write_config must opt a new join into env refresh" +ok "a new join opts into env refresh" + +updater="$join_fixture/warmbly-node-update" +sh "$SCRIPT" --print-updater > "$updater" || fail "--print-updater failed" +parse_check "$updater" +if command -v shellcheck >/dev/null 2>&1; then + shellcheck -s sh "$updater" || fail "shellcheck failed on the rendered updater" +fi +[ "$(tail -n 1 "$updater")" = 'main "$@"' ] || fail "the rendered updater must end with main \"\$@\"" +ok "rendered updater parses, passes shellcheck and ends with main" + +mkdir -p "$join_fixture/bin" +cat > "$join_fixture/bin/docker" <<'FAKE' +#!/bin/sh +case "$1" in + pull) printf '%s\n' "$2" >> "$FIX/pulls"; [ "${FIX_PULL:-ok}" = ok ] ;; + inspect) + n=$(cat "$FIX/inspects" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$FIX/inspects" + if [ -s "$FIX/crash-if" ] && cat "$WARMBLY_UPDATER_ROOT/var/lib/warmbly/image-ref" \ + "$WARMBLY_UPDATER_ROOT/etc/warmbly/node.env" | grep -q -f "$FIX/crash-if"; then + echo "true start-$n" + else + echo "true start-0" + fi ;; +esac +FAKE +cat > "$join_fixture/bin/systemctl" <<'FAKE' +#!/bin/sh +printf '%s\n' "$*" >> "$FIX/systemctl" +FAKE +cat > "$join_fixture/bin/curl" <<'FAKE' +#!/bin/sh +printf '%s\n' "$*" >> "$FIX/curl-argv" +out="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) out="$2"; shift 2 ;; + -K) [ "$2" = - ] && cat >> "$FIX/curl-stdin"; shift 2 ;; + *) shift ;; + esac +done +cat "$FIX/env-reply" > "$out" +printf '%s' "${FIX_CODE:-200}" +FAKE +chmod +x "$join_fixture/bin/docker" "$join_fixture/bin/systemctl" "$join_fixture/bin/curl" + +U="$join_fixture/upd" +NODE_ID="44444444-4444-4444-8444-444444444444" +DIGEST="sha256:$(printf '%064d' 7)" +upd_reset() { + rm -rf "$U" + mkdir -p "$U/root/etc/warmbly" "$U/root/var/lib/warmbly/node" "$U/root/run" "$U/fix" + printf 'ghcr.io/warmbly/warmbly/worker\n' > "$U/root/var/lib/warmbly/image" + printf 'WARMBLY_IMAGE_REF=ghcr.io/warmbly/warmbly/worker:v1\n' > "$U/root/var/lib/warmbly/image-ref" + printf 'WARMBLY_NODE_ID=%s\nWARMBLY_NODE_ROLE=worker\nWARMBLY_BACKEND_URL=https://fixture.invalid\nNODE_BROKER_TOKEN=fixture-broker-old\nWARMBLY_VERSION=v1\nNODE_LOG_TOKEN=fixture-log\n' \ + "$NODE_ID" > "$U/root/etc/warmbly/node.env" + : > "$U/fix/pulls"; : > "$U/fix/systemctl"; : > "$U/fix/curl-argv"; : > "$U/fix/curl-stdin" +} +run_updater() { + PATH="$join_fixture/bin:$PATH" FIX="$U/fix" WARMBLY_UPDATER_ROOT="$U/root" \ + WARMBLY_UPDATE_SETTLE_SECONDS=0 WARMBLY_UPDATE_HEALTH_SECONDS=1 WARMBLY_UPDATE_POLL_SECONDS=1 \ + sh "$updater" "$@" > "$U/fix/out" 2>&1 +} +agent() { printf '%s\n' "$2" > "$U/root/var/lib/warmbly/node/$1"; } +count() { wc -l < "$1" | tr -d ' '; } + +upd_reset +agent target-version v2 +agent target-digest "v2 $DIGEST" +run_updater || fail "a healthy update failed" +grep -qx "ghcr.io/warmbly/warmbly/worker@$DIGEST" "$U/fix/pulls" || fail "a pinned digest must be pulled by digest" +grep -qx "WARMBLY_IMAGE_REF=ghcr.io/warmbly/warmbly/worker@$DIGEST" "$U/root/var/lib/warmbly/image-ref" \ + || fail "the unit must run the digest that was pulled" +grep -qx 'WARMBLY_VERSION=v2' "$U/root/etc/warmbly/node.env" || fail "a healthy update must record the new version" +ok "updater pulls the pinned digest and keeps a healthy update" + +upd_reset +agent target-version v2 +agent target-digest "v1 $DIGEST" +run_updater || fail "update by tag failed" +grep -qx 'ghcr.io/warmbly/warmbly/worker:v2' "$U/fix/pulls" || fail "a digest pinned for another tag must not be used" +ok "a digest for another tag falls back to the tag" + +upd_reset +agent target-version v2 +printf 'keep\n' > "$U/fix/victim" +ln -s "$U/fix/victim" "$U/root/var/lib/warmbly/node/update-failed" +printf 'worker:v2\n' > "$U/fix/crash-if" +run_updater || fail "a rolled-back update must still exit 0" +grep -qx 'WARMBLY_VERSION=v1' "$U/root/etc/warmbly/node.env" || fail "an unhealthy update must restore the old version" +grep -qx 'WARMBLY_IMAGE_REF=ghcr.io/warmbly/warmbly/worker:v1' "$U/root/var/lib/warmbly/image-ref" \ + || fail "an unhealthy update must restore the old image" +[ "$(count "$U/fix/systemctl")" = "2" ] || fail "a rollback must restart onto the old image" +[ ! -L "$U/root/var/lib/warmbly/node/update-failed" ] || fail "write_agent followed a symlink the node planted" +[ "$(cat "$U/fix/victim")" = "keep" ] || fail "write_agent wrote through a symlink the node planted" +grep -qx v2 "$U/root/var/lib/warmbly/node/update-failed" || fail "a rollback must report the failed tag" +pulls=$(count "$U/fix/pulls") +run_updater || fail "second pass after a rollback failed" +[ "$(count "$U/fix/pulls")" = "$pulls" ] || fail "a tag that failed its health check was retried" +ok "an unhealthy update rolls back, reports the tag and is not retried" + +upd_reset +printf 'v3\n' > "$U/fix/outside" +ln -s "$U/fix/outside" "$U/root/var/lib/warmbly/node/target-version" +run_updater || fail "updater failed on a symlinked target" +[ "$(count "$U/fix/pulls")" = "0" ] || fail "the updater followed a symlinked target-version" +agent target-version 'v3;reboot' +run_updater || fail "updater failed on a malformed target" +[ "$(count "$U/fix/pulls")" = "0" ] || fail "the updater acted on a malformed target" +ok "symlinked or malformed node files are ignored" + +reply() { + printf 'WARMBLY_NODE_ID=%s\nWARMBLY_NODE_ROLE=worker\nWARMBLY_BACKEND_URL=https://fixture.invalid\nNODE_BROKER_TOKEN=fixture-broker-new\n%s' \ + "${1:-$NODE_ID}" "${2:-}" > "$U/fix/env-reply" + agent target-env-revision "$(sha256sum "$U/fix/env-reply" | cut -c1-32)" +} + +upd_reset +reply +run_updater || fail "updater failed without env refresh" +[ "$(count "$U/fix/curl-argv")" = "0" ] || fail "env was fetched without WARMBLY_ENV_REFRESH=true" +ok "env refresh stays off unless node.env opts in" + +upd_reset +printf 'WARMBLY_ENV_REFRESH=true\n' >> "$U/root/etc/warmbly/node.env" +reply +run_updater || fail "env refresh failed" +envf="$U/root/etc/warmbly/node.env" +grep -qx 'NODE_BROKER_TOKEN=fixture-broker-new' "$envf" || fail "the refreshed env was not applied" +for kept in WARMBLY_VERSION=v1 NODE_LOG_TOKEN=fixture-log WARMBLY_ENV_REFRESH=true; do + grep -qx "$kept" "$envf" || fail "an env refresh dropped $kept" +done +[ "$(stat -c %a "$envf")" = "600" ] || fail "the refreshed node.env must be mode 600" +[ "$(cat "$U/root/var/lib/warmbly/node/env-revision")" = "$(cat "$U/root/var/lib/warmbly/node/target-env-revision")" ] \ + || fail "a refresh must report the revision it applied" +if grep -q 'fixture-broker-old' "$U/fix/curl-argv"; then fail "the node token reached curl's argv"; fi +grep -q 'Authorization: Bearer fixture-broker-old' "$U/fix/curl-stdin" || fail "the node token must reach curl on stdin" +ok "env refresh applies, keeps local keys and never puts the token on argv" + +upd_reset +printf 'WARMBLY_ENV_REFRESH=true\n' >> "$U/root/etc/warmbly/node.env" +cp "$U/root/etc/warmbly/node.env" "$U/fix/before" +reply "" "CRASH=1 +" +printf 'CRASH=1\n' > "$U/fix/crash-if" +run_updater || fail "a rolled-back env refresh must still exit 0" +cmp -s "$U/fix/before" "$U/root/etc/warmbly/node.env" || fail "an unhealthy env refresh must restore the previous env" +grep -qx "$(cat "$U/root/var/lib/warmbly/node/target-env-revision")-failed" "$U/root/var/lib/warmbly/node/env-revision" \ + || fail "an unhealthy env refresh must report the failed revision" +fetches=$(count "$U/fix/curl-argv") +run_updater || fail "second pass after an env rollback failed" +[ "$(count "$U/fix/curl-argv")" = "$fetches" ] || fail "a revision that failed its health check was retried" +ok "an unhealthy env refresh restores the previous env and is not retried" + +upd_reset +printf 'WARMBLY_ENV_REFRESH=true\n' >> "$U/root/etc/warmbly/node.env" +cp "$U/root/etc/warmbly/node.env" "$U/fix/before" +reply "55555555-5555-4555-8555-555555555555" +run_updater || fail "updater failed on another node's env" +cmp -s "$U/fix/before" "$U/root/etc/warmbly/node.env" || fail "an env naming another node was applied" +ok "an env naming another node is refused" + +upd_reset +printf 'WORKER_PUBLIC_IP=203.0.113.9\n' >> "$U/root/etc/warmbly/node.env" +reply +if run_updater --check-env; then fail "--check-env passed with a key a refresh would drop"; fi +grep -q 'WORKER_PUBLIC_IP' "$U/fix/out" || fail "--check-env must name the key a refresh would drop" +if grep -q '203.0.113.9' "$U/fix/out"; then fail "--check-env printed a value"; fi +printf 'WORKER_PUBLIC_IP=203.0.113.9\n' > "$U/root/etc/warmbly/node.local.env" +run_updater --check-env || fail "--check-env must accept a key kept in node.local.env" +[ "$(count "$U/fix/systemctl")" = "0" ] || fail "--check-env restarted the node" +ok "--check-env names dropped keys without values and changes nothing" + +# --adopt must refuse before writing anything when the control plane would move the node. +cat > "$join_fixture/adopt.sh" <<'FIXTURE' +#!/bin/sh +set -eu +. "$JOIN_FUNCTIONS" +CONFIG_DIR="$ADOPT_ROOT/etc/warmbly" +STATE_DIR="$ADOPT_ROOT/var/lib/warmbly" +AGENT_DIR="$STATE_DIR/node" +ADOPT_ACCEPT_TARGET="false" +id() { echo 0; } +adopt +FIXTURE +upd_reset +rm -f "$U/root/var/lib/warmbly/image" +agent target-version v2 +if PATH="$join_fixture/bin:$PATH" JOIN_FUNCTIONS="$join_fixture/functions.sh" ADOPT_ROOT="$U/root" \ + sh "$join_fixture/adopt.sh" > "$U/fix/out" 2>&1; then + fail "--adopt accepted a node the control plane would move" +fi +grep -q 'accept-target' "$U/fix/out" || fail "--adopt must explain how to accept the target" +[ ! -e "$U/root/var/lib/warmbly/image" ] || fail "--adopt wrote state before refusing" +ok "--adopt refuses a version mismatch before writing anything" + printf 'check-join-script: all checks passed\n'