feat: deploy stable releases to AWS automatically with provenance checks, a migration gate, alarm-watched waves and reverse rollback, move worker and consumer hosts in canary rollouts with host-side health rollback and opt-in env refresh, rotate instance keys inside AWS through a rotator task and the rotate-keys workflow, plan and apply the AWS Terraform from CI with approval for replacements, and add the admin Keys tab and the matching runbooks

This commit is contained in:
Matthew Meszaros
2026-10-24 09:10:52 +02:00
parent 9e8077c907
commit 9c2cffae58
58 changed files with 5422 additions and 798 deletions
+356 -69
View File
@@ -1,8 +1,26 @@
// Deploys one published release to the AWS control plane: render every task
// definition from the revision Terraform last registered, swap in the release
// digests, gate on a dependency check and the migration, roll the services in
// waves with a bake, then hand the workers a canary rollout. It never builds,
// never reads a secret value, and restores everything it changed on failure.
const {
ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand,
ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand, RegisterTaskDefinitionCommand,
RunTaskCommand, DescribeTasksCommand, UpdateServiceCommand,
} = require("@aws-sdk/client-ecs");
const { SecretsManagerClient, DescribeSecretCommand, ListSecretVersionIdsCommand } = require("@aws-sdk/client-secrets-manager");
const { CloudWatchClient, DescribeAlarmsCommand } = require("@aws-sdk/client-cloudwatch");
const { CloudWatchLogsClient, GetLogEventsCommand } = require("@aws-sdk/client-cloudwatch-logs");
const { createHash } = require("node:crypto");
const { readFileSync } = require("node:fs");
const SERVICES = ["backend", "consumer", "forms", "tracking", "realtime"];
const WAVES = [["backend"], ["consumer"], ["forms", "tracking", "realtime"]];
const MODES = ["deploy", "rollback", "redeploy", "refresh-secrets"];
const RELEASE_TAG = "warmbly-release";
const VARIANT_TAG = "warmbly-image-variant";
const TAG_RE = /^v(\d+)\.(\d+)\.(\d+)$/;
const DIGEST_RE = /^sha256:[a-f0-9]{64}$/;
const VERSION_RE = /^[A-Za-z0-9-]{32,64}$/;
function configuration(env) {
const cluster = env.AWS_ECS_CLUSTER;
@@ -10,22 +28,38 @@ function configuration(env) {
if (!match) throw new Error("AWS_ECS_CLUSTER must be a us-east-1 cluster ARN");
const prefix = env.AWS_ECS_SERVICE_PREFIX || match[2];
if (!/^[\w-]+$/.test(prefix)) throw new Error("Invalid service prefix");
const definitions = {};
const counts = {};
for (const name of ["migration", "backend", "consumer"]) {
const value = env[`AWS_${name.toUpperCase()}_TASK_DEFINITION`];
const expected = `arn:aws:ecs:us-east-1:${match[1]}:task-definition/${prefix}-${name}:`;
if (!value?.startsWith(expected) || !/^\d+$/.test(value.slice(expected.length))) {
throw new Error(`Provide an explicit ${name} task definition revision`);
}
definitions[name] = value;
if (name !== "migration") {
const raw = env[`AWS_${name.toUpperCase()}_DESIRED_COUNT`] || "preserve";
if (raw !== "preserve" && !/^[0-5]$/.test(raw)) throw new Error("Desired counts must be preserve or 0 through 5");
counts[name] = raw === "preserve" ? undefined : Number(raw);
const mode = env.DEPLOY_MODE || "deploy";
if (!MODES.includes(mode)) throw new Error(`DEPLOY_MODE must be one of ${MODES.join(", ")}`);
const tag = (env.DEPLOY_TAG || "").trim();
if (tag && !TAG_RE.test(tag)) throw new Error("DEPLOY_TAG must be a stable vX.Y.Z tag; prereleases never deploy");
if (!tag && (mode === "deploy" || mode === "rollback")) throw new Error(`${mode} needs DEPLOY_TAG`);
const raw = (env.DEPLOY_SERVICES || "all").trim();
const services = raw === "all" ? [...SERVICES] : raw.split(",").map((s) => s.trim()).filter(Boolean);
if (!services.length || services.some((s) => !SERVICES.includes(s))) throw new Error(`DEPLOY_SERVICES must be all or a list of ${SERVICES.join(", ")}`);
let secretVersions = {};
if (env.DEPLOY_SECRET_VERSIONS) {
secretVersions = JSON.parse(env.DEPLOY_SECRET_VERSIONS);
for (const [k, v] of Object.entries(secretVersions)) {
if (!SERVICES.includes(k) || !VERSION_RE.test(v)) throw new Error("DEPLOY_SECRET_VERSIONS maps a service to a secret version id");
}
if (mode !== "refresh-secrets") throw new Error("DEPLOY_SECRET_VERSIONS is only accepted with refresh-secrets");
}
return { cluster, prefix, definitions, counts, run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}` };
const bake = Number(env.DEPLOY_BAKE_SECONDS ?? 300);
if (!Number.isInteger(bake) || bake < 0 || bake > 3600) throw new Error("DEPLOY_BAKE_SECONDS must be 0 through 3600");
return {
cluster, prefix, mode, tag, services, secretVersions, bake,
fleet: env.DEPLOY_FLEET !== "false",
allowDroppedSecrets: env.DEPLOY_ALLOW_DROPPED_SECRETS === "true",
images: env.DEPLOY_IMAGES_JSON ? JSON.parse(readFileSync(env.DEPLOY_IMAGES_JSON, "utf8")) : null,
run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}`,
};
}
function compareTags(a, b) {
const x = TAG_RE.exec(a).slice(1).map(Number);
const y = TAG_RE.exec(b).slice(1).map(Number);
for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] - y[i];
return 0;
}
function assertResponse(out) {
@@ -33,6 +67,141 @@ function assertResponse(out) {
return out;
}
const tagValue = (tags, key) => tags?.find((t) => t.key === key)?.value;
// parseRef splits arn:aws:secretsmanager:<region>:<acct>:secret:<name>:<key>:<stage>:<version>.
function parseRef(valueFrom) {
const parts = (valueFrom || "").split(":");
if (parts[2] !== "secretsmanager" || parts.length !== 10) return null;
return { arn: parts.slice(0, 7).join(":"), key: parts[7], stage: parts[8], version: parts[9] };
}
// secretService names the service a secret belongs to: <prefix>/<service>-XXXXXX.
function secretService(arn, prefix) {
const name = arn.split(":")[6] || "";
const m = new RegExp(`^${prefix}/([a-z]+)-[A-Za-z0-9]{6}$`).exec(name);
return m ? m[1] : null;
}
const REGISTER_FIELDS = [
"family", "taskRoleArn", "executionRoleArn", "networkMode", "containerDefinitions", "volumes",
"placementConstraints", "requiresCompatibilities", "cpu", "memory", "pidMode", "ipcMode",
"proxyConfiguration", "inferenceAccelerators", "ephemeralStorage", "runtimePlatform", "enableFaultInjection",
];
async function describe(client, taskDefinition) {
const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition, include: ["TAGS"] }));
if (!out.taskDefinition || out.taskDefinition.status !== "ACTIVE") throw new Error(`${taskDefinition} is not an active task definition`);
return { definition: out.taskDefinition, tags: out.tags || [] };
}
async function resolveVersions(clients, config, refs) {
const pins = {};
for (const [arn, seen] of Object.entries(refs)) {
const service = secretService(arn, config.prefix);
if (config.secretVersions[service]) {
pins[arn] = config.secretVersions[service];
continue;
}
// Explicit versions mean a targeted roll; everyone else keeps the newest pin.
if (config.mode === "refresh-secrets" && !Object.keys(config.secretVersions).length) {
const out = await clients.secrets.send(new DescribeSecretCommand({ SecretId: arn }));
const current = Object.entries(out.VersionIdsToStages || {}).find(([, stages]) => stages.includes("AWSCURRENT"));
if (!current) throw new Error(`${service || "a"} secret has no AWSCURRENT version`);
pins[arn] = current[0];
continue;
}
// Rotation pins newer versions than Terraform's, so the newest pin anyone uses wins.
const out = await clients.secrets.send(new ListSecretVersionIdsCommand({ SecretId: arn, IncludeDeprecated: true, MaxResults: 100 }));
const created = new Map((out.Versions || []).map((v) => [v.VersionId, new Date(v.CreatedDate).getTime()]));
const known = [...seen].filter((v) => created.has(v));
if (!known.length) throw new Error(`no pinned version of the ${service || ""} secret still exists`);
pins[arn] = known.sort((a, b) => created.get(b) - created.get(a))[0];
}
return pins;
}
function repin(container, pins) {
for (const secret of container.secrets || []) {
const ref = parseRef(secret.valueFrom);
if (ref && ref.version && pins[ref.arn]) {
secret.valueFrom = `${ref.arn}:${ref.key}:${ref.stage}:${pins[ref.arn]}`;
}
}
}
function same(a, b) {
return JSON.stringify(REGISTER_FIELDS.map((f) => a[f] ?? null)) === JSON.stringify(REGISTER_FIELDS.map((f) => b[f] ?? null));
}
async function register(client, source, rendered, tags, release, report) {
const carried = tags.filter((t) => t.key !== RELEASE_TAG && !t.key.startsWith("aws:"));
if (release) carried.push({ key: RELEASE_TAG, value: release });
if (same(source.definition, rendered) && tagValue(source.tags, RELEASE_TAG) === release) return source.definition.taskDefinitionArn;
const input = {};
for (const f of REGISTER_FIELDS) if (rendered[f] !== undefined && rendered[f] !== null) input[f] = rendered[f];
input.tags = carried;
const out = await client.send(new RegisterTaskDefinitionCommand(input));
const arn = out.taskDefinition?.taskDefinitionArn;
if (!arn) throw new Error(`registering ${rendered.family} returned no revision`);
report(`Registered ${arn.split("/").pop()}`);
return arn;
}
async function logTail(clients, definition, task, report, quiet) {
const container = definition.containerDefinitions[0];
const options = container.logConfiguration?.options || {};
if (!options["awslogs-group"] || !options["awslogs-stream-prefix"]) return [];
try {
const out = await clients.logs.send(new GetLogEventsCommand({
logGroupName: options["awslogs-group"],
logStreamName: `${options["awslogs-stream-prefix"]}/${container.name}/${task.split("/").pop()}`,
startFromHead: false, limit: 60,
}));
const lines = (out.events || []).map((e) => String(e.message).slice(0, 2000));
if (!quiet) for (const line of lines) report(` | ${line}`);
return lines;
} catch {
report(" | (no log events could be read for this task)");
return [];
}
}
// runTask runs a one-off task to completion; capture may be a predicate the log tail must meet.
async function runTask(clients, config, network, arn, definition, label, sleep, report, options = {}) {
const { command, environment, capture } = Array.isArray(options) ? { command: options } : options;
const token = createHash("sha256").update(`${config.run}/${arn}/${label}/${JSON.stringify(command || [])}/${JSON.stringify(environment || [])}`).digest("hex");
const name = definition.containerDefinitions[0].name;
const override = { name, ...(command ? { command } : {}), ...(environment ? { environment } : {}) };
const started = assertResponse(await clients.ecs.send(new RunTaskCommand({
cluster: config.cluster, taskDefinition: arn, launchType: "FARGATE", platformVersion: "1.4.0", count: 1,
networkConfiguration: network, clientToken: token.slice(0, 64), startedBy: `${label.replace(/\W+/g, "-")}-${token.slice(0, 20)}`.slice(0, 128),
...(command || environment ? { overrides: { containerOverrides: [override] } } : {}),
})));
const task = started.tasks?.[0]?.taskArn;
if (!task) throw new Error(`${label} task was not created`);
report(`${label} task: ${task.split("/").pop()}`);
for (let attempt = 0; attempt < 240; attempt++) {
const out = assertResponse(await clients.ecs.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] })));
const status = out.tasks?.[0];
if (!status) throw new Error(`${label} task is missing`);
if (status.lastStatus === "STOPPED") {
const container = status.containers?.find((c) => c.name === name);
const ok = status.stopCode === "EssentialContainerExited" && container?.exitCode === 0;
let lines = !ok || command || capture ? await logTail(clients, definition, task, report, ok && capture) : [];
if (!ok) throw new Error(`${label} failed; its log tail is above`);
// Log delivery can trail the task stopping by a few seconds.
for (let retry = 0; typeof capture === "function" && !capture(lines) && retry < 6; retry++) {
await sleep(5000);
lines = await logTail(clients, definition, task, report, true);
}
return lines;
}
await sleep(5000);
}
throw new Error(`${label} is unresolved after 20 minutes; no services were updated. Inspect the task before retrying`);
}
async function waitForService(client, config, name, definition, sleep) {
for (let attempt = 0; attempt < 180; attempt++) {
const out = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] })));
@@ -47,78 +216,196 @@ async function waitForService(client, config, name, definition, sleep) {
throw new Error(`${name} deployment did not become healthy within 15 minutes`);
}
async function deploy(client, config, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), report = console.log) {
const definitions = {};
for (const name of ["migration", "backend", "consumer"]) {
const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition: config.definitions[name] }));
const definition = out.taskDefinition;
const container = definition?.containerDefinitions?.find((c) => c.name === name);
if (!container || definition.status !== "ACTIVE" || !/@sha256:[a-f0-9]{64}$/.test(container.image || "")) throw new Error(`Invalid or unpinned ${name} task definition`);
definitions[name] = { definition, container };
// bake watches a stable service and its alarms; ECS rolling back counts as failure.
async function bake(clients, config, name, definition, sleep, report) {
for (let waited = 0; waited < config.bake; waited += 15) {
const alarms = await clients.cloudwatch.send(new DescribeAlarmsCommand({ AlarmNamePrefix: `${config.prefix}-${name}-`, StateValue: "ALARM" }));
const firing = (alarms.MetricAlarms || []).map((a) => a.AlarmName);
if (firing.length) throw new Error(`${name} raised ${firing.join(", ")} during the bake`);
const out = assertResponse(await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] })));
if (out.services?.[0]?.taskDefinition !== definition) throw new Error(`${name} was rolled back during the bake`);
await sleep(15000);
}
const migration = definitions.migration.container;
if (migration.image !== definitions.backend.container.image || migration.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image");
const migrationSecret = migration.secrets?.find((s) => s.name === "PRIMARY_DB");
const directSecret = definitions.backend.container.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT");
if (!directSecret || directSecret.valueFrom !== migrationSecret?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version");
if (!definitions.backend.container.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment");
const current = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`, `${config.prefix}-consumer`] })));
if (current.services?.length !== 2 || current.services.some((s) => s.status !== "ACTIVE")) throw new Error("Both ECS services must already exist");
const backend = current.services.find((s) => s.serviceName === `${config.prefix}-backend`);
if (config.bake) report(`${name} baked for ${config.bake}s with no alarm`);
}
function imageFor(config, key, fallback) {
if (!config.images) return fallback;
const digest = config.images.images?.[key];
if (!DIGEST_RE.test(digest || "")) throw new Error(`images.json for ${config.tag} has no ${key} digest`);
return `${config.images.registry}/${key.replace(/-kafka$/, "")}@${digest}`;
}
async function deploy(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log) {
const ecs = clients.ecs;
const names = SERVICES.map((s) => `${config.prefix}-${s}`);
const current = assertResponse(await ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: names })));
const live = {};
for (const s of current.services || []) live[s.serviceName.slice(config.prefix.length + 1)] = s;
for (const name of config.services) {
if (live[name]?.status !== "ACTIVE") throw new Error(`${name} must already exist as an ECS service`);
}
const backend = live.backend;
if (!backend?.networkConfiguration?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
const token = createHash("sha256").update(`${config.run}/${config.definitions.migration}`).digest("hex");
const started = assertResponse(await client.send(new RunTaskCommand({
cluster: config.cluster, taskDefinition: config.definitions.migration,
launchType: "FARGATE", platformVersion: "1.4.0", count: 1,
networkConfiguration: backend.networkConfiguration, clientToken: token,
startedBy: `migration-${token.slice(0, 24)}`,
})));
const task = started.tasks?.[0]?.taskArn;
if (!task) throw new Error("Migration task was not created");
report(`Migration task: ${task}`);
let migrated = false;
for (let attempt = 0; attempt < 240; attempt++) {
const out = assertResponse(await client.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] })));
const status = out.tasks?.[0];
if (!status) throw new Error("Migration task is missing");
if (status.lastStatus === "STOPPED") {
const container = status.containers?.find((c) => c.name === "migration");
if (status.stopCode !== "EssentialContainerExited" || container?.exitCode !== 0) throw new Error("Migration failed; inspect its CloudWatch logs before retrying");
migrated = true;
break;
}
await sleep(5000);
const liveDefs = {};
for (const [name, s] of Object.entries(live)) liveDefs[name] = await describe(ecs, s.taskDefinition);
const liveTag = tagValue(liveDefs.backend.tags, RELEASE_TAG);
const tag = config.tag || liveTag;
if (!tag) throw new Error("The live backend carries no release tag yet; pass DEPLOY_TAG");
if (config.tag && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) < 0 && config.mode !== "rollback") {
throw new Error(`${config.tag} is older than the live ${liveTag}; use mode=rollback to go back`);
}
if (!migrated) throw new Error("Migration status is unresolved; no services were updated. Inspect the running task before retrying");
report("Migration completed successfully");
if (config.mode === "rollback" && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) > 0) {
throw new Error(`rollback target ${config.tag} is newer than the live ${liveTag}`);
}
if (config.images && config.images.tag !== tag) throw new Error(`images.json describes ${config.images.tag}, not ${tag}`);
if (!config.images && config.mode !== "refresh-secrets") throw new Error("DEPLOY_IMAGES_JSON is required unless refreshing secrets");
report(`Release ${tag} (${config.mode}); live backend reports ${liveTag || "no release"}`);
const deployBackend = config.services.includes("backend");
const families = [...config.services];
const extra = {};
for (const name of ["migration", "validation", "ops"]) {
if (name === "migration" && (!deployBackend || config.mode === "rollback")) continue;
try {
extra[name] = await describe(ecs, `${config.prefix}-${name}`);
families.push(name);
} catch (error) {
if (name === "migration") throw error;
report(`No ${name} task definition; skipping it`);
}
}
const sources = {};
for (const name of config.services) sources[name] = await describe(ecs, `${config.prefix}-${name}`);
Object.assign(sources, extra);
const refs = {};
const collect = (def) => {
for (const c of def.containerDefinitions || []) for (const s of c.secrets || []) {
const ref = parseRef(s.valueFrom);
if (ref?.version) (refs[ref.arn] ||= new Set()).add(ref.version);
}
};
for (const name of families) collect(sources[name].definition);
for (const def of Object.values(liveDefs)) collect(def.definition);
const pins = await resolveVersions(clients, config, refs);
const variant = (name) => {
const v = tagValue(sources[name].tags, VARIANT_TAG);
if (v !== "kafka" && v !== "default") throw new Error(`${config.prefix}-${name} has no ${VARIANT_TAG} tag; apply the runtime Terraform first`);
return v;
};
const backendKey = deployBackend || families.includes("ops") ? `backend${variant(deployBackend ? "backend" : "ops") === "kafka" ? "-kafka" : ""}` : null;
const rendered = {};
for (const name of families) {
const def = structuredClone(sources[name].definition);
const container = def.containerDefinitions?.[0];
if (!container || def.containerDefinitions.length !== 1) throw new Error(`${name} must have exactly one container`);
repin(container, pins);
if (SERVICES.includes(name)) {
if (container.name !== name) throw new Error(`${name} container is named ${container.name}`);
const key = `${name}${variant(name) === "kafka" ? "-kafka" : ""}`;
if (config.images && !container.image.startsWith(`${config.images.registry}/${name}@`)) {
throw new Error(`${name} does not run a digest from ${config.images.registry}; fix its image in Terraform first`);
}
container.image = imageFor(config, key, liveDefs[name].definition.containerDefinitions[0].image);
} else if (name !== "validation") {
container.image = imageFor(config, backendKey, liveDefs.backend.definition.containerDefinitions[0].image);
}
rendered[name] = def;
}
for (const name of config.services) {
const before = new Set((liveDefs[name].definition.containerDefinitions[0].secrets || []).map((s) => s.name));
const after = new Set((rendered[name].containerDefinitions[0].secrets || []).map((s) => s.name));
const dropped = [...before].filter((k) => !after.has(k));
if (dropped.length && !config.allowDroppedSecrets) {
throw new Error(`${name} would lose ${dropped.join(", ")}; restore them in Terraform or set DEPLOY_ALLOW_DROPPED_SECRETS=true`);
}
}
if (rendered.migration) {
const m = rendered.migration.containerDefinitions[0];
const b = rendered.backend.containerDefinitions[0];
if (m.image !== b.image || m.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image");
const direct = b.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT");
if (!direct || direct.valueFrom !== m.secrets?.find((s) => s.name === "PRIMARY_DB")?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version");
if (!b.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment");
}
const arns = {};
for (const name of families) arns[name] = await register(ecs, sources[name], rendered[name], sources[name].tags, tag, report);
const network = backend.networkConfiguration;
if (arns.validation) await runTask(clients, config, network, arns.validation, rendered.validation, "Dependency check", sleep, report);
if (arns.migration) {
await runTask(clients, config, network, arns.migration, rendered.migration, "Migration", sleep, report);
report("Migration completed successfully");
}
const updated = [];
try {
for (const name of ["backend", "consumer"]) {
const previous = current.services.find((s) => s.serviceName === `${config.prefix}-${name}`);
updated.push(previous);
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: config.definitions[name], ...(config.counts[name] === undefined ? {} : { desiredCount: config.counts[name] }) }));
await waitForService(client, config, name, config.definitions[name], sleep);
report(`${name} service is stable on the requested revision`);
for (const wave of WAVES) {
const group = wave.filter((n) => config.services.includes(n));
// allSettled: a sibling still rolling must finish before anything is restored.
const results = await Promise.allSettled(group.map(async (name) => {
const previous = live[name];
const unchanged = previous.taskDefinition === arns[name];
if (unchanged && config.mode !== "redeploy") {
report(`${name} already runs ${arns[name].split("/").pop()}`);
return;
}
updated.push(previous);
await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: arns[name], forceNewDeployment: unchanged }));
await waitForService(ecs, config, name, arns[name], sleep);
report(`${name} is stable on ${arns[name].split("/").pop()}`);
await bake(clients, config, name, arns[name], sleep, report);
}));
const failed = results.find((r) => r.status === "rejected");
if (failed) throw failed.reason;
}
} catch (error) {
for (const previous of updated.reverse()) {
try {
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount }));
report(`Restored previous deployment configuration for ${previous.serviceName}; verify its health`);
await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount }));
report(`Restored ${previous.serviceName} to ${previous.taskDefinition.split("/").pop()}; verify its health`);
} catch {
report(`Rollback needs operator attention for ${previous.serviceName}`);
}
}
throw error;
}
if (config.fleet && arns.ops && config.images && (config.mode === "deploy" || config.mode === "rollback")) {
const digests = ["worker", "worker-kafka", "consumer", "consumer-kafka"]
.filter((k) => DIGEST_RE.test(config.images.images?.[k] || ""))
.map((k) => `${k}=${config.images.images[k]}`);
const command = ["fleet", "rollout", "start", tag, "--canary", "1", "--waves", "100", "--bake", "30m"];
if (digests.length) command.push("--digests", digests.join(","));
await runTask(clients, config, network, arns.ops, rendered.ops, "Fleet rollout", sleep, report, { command });
report(`Workers move to ${tag} in canary waves; follow with \`warmblyctl fleet rollout status\``);
}
return { tag, arns, pins: Object.fromEntries(Object.entries(pins).map(([arn, v]) => [secretService(arn, config.prefix) || arn, v])) };
}
if (require.main === module) {
const client = new ECSClient({ region: "us-east-1", maxAttempts: 2 });
Promise.resolve().then(() => deploy(client, configuration(process.env))).catch((error) => {
console.error(error?.$metadata ? `AWS request failed: ${error.name}` : error.message);
const region = { region: "us-east-1", maxAttempts: 3 };
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
const lines = [];
const report = (line) => { console.log(line); lines.push(line); };
Promise.resolve().then(() => deploy(clients, configuration(process.env), undefined, report)).then((out) => {
report(`Deployed ${out.tag}. Secret versions: ${Object.entries(out.pins).map(([k, v]) => `${k}=${v.slice(0, 8)}`).join(", ") || "none"}`);
}).catch((error) => {
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
console.error(message);
lines.push(`**Failed:** ${message}`);
process.exitCode = 1;
}).finally(() => client.destroy());
}).finally(() => {
if (process.env.GITHUB_STEP_SUMMARY) {
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### AWS deploy\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
}
for (const c of Object.values(clients)) c.destroy();
});
}
module.exports = { configuration, deploy };
module.exports = { configuration, deploy, runTask, describe, compareTags, parseRef, secretService, SERVICES };
+277 -61
View File
@@ -1,89 +1,305 @@
const { test } = require("node:test");
const assert = require("node:assert/strict");
const { configuration, deploy } = require("./aws-deploy.cjs");
const { mkdtempSync, writeFileSync } = require("node:fs");
const { join } = require("node:path");
const { tmpdir } = require("node:os");
const { configuration, deploy, compareTags, parseRef } = require("./aws-deploy.cjs");
const root = "arn:aws:ecs:us-east-1:000000000000:";
const digest = "sha256:" + "a".repeat(64);
const secret = "arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture:PRIMARY_DB_DIRECT::version";
function config() {
const registry = "ghcr.io/warmbly/warmbly";
const d = (c) => "sha256:" + c.repeat(64);
const KAFKA = ["backend", "consumer", "tracking", "migration", "ops"];
const FAMILIES = ["backend", "consumer", "forms", "tracking", "realtime", "migration", "validation", "ops"];
const secretArn = (svc) => `arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture/${svc}-AbCdEf`;
function imagesFile(tag = "v1.3.0") {
const dir = mkdtempSync(join(tmpdir(), "aws-deploy-"));
const images = {};
for (const svc of ["backend", "consumer", "worker", "forms", "tracking", "realtime"]) images[svc] = d("b");
for (const svc of ["backend", "consumer", "worker", "tracking"]) images[`${svc}-kafka`] = d("c");
writeFileSync(join(dir, "images.json"), JSON.stringify({ tag, registry, images }));
return join(dir, "images.json");
}
function config(env = {}) {
return configuration({
AWS_ECS_CLUSTER: root + "cluster/fixture",
AWS_MIGRATION_TASK_DEFINITION: root + "task-definition/fixture-migration:2",
AWS_BACKEND_TASK_DEFINITION: root + "task-definition/fixture-backend:2",
AWS_CONSUMER_TASK_DEFINITION: root + "task-definition/fixture-consumer:2",
AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_TAG: "v1.3.0", DEPLOY_IMAGES_JSON: imagesFile(env.DEPLOY_TAG || "v1.3.0"),
DEPLOY_BAKE_SECONDS: "30", ...env,
});
}
function definition(family, revision, options) {
const secretSvc = ["migration", "validation", "ops"].includes(family) ? "backend" : family;
const version = family === "backend" && revision === 1 ? "rotated" : options.versions?.[family] || "terraform";
const secrets = family === "migration"
? [{ name: "PRIMARY_DB", valueFrom: `${secretArn("backend")}:PRIMARY_DB_DIRECT::${version}` }]
: [
{ name: "PRIMARY_DB_DIRECT", valueFrom: `${secretArn(secretSvc)}:PRIMARY_DB_DIRECT::${version}` },
{ name: "REDIS_PASSWORD", valueFrom: `${secretArn(secretSvc)}:REDIS_PASSWORD::` },
];
if (family === "backend" && (revision === 1 || !options.dropReadKeys)) secrets.push({ name: "CREDENTIALS_ENCRYPTION_READ_KEYS", valueFrom: `${secretArn("backend")}:CREDENTIALS_ENCRYPTION_READ_KEYS::${version}` });
const image = family === "validation" ? "node:22-alpine"
: `${options.registry || registry}/${["migration", "ops"].includes(family) ? "backend" : family}@${d("a")}`;
return {
taskDefinitionArn: `${root}task-definition/fixture-${family}:${revision}`, family: `fixture-${family}`, revision, status: "ACTIVE",
networkMode: "awsvpc", cpu: "256", memory: "512", requiresCompatibilities: ["FARGATE"],
containerDefinitions: [{
name: family, image, secrets,
entryPoint: family === "migration" ? ["/app/migrate"] : undefined,
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": "/fixture", "awslogs-stream-prefix": family } },
}],
};
}
function fixture(options = {}) {
const calls = [];
const services = ["backend", "consumer"].map((name) => ({
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 1, runningCount: 1, pendingCount: 0,
taskDefinition: root + `task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "ENABLED" } },
}));
const client = { async send(command) {
const name = command.constructor.name;
calls.push({ name, input: command.input });
if (name === "DescribeTaskDefinitionCommand") {
const service = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
return { taskDefinition: { status: "ACTIVE", containerDefinitions: [{
name: service, image: `registry/${service === "migration" ? "backend" : service}@${service === "migration" && options.imageMismatch ? "sha256:" + "b".repeat(64) : digest}`,
entryPoint: service === "migration" ? ["/app/migrate"] : undefined,
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
secrets: [{ name: service === "migration" ? "PRIMARY_DB" : "PRIMARY_DB_DIRECT", valueFrom: secret }],
}] } };
const defs = {};
const tagsOf = {};
for (const family of FAMILIES) {
for (const revision of [1, 2]) {
const arn = `${root}task-definition/fixture-${family}:${revision}`;
defs[arn] = definition(family, revision, options);
const tags = [{ key: "Environment", value: "production" }];
if (!(options.noVariant && family === "backend" && revision === 2)) tags.push({ key: "warmbly-image-variant", value: KAFKA.includes(family) ? "kafka" : "default" });
if (revision === 1) tags.push({ key: "warmbly-release", value: options.liveTag || "v1.2.0" });
tagsOf[arn] = tags;
}
if (name === "DescribeServicesCommand") return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) };
if (name === "RunTaskCommand") return options.placementFailure ? { failures: [{ reason: "fixture" }] } : { tasks: [{ taskArn: root + "task/fixture/migration" }] };
if (name === "DescribeTasksCommand") return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: "migration", exitCode: options.migrationFailure ? 1 : 0 }] }] };
if (name === "UpdateServiceCommand") {
if (command.input.service === "fixture-consumer" && command.input.taskDefinition.endsWith(":2") && options.consumerFailure) throw new Error("consumer update failed");
}
const latest = (family) => Object.values(defs).filter((x) => x.family === `fixture-${family}`).sort((a, b) => b.revision - a.revision)[0];
const services = ["backend", "consumer", "forms", "tracking", "realtime"].map((name) => ({
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 2, runningCount: 2, pendingCount: 0,
taskDefinition: `${root}task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "DISABLED" } },
}));
const tasks = {};
const record = (client) => ({ async send(command) {
const name = command.constructor.name;
calls.push({ client, name, input: structuredClone(command.input) });
switch (name) {
case "DescribeTaskDefinitionCommand": {
const ref = command.input.taskDefinition;
const def = ref.startsWith("arn:") ? defs[ref] : latest(ref.replace(/^fixture-/, ""));
if (!def) throw new Error(`no task definition ${ref}`);
return { taskDefinition: structuredClone(def), tags: structuredClone(tagsOf[def.taskDefinitionArn]) };
}
case "RegisterTaskDefinitionCommand": {
const prev = latest(command.input.family.replace(/^fixture-/, ""));
const arn = `${root}task-definition/${command.input.family}:${prev.revision + 1}`;
defs[arn] = { ...structuredClone(command.input), revision: prev.revision + 1, taskDefinitionArn: arn, status: "ACTIVE" };
delete defs[arn].tags;
tagsOf[arn] = command.input.tags;
return { taskDefinition: { taskDefinitionArn: arn } };
}
case "DescribeServicesCommand":
return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) };
case "UpdateServiceCommand": {
const service = services.find((s) => s.serviceName === command.input.service);
const svc = service.serviceName.replace("fixture-", "");
if (options.updateFailure === svc && !command.input.taskDefinition.endsWith(":1")) throw new Error(`${svc} update failed`);
service.taskDefinition = command.input.taskDefinition;
if (command.input.desiredCount !== undefined) service.runningCount = service.desiredCount = command.input.desiredCount;
if (options.circuitBreakerRollback && command.input.taskDefinition.endsWith(":2")) service.taskDefinition = root + "task-definition/fixture-backend:1";
if (command.input.desiredCount !== undefined) service.desiredCount = command.input.desiredCount;
service.deployments = [{ rolloutState: options.rolloutFailure === svc && !command.input.taskDefinition.endsWith(":1") ? "FAILED" : "COMPLETED" }];
if (options.circuitBreaker === svc) service.taskDefinition = `${root}task-definition/fixture-${svc}:1`;
return {};
}
throw new Error(`Unexpected command ${name}`);
} };
return { client, calls };
case "RunTaskCommand": {
if (options.placementFailure) return { failures: [{ reason: "fixture" }] };
const family = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
const arn = `${root}task/fixture/${family}-${Object.keys(tasks).length}`;
tasks[arn] = family;
return { tasks: [{ taskArn: arn }] };
}
case "DescribeTasksCommand": {
const family = tasks[command.input.tasks[0]];
const exitCode = options.taskFailure === family ? 1 : 0;
return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: family, exitCode }] }] };
}
case "GetLogEventsCommand":
return { events: [{ message: `log line from ${command.input.logStreamName}` }] };
case "DescribeAlarmsCommand": {
const firing = options.alarm && command.input.AlarmNamePrefix === `fixture-${options.alarm}-`;
return { MetricAlarms: firing ? [{ AlarmName: `fixture-${options.alarm}-5xx` }] : [] };
}
case "ListSecretVersionIdsCommand":
return { Versions: [
{ VersionId: "terraform", CreatedDate: new Date("2026-01-01") },
{ VersionId: "rotated", CreatedDate: new Date("2026-02-01") },
{ VersionId: "stale", CreatedDate: new Date("2025-01-01") },
] };
case "DescribeSecretCommand":
return { VersionIdsToStages: { terraform: ["AWSPREVIOUS"], "current-version-id-000000000000000000": ["AWSCURRENT"] } };
default:
throw new Error(`Unexpected command ${name}`);
}
} });
const clients = { ecs: record("ecs"), secrets: record("secrets"), cloudwatch: record("cloudwatch"), logs: record("logs") };
return { clients, calls, defs, tagsOf, services };
}
test("migration success gates both service updates and preserves desired counts", async () => {
const { client, calls } = fixture();
await deploy(client, config(), async () => {}, () => {});
const migration = calls.findIndex((c) => c.name === "DescribeTasksCommand");
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
assert.equal(updates.length, 2);
assert.ok(calls.indexOf(updates[0]) > migration);
assert.ok(updates.every((c) => !Object.hasOwn(c.input, "desiredCount")));
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
const run = (f, cfg, lines = []) => deploy(f.clients, cfg, async () => {}, (l) => lines.push(l));
const registered = (f) => f.calls.filter((c) => c.name === "RegisterTaskDefinitionCommand").map((c) => c.input);
const updates = (f) => f.calls.filter((c) => c.name === "UpdateServiceCommand").map((c) => c.input);
const runs = (f) => f.calls.filter((c) => c.name === "RunTaskCommand").map((c) => c.input);
const valueFrom = (def, key) => def.containerDefinitions[0].secrets.find((s) => s.name === key).valueFrom;
test("configuration refuses prereleases, bad modes and secret pins outside refresh-secrets", () => {
assert.throws(() => config({ DEPLOY_TAG: "v1.3.0-rc.1" }), /prereleases never deploy/);
assert.throws(() => config({ DEPLOY_MODE: "yolo" }), /DEPLOY_MODE/);
assert.throws(() => config({ DEPLOY_SERVICES: "backend,worker" }), /DEPLOY_SERVICES/);
assert.throws(() => config({ DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "a".repeat(36) }) }), /refresh-secrets/);
assert.throws(() => config({ DEPLOY_MODE: "refresh-secrets", DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "x" }) }), /secret version id/);
assert.ok(compareTags("v1.10.0", "v1.9.9") > 0);
assert.deepEqual(parseRef(`${secretArn("backend")}:K::v`), { arn: secretArn("backend"), key: "K", stage: "", version: "v" });
});
for (const failure of ["migrationFailure", "placementFailure", "imageMismatch"]) {
test(`${failure} leaves existing services untouched`, async () => {
const { client, calls } = fixture({ [failure]: true });
await assert.rejects(deploy(client, config(), async () => {}, () => {}));
assert.ok(calls.every((c) => c.name !== "UpdateServiceCommand"));
test("a release renders from the latest revision, swaps variant digests and carries tags", async () => {
const f = fixture();
await run(f, config());
const byFamily = Object.fromEntries(registered(f).map((r) => [r.family.replace("fixture-", ""), r]));
assert.equal(byFamily.backend.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.forms.containerDefinitions[0].image, `${registry}/forms@${d("b")}`);
assert.equal(byFamily.tracking.containerDefinitions[0].image, `${registry}/tracking@${d("c")}`);
assert.equal(byFamily.migration.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.ops.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.validation.containerDefinitions[0].image, "node:22-alpine");
for (const r of Object.values(byFamily)) {
assert.ok(r.tags.some((t) => t.key === "Environment"));
assert.deepEqual(r.tags.filter((t) => t.key === "warmbly-release"), [{ key: "warmbly-release", value: "v1.3.0" }]);
assert.equal(r.cpu, "256");
}
});
test("the newest pinned secret version wins so a stale Terraform pin cannot undo a rotation", async () => {
const f = fixture();
await run(f, config());
const backend = registered(f).find((r) => r.family === "fixture-backend");
const migration = registered(f).find((r) => r.family === "fixture-migration");
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
assert.ok(valueFrom(migration, "PRIMARY_DB").endsWith("::rotated"));
assert.ok(valueFrom(backend, "REDIS_PASSWORD").endsWith("REDIS_PASSWORD::"));
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::terraform"));
});
test("refresh-secrets pins every secret to AWSCURRENT and keeps the live images", async () => {
const f = fixture();
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0" }));
const backend = registered(f).find((r) => r.family === "fixture-backend");
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
assert.equal(backend.containerDefinitions[0].image, `${registry}/backend@${d("a")}`);
assert.ok(backend.tags.some((t) => t.key === "warmbly-release" && t.value === "v1.2.0"));
assert.ok(!runs(f).some((r) => r.overrides), "refresh-secrets starts no fleet rollout");
});
test("explicit secret versions roll only what they name; other secrets keep their newest pin", async () => {
const f = fixture();
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0",
DEPLOY_SERVICES: "consumer", DEPLOY_SECRET_VERSIONS: JSON.stringify({ consumer: "explicit-version-id-0000000000000000" }) }));
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
const validation = registered(f).find((r) => r.family === "fixture-validation");
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::explicit-version-id-0000000000000000"));
assert.ok(valueFrom(validation, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
assert.ok(!f.calls.some((c) => c.name === "DescribeSecretCommand"));
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-consumer"]);
});
test("validation and migration gate every service update", async () => {
const f = fixture();
await run(f, config());
const firstUpdate = f.calls.findIndex((c) => c.name === "UpdateServiceCommand");
const taskRuns = f.calls.map((c, i) => [c, i]).filter(([c]) => c.name === "RunTaskCommand");
assert.deepEqual(taskRuns.slice(0, 2).map(([c]) => /fixture-(\w+):/.exec(c.input.taskDefinition)[1]), ["validation", "migration"]);
assert.ok(taskRuns[1][1] < firstUpdate);
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-backend", "fixture-consumer", "fixture-forms", "fixture-tracking", "fixture-realtime"]);
assert.ok(updates(f).every((u) => !Object.hasOwn(u, "desiredCount")));
});
for (const [failure, message] of [
[{ taskFailure: "migration" }, /Migration failed/], [{ taskFailure: "validation" }, /Dependency check failed/],
[{ placementFailure: true }, /task-placement failure/], [{ noVariant: true }, /no warmbly-image-variant tag/],
[{ registry: "docker.io/someone" }, /does not run a digest from ghcr/],
]) {
test(`${JSON.stringify(failure)} leaves every service untouched`, async () => {
const f = fixture(failure);
await assert.rejects(run(f, config()), message);
assert.equal(updates(f).length, 0);
});
}
test("consumer rollout failure restores previous service revisions without database rollback", async () => {
const { client, calls } = fixture({ consumerFailure: true });
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /consumer update failed/);
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
assert.equal(updates.length, 4);
assert.ok(updates.slice(2).every((c) => c.input.taskDefinition.endsWith(":1") && c.input.desiredCount === 1));
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
test("a failed migration prints its log tail", async () => {
const f = fixture({ taskFailure: "migration" });
const lines = [];
await assert.rejects(run(f, config(), lines), /Migration failed/);
assert.ok(lines.some((l) => l.includes("log line from migration/migration/")));
});
test("an older tag is refused unless the mode is rollback, and rollback skips the migration", async () => {
await assert.rejects(run(fixture({ liveTag: "v1.4.0" }), config()), /older than the live v1.4.0/);
await assert.rejects(run(fixture(), config({ DEPLOY_MODE: "rollback" })), /newer than the live/);
const f = fixture({ liveTag: "v1.4.0" });
await run(f, config({ DEPLOY_MODE: "rollback" }));
assert.ok(!runs(f).some((r) => r.taskDefinition.includes("fixture-migration")));
assert.ok(runs(f).some((r) => r.overrides), "rollback still moves the fleet");
});
test("a revision that drops a secret the live service reads is refused", async () => {
const f = fixture({ dropReadKeys: true });
await assert.rejects(run(f, config()), /would lose CREDENTIALS_ENCRYPTION_READ_KEYS/);
assert.equal(registered(f).length, 0);
const g = fixture({ dropReadKeys: true });
await run(g, config({ DEPLOY_ALLOW_DROPPED_SECRETS: "true" }));
});
test("an alarm during the bake restores every updated service in reverse order", async () => {
const f = fixture({ alarm: "consumer" });
await assert.rejects(run(f, config()), /fixture-consumer-5xx/);
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1"));
assert.deepEqual(restores.map((u) => u.service), ["fixture-consumer", "fixture-backend"]);
assert.ok(restores.every((u) => u.desiredCount === 2));
assert.ok(!updates(f).some((u) => u.service === "fixture-forms"));
});
test("a failure in the parallel wave waits for its siblings and then restores all of them", async () => {
const f = fixture({ rolloutFailure: "forms" });
await assert.rejects(run(f, config()), /forms deployment failed/);
const forward = updates(f).filter((u) => !u.taskDefinition.endsWith(":1")).map((u) => u.service);
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")).map((u) => u.service);
assert.equal(forward.length, 5);
assert.deepEqual([...restores].sort(), [...forward].sort());
assert.deepEqual(restores.slice(-2), ["fixture-consumer", "fixture-backend"]);
});
test("a circuit breaker rollback cannot be reported as a successful deployment", async () => {
const { client } = fixture({ circuitBreakerRollback: true });
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /rolled back/);
const f = fixture({ circuitBreaker: "backend" });
await assert.rejects(run(f, config()), /backend deployment rolled back/);
});
test("deployment requires explicit revisions in the selected account and region", () => {
assert.throws(() => configuration({ AWS_ECS_CLUSTER: root.replace("us-east-1", "eu-west-1") + "cluster/fixture" }));
const c = config();
assert.equal(c.counts.backend, undefined);
test("the fleet rollout gets the worker and consumer digests per variant", async () => {
const f = fixture();
await run(f, config());
const ops = runs(f).find((r) => r.overrides);
const command = ops.overrides.containerOverrides[0].command;
assert.deepEqual(command.slice(0, 4), ["fleet", "rollout", "start", "v1.3.0"]);
assert.equal(command[command.indexOf("--digests") + 1], `worker=${d("b")},worker-kafka=${d("c")},consumer=${d("b")},consumer-kafka=${d("c")}`);
assert.equal(ops.overrides.containerOverrides[0].name, "ops");
});
test("an unchanged definition reuses its revision and redeploy forces a fresh deployment", async () => {
const f = fixture();
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_FLEET: "false" }));
const formsArn = updates(f)[0].taskDefinition;
f.services.find((s) => s.serviceName === "fixture-forms").taskDefinition = formsArn;
const before = registered(f).length;
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_MODE: "redeploy", DEPLOY_FLEET: "false" }));
assert.equal(registered(f).length, before);
assert.deepEqual(updates(f).at(-1), { cluster: root + "cluster/fixture", service: "fixture-forms", taskDefinition: formsArn, forceNewDeployment: true });
});
test("no secret value or version-free reference is ever read", async () => {
const f = fixture();
await run(f, config());
assert.ok(!f.calls.some((c) => c.name === "GetSecretValueCommand"));
});
+188
View File
@@ -0,0 +1,188 @@
// Drives a key rotation from GitHub without ever holding a value: the rotator
// task writes new secret versions inside AWS and reports their ids, the deploy
// engine rolls services onto exactly those versions, and warmblyctl in the ops
// task proves each step is safe before the old value is retired.
const { DescribeServicesCommand } = require("@aws-sdk/client-ecs");
const engine = require("./aws-deploy.cjs");
const { configuration, SERVICES } = engine;
const KINDS = {
CREDENTIALS_ENCRYPTION_KEY: "seal",
TWOFA_SECRET: "seal",
UNSUBSCRIBE_SIGNING_SECRET: "sign",
INTERNAL_API_TOKEN: "token",
NODE_BROKER_TOKEN: "token",
};
const ACTIONS = ["status", "prepare", "rotate", "retire", "abort"];
function rotationConfig(env) {
const action = env.ROTATE_ACTION;
const key = env.ROTATE_KEY;
if (!ACTIONS.includes(action)) throw new Error(`ROTATE_ACTION must be one of ${ACTIONS.join(", ")}`);
if (!KINDS[key]) throw new Error(`ROTATE_KEY must be one of ${Object.keys(KINDS).join(", ")}`);
const wait = Number(env.ROTATE_NODE_WAIT_MINUTES ?? 30);
if (!Number.isInteger(wait) || wait < 0 || wait > 240) throw new Error("ROTATE_NODE_WAIT_MINUTES must be 0 through 240");
const base = configuration({ ...env, DEPLOY_MODE: "refresh-secrets", DEPLOY_TAG: "", DEPLOY_SERVICES: "all", DEPLOY_SECRET_VERSIONS: "", DEPLOY_IMAGES_JSON: "", DEPLOY_FLEET: "false" });
return { ...base, action, key, kind: KINDS[key], nodeWaitMs: wait * 60000 };
}
async function rotateKeys(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log, deps = engine) {
const { deploy, runTask, describe } = deps;
// ECS run tokens are idempotent, so every task and roll in one run needs its own.
let seq = 0;
const step = () => `${config.run}/${++seq}`;
const services = await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`] }));
const network = services.services?.[0]?.networkConfiguration;
if (!network?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
const rotator = async (action) => {
const { definition } = await describe(clients.ecs, `${config.prefix}-rotator`);
const environment = [
{ name: "ROTATE_ACTION", value: action }, { name: "ROTATE_KEY", value: config.key }, { name: "ROTATE_RUN", value: `${step()}/${action}` },
];
const lines = await runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, `Rotator ${action}`, sleep, report, { environment, capture: (ls) => ls.some((l) => l.startsWith("ROTATION_RESULT ")) });
const line = lines.filter((l) => l.startsWith("ROTATION_RESULT ")).pop();
if (!line) throw new Error(`the rotator reported no result for ${action}`);
const result = JSON.parse(line.slice("ROTATION_RESULT ".length));
for (const l of lines) if (!l.startsWith("ROTATION_RESULT ")) report(` rotator: ${l}`);
return result;
};
const ops = async (label, command) => {
const { definition } = await describe(clients.ecs, `${config.prefix}-ops`);
return runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, label, sleep, report, { command });
};
const roll = async (versions) => {
const names = Object.keys(versions || {}).filter((s) => SERVICES.includes(s));
if (!names.length) return;
report(`Rolling ${names.join(", ")} onto the new secret versions`);
await deploy(clients, { ...config, run: step(), services: names, secretVersions: versions }, sleep, report);
};
const nodesCurrent = async () => {
const checks = Math.floor(config.nodeWaitMs / 120000) + 1;
for (let i = 1; ; i++) {
try {
await ops("Fleet env status", ["fleet", "env-status"]);
report("Every live node has applied the new env");
return true;
} catch {
if (i >= checks) return false;
report("Some nodes have not applied the new env yet; checking again in two minutes");
await sleep(120000);
}
}
};
const resealed = async () => {
await ops("Reseal", ["keys", "reseal", "--apply"]);
try {
await ops("Key status", ["keys", "status", "--require-primary"]);
return true;
} catch {
return false;
}
};
const retire = async () => {
const result = await rotator("retire");
await roll(result.versions);
report(`${config.key} rotated; the previous value is retired`);
};
// Nodes first: a node still on the old env would seal under the old key after the reseal.
const gate = async () => {
if ((config.kind === "token" || config.key === "CREDENTIALS_ENCRYPTION_KEY") && !(await nodesCurrent())) {
return "some nodes have not applied the new env and still work on the previous value. Run retire once `warmblyctl fleet env-status` is clean";
}
if (config.kind === "seal" && !(await resealed())) return `some values still open only under the previous ${config.key}; run retire once \`warmblyctl keys status\` is clean`;
return null;
};
const status = await rotator("status");
if (status.key && status.key !== config.key) throw new Error(`${status.key} is mid-rotation; finish or abort it first`);
let phase = status.key === config.key ? status.phase : "none";
report(`${config.key}: ${phase}`);
// A run that died after writing resumes here: rolling versions already live is a no-op.
const resume = () => roll(status.pinned);
switch (config.action) {
case "status":
if (config.kind === "seal") await ops("Key status", ["keys", "status"]).catch(() => report("keys status reports unreadable values"));
return { phase };
case "prepare": {
const result = await rotator("prepare");
await roll(result.versions);
report(`Overlap keys exist for ${config.key}. Add them to secret_keys for ${result.holders.join(", ")} and apply the runtime Terraform before rotating.`);
return { phase };
}
case "abort": {
if (phase === "none") throw new Error(`no rotation of ${config.key} is in progress`);
const result = await rotator("abort");
await roll(result.versions);
report(`${config.key} restored to its previous value`);
return { phase: "none" };
}
case "retire": {
if (phase !== "staged") throw new Error(`${config.key} has no staged rotation to retire`);
await resume();
const blocked = await gate();
if (blocked) throw new Error(blocked);
await retire();
return { phase: "none" };
}
case "rotate": {
if (phase !== "none") await resume();
if (phase === "staged" && config.kind === "sign") {
report(`${config.key} is staged; run retire after ${status.until}`);
return { phase };
}
if (phase === "none") {
const result = await rotator("stage");
await roll(result.versions);
phase = result.phase;
}
if (phase === "introduced") {
if (!(await nodesCurrent())) {
report("The new key is readable everywhere except nodes that have not refreshed; run rotate again to make it primary");
return { phase };
}
const result = await rotator("stage");
await roll(result.versions);
phase = result.phase;
}
if (config.kind === "sign") {
report(`${config.key} rotated. Links signed with the previous value keep working for a year, the life of a link; run retire after that`);
return { phase };
}
const blocked = await gate();
if (blocked) {
report(`${config.key} is staged: ${blocked}`);
return { phase };
}
await retire();
return { phase: "none" };
}
}
return { phase };
}
if (require.main === module) {
const { ECSClient } = require("@aws-sdk/client-ecs");
const { SecretsManagerClient } = require("@aws-sdk/client-secrets-manager");
const { CloudWatchClient } = require("@aws-sdk/client-cloudwatch");
const { CloudWatchLogsClient } = require("@aws-sdk/client-cloudwatch-logs");
const region = { region: "us-east-1", maxAttempts: 3 };
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
const lines = [];
const report = (line) => { console.log(line); lines.push(line); };
Promise.resolve().then(() => rotateKeys(clients, rotationConfig(process.env), undefined, report)).catch((error) => {
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
console.error(message);
lines.push(`**Failed:** ${message}`);
process.exitCode = 1;
}).finally(() => {
if (process.env.GITHUB_STEP_SUMMARY) {
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Key rotation\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
}
for (const c of Object.values(clients)) c.destroy();
});
}
module.exports = { rotationConfig, rotateKeys };
+153
View File
@@ -0,0 +1,153 @@
const { test } = require("node:test");
const assert = require("node:assert/strict");
const { rotationConfig, rotateKeys } = require("./aws-rotate.cjs");
const CLUSTER = "arn:aws:ecs:us-east-1:000000000000:cluster/fixture";
const V = (c) => c.repeat(32);
// world plays the rotator, the ops task and the deploy engine against a phase machine.
function world({ key = "INTERNAL_API_TOKEN", action = "rotate", state = {}, envLagging = 0, unsealed = false, holders } = {}) {
const config = rotationConfig({ AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: action, ROTATE_KEY: key, ROTATE_NODE_WAIT_MINUTES: "4", GITHUB_RUN_ID: "9", GITHUB_RUN_ATTEMPT: "1" });
const log = [];
const tokens = new Set();
let lag = envLagging;
const owners = holders || (key.startsWith("CREDENTIALS") ? ["backend", "consumer"] : ["backend", "tracking"]);
const versions = () => Object.fromEntries(owners.map((s, i) => [s, V(String(i + 1))]));
const rotator = (act) => {
log.push(`rotator ${act}`);
if (act === "status") return { phase: state.phase || "none", key: state.key || null, versions: {}, pinned: state.key ? versions() : {}, until: state.until || null };
if (act === "stage") {
state = key === "CREDENTIALS_ENCRYPTION_KEY" && !state.key ? { key, phase: "introduced" } : { key, phase: "staged" };
return { phase: state.phase, versions: versions(), holders: owners };
}
state = {};
return { phase: "none", versions: act === "prepare" ? { backend: V("p") } : versions(), holders: owners };
};
const deps = {
async describe(_, family) {
return { definition: { taskDefinitionArn: `${family}:1`, family, containerDefinitions: [{ name: family.split("-").pop() }] } };
},
async runTask(_, cfg, network, arn, definition, label, sleep, report, options) {
assert.ok(network.awsvpcConfiguration);
const token = `${cfg.run}/${label}`;
assert.ok(!tokens.has(token), `run token reused for ${label}`);
tokens.add(token);
if (arn.startsWith("fixture-rotator")) {
const act = options.environment.find((e) => e.name === "ROTATE_ACTION").value;
assert.equal(options.environment.find((e) => e.name === "ROTATE_KEY").value, key);
const lines = ["phase written", `ROTATION_RESULT ${JSON.stringify(rotator(act))}`];
assert.ok(options.capture(lines) && !options.capture(["phase written"]));
return lines;
}
const command = options.command.join(" ");
log.push(`ops ${command}`);
if (command === "fleet env-status" && lag-- > 0) throw new Error("not current");
if (command === "keys status --require-primary" && unsealed) throw new Error("on read key");
return [];
},
async deploy(_, cfg) {
assert.equal(cfg.mode, "refresh-secrets");
assert.equal(cfg.fleet, false);
assert.equal(cfg.tag, "");
log.push(`roll ${cfg.services.join(",")}`);
return {};
},
};
const clients = { ecs: { async send() {
return { services: [{ networkConfiguration: { awsvpcConfiguration: { subnets: ["s"] } } }] };
} } };
const report = [];
const run = () => rotateKeys(clients, config, async () => {}, (l) => report.push(l), deps);
return { run, log, report, state: () => state };
}
test("configuration refuses unknown keys, actions and waits", () => {
const base = { AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: "rotate", ROTATE_KEY: "INTERNAL_API_TOKEN" };
assert.throws(() => rotationConfig({ ...base, ROTATE_KEY: "AUTH_SECRET" }), /ROTATE_KEY/);
assert.throws(() => rotationConfig({ ...base, ROTATE_ACTION: "delete" }), /ROTATE_ACTION/);
assert.throws(() => rotationConfig({ ...base, ROTATE_NODE_WAIT_MINUTES: "999" }), /0 through 240/);
assert.equal(rotationConfig(base).kind, "token");
});
test("a token is staged receiver first, waits for nodes, then retires in one run", async () => {
const w = world({ envLagging: 1 });
assert.deepEqual(await w.run(), { phase: "none" });
assert.deepEqual(w.log, [
"rotator status", "rotator stage", "roll backend,tracking",
"ops fleet env-status", "ops fleet env-status", "rotator retire", "roll backend,tracking",
]);
});
test("a token stays staged when nodes never catch up, and says how to finish", async () => {
const w = world({ envLagging: 99 });
assert.deepEqual(await w.run(), { phase: "staged" });
assert.ok(!w.log.includes("rotator retire"));
assert.equal(w.log.filter((l) => l === "ops fleet env-status").length, 3);
assert.match(w.report.at(-1), /is staged: .*fleet env-status/);
});
test("the credentials key is introduced, made primary once nodes read it, resealed, then retired", async () => {
const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY" });
assert.deepEqual(await w.run(), { phase: "none" });
assert.deepEqual(w.log, [
"rotator status", "rotator stage", "roll backend,consumer",
"ops fleet env-status", "rotator stage", "roll backend,consumer",
"ops fleet env-status", "ops keys reseal --apply", "ops keys status --require-primary",
"rotator retire", "roll backend,consumer",
]);
});
test("the credentials key waits at introduced when a node has not refreshed", async () => {
const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY", envLagging: 99 });
assert.deepEqual(await w.run(), { phase: "introduced" });
assert.equal(w.log.filter((l) => l === "rotator stage").length, 1);
});
test("a seal key is not retired while values still need the old key", async () => {
const w = world({ key: "TWOFA_SECRET", unsealed: true, holders: ["backend"] });
assert.deepEqual(await w.run(), { phase: "staged" });
assert.ok(!w.log.includes("ops fleet env-status"));
assert.ok(!w.log.includes("rotator retire"));
assert.match(w.report.at(-1), /keys status/);
});
test("a signing secret stops at staged", async () => {
const w = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"] });
assert.deepEqual(await w.run(), { phase: "staged" });
assert.deepEqual(w.log, ["rotator status", "rotator stage", "roll backend"]);
assert.match(w.report.at(-1), /for a year/);
});
test("another key mid-rotation refuses before anything is written", async () => {
const w = world({ state: { key: "NODE_BROKER_TOKEN", phase: "staged" } });
await assert.rejects(w.run(), /NODE_BROKER_TOKEN is mid-rotation/);
assert.deepEqual(w.log, ["rotator status"]);
});
test("retire and abort refuse without a rotation and roll when there is one", async () => {
await assert.rejects(world({ action: "retire" }).run(), /no staged rotation/);
await assert.rejects(world({ action: "abort" }).run(), /no rotation/);
const abort = world({ action: "abort", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
await abort.run();
assert.deepEqual(abort.log, ["rotator status", "rotator abort", "roll backend,tracking"]);
const retire = world({ action: "retire", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
await retire.run();
assert.deepEqual(retire.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]);
});
test("prepare rolls only what changed and says to reference the keys in Terraform", async () => {
const w = world({ action: "prepare" });
await w.run();
assert.deepEqual(w.log, ["rotator status", "rotator prepare", "roll backend"]);
assert.match(w.report.at(-1), /secret_keys/);
});
test("rotate resumes a run that stopped after staging instead of refusing", async () => {
const w = world({ state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
assert.deepEqual(await w.run(), { phase: "none" });
assert.deepEqual(w.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]);
const sign = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"], state: { key: "UNSUBSCRIBE_SIGNING_SECRET", phase: "staged", until: "2027-10-25T00:00:00.000Z" } });
assert.deepEqual(await sign.run(), { phase: "staged" });
assert.deepEqual(sign.log, ["rotator status", "roll backend"]);
assert.match(sign.report.at(-1), /after 2027-10-25/);
});
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env bash
# Plans or applies one AWS stack without putting a value in a public log:
# var files come from the private state bucket and are masked first, plan and
# apply output goes to a private file, and only resource addresses and actions
# are printed.
#
# terraform-stack.sh plan <foundation|runtime|workers>
# terraform-stack.sh apply <stack> applies the plan saved by plan
# terraform-stack.sh classify <plan.json>
#
# plan writes $WORK/<stack>.summary (one "action address" per line) and prints
# safe, gated or none. Safe means only creates, in-place updates, and new
# revisions of task definitions, which keep their old revision.
set -euo pipefail
classify() {
jq -r '
[.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"])] as $c |
if ($c | length) == 0 then "none"
elif any($c[]; (.change.actions | index("delete")) and
((.type == "aws_ecs_task_definition" and (.change.actions | length) == 2) | not)) then "gated"
else "safe" end' "$1"
}
summary() {
jq -r '.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"]) |
"\(.change.actions | join("/")) \(.address)"' "$1" | sort
}
main() {
local action=$1 stack=${2:-}
if [ "$action" = classify ]; then classify "$stack"; return 0; fi
case "$stack" in foundation|runtime|workers) ;; *) echo "unknown stack $stack" >&2; return 1 ;; esac
: "${STATE_BUCKET:?}" "${WORK:?}"
local dir="deploy/terraform/aws/$stack" vars="$WORK/$stack.tfvars.json" out="$WORK/$stack.log"
mkdir -p "$WORK" && chmod 700 "$WORK"
if [ "$action" = plan ]; then
aws s3 cp --only-show-errors "s3://$STATE_BUCKET/warmbly/production/vars/$stack.tfvars.json" "$vars"
chmod 600 "$vars"
# Mask every value before anything can echo one.
jq -r '.. | strings | select(length >= 6)' "$vars" | while IFS= read -r line; do
[ -n "$line" ] && echo "::add-mask::$line"
done
terraform -chdir="$dir" init -input=false -no-color -backend-config="bucket=$STATE_BUCKET" >"$out" 2>&1 || { tail -n 20 "$out" >&2; return 1; }
local code=0
terraform -chdir="$dir" plan -input=false -no-color -lock-timeout=5m -var-file="$vars" -var="state_bucket=$STATE_BUCKET" \
-out="$WORK/$stack.tfplan" -detailed-exitcode >>"$out" 2>"$WORK/$stack.err" || code=$?
if [ "$code" = 1 ]; then
echo "terraform plan failed for $stack:" >&2
grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true
return 1
fi
terraform -chdir="$dir" show -json "$WORK/$stack.tfplan" >"$WORK/$stack.json"
summary "$WORK/$stack.json" >"$WORK/$stack.summary"
classify "$WORK/$stack.json"
rm -f "$WORK/$stack.json"
return 0
fi
if [ "$action" = apply ]; then
terraform -chdir="$dir" apply -input=false -no-color -lock-timeout=5m "$WORK/$stack.tfplan" >"$out" 2>"$WORK/$stack.err" || {
echo "terraform apply failed for $stack:" >&2
grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true
return 1
}
grep -E '^Apply complete' "$out" || true
return 0
fi
echo "unknown action $action" >&2
return 1
}
main "$@"